Evaluate D3D9 quirks before calling CreateDeviceEx()

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -369,12 +369,12 @@ bool DXVA2Renderer::initializeRenderer()
     return true;
 }
 
-bool DXVA2Renderer::initializeDeviceQuirks()
+bool DXVA2Renderer::initializeQuirksForAdapter(IDirect3D9Ex* d3d9ex, int adapterIndex)
 {
-    IDirect3D9* d3d9;
     HRESULT hr;
 
     SDL_assert(m_DeviceQuirks == 0);
+    SDL_assert(m_Device == nullptr);
 
     {
         bool ok;
@@ -388,11 +388,7 @@ bool DXVA2Renderer::initializeDeviceQuirks()
         }
     }
 
-    hr = m_Device->GetDirect3D(&d3d9);
+    UINT adapterCount = d3d9ex->GetAdapterCount();
-    if (SUCCEEDED(hr)) {
-        D3DCAPS9 caps;
-
-        UINT adapterCount = d3d9->GetAdapterCount();
     if (adapterCount > 1) {
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                     "Detected multi-GPU system with %d GPUs",
@@ -400,11 +396,12 @@ bool DXVA2Renderer::initializeDeviceQuirks()
         m_DeviceQuirks |= DXVA2_QUIRK_MULTI_GPU;
     }
 
-        hr = m_Device->GetDeviceCaps(&caps);
+    D3DCAPS9 caps;
+    hr = d3d9ex->GetDeviceCaps(adapterIndex, D3DDEVTYPE_HAL, &caps);
     if (SUCCEEDED(hr)) {
         D3DADAPTER_IDENTIFIER9 id;
 
-            hr = d3d9->GetAdapterIdentifier(caps.AdapterOrdinal, 0, &id);
+        hr = d3d9ex->GetAdapterIdentifier(adapterIndex, 0, &id);
         if (SUCCEEDED(hr)) {
             if (id.VendorId == 0x8086) {
                 SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
@@ -459,13 +456,6 @@ bool DXVA2Renderer::initializeDeviceQuirks()
                      "GetDeviceCaps() failed: %x", hr);
     }
 
-        d3d9->Release();
-    }
-    else {
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "GetDirect3D() failed: %x", hr);
-    }
-
     return false;
 }
 
@@ -564,6 +554,48 @@ bool DXVA2Renderer::initializeDevice(SDL_Window* window, bool enableVsync)
     int adapterIndex = SDL_Direct3D9GetAdapterIndex(SDL_GetWindowDisplayIndex(window));
     Uint32 windowFlags = SDL_GetWindowFlags(window);
 
+    // Initialize quirks *before* calling CreateDeviceEx() to allow our below
+    // logic to avoid a hang with NahimicOSD.dll's broken full-screen handling.
+    if (!initializeQuirksForAdapter(d3d9ex, adapterIndex)) {
+        d3d9ex->Release();
+        return false;
+    }
+
+    // If we have a WDDM 2.0 or later display driver and we're not running in
+    // full-screen exclusive mode (or we're on a multi-GPU system in FSE),
+    // prefer the D3D11VA renderer.
+    //
+    // D3D11VA is better in this case because it can enable tearing in non-FSE
+    // modes when the user has V-Sync disabled. In non-FSE V-Sync cases, D3D11VA
+    // provides lower display latency on systems that support Independent Flip
+    // in windowed mode. When using D3D9, DWM will not promote us to IFlip unless
+    // we're full-screen (exclusive or not).
+    //
+    // We prefer D3D11VA in FSE multi-GPU cases due to a plethora of issues with
+    // D3D9Ex PresentEx()/D3DPRESENT_DONOTWAIT on hybrid graphics systems (See
+    // issues #235, #240, #386, and #951 on GitHub). Clearly this codepath is not
+    // well tested by Microsoft or GPU vendors, so stick to the more common
+    // D3D11-based renderer which is much more likely to behave.
+    //
+    // NB: The reason we only do this for WDDM 2.0 and later is because older
+    // AMD drivers (such as those for the HD 5570) render garbage when using
+    // the D3D11VA renderer.
+    if (m_DecoderSelectionPass == 0 &&
+            (m_DeviceQuirks & DXVA2_QUIRK_WDDM_20_PLUS)) {
+        if (!((SDL_GetWindowFlags(window) & SDL_WINDOW_FULLSCREEN_DESKTOP) == SDL_WINDOW_FULLSCREEN)) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Defaulting to D3D11VA for non-FSE mode");
+            d3d9ex->Release();
+            return false;
+        }
+        else if (m_DeviceQuirks & DXVA2_QUIRK_MULTI_GPU) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Defaulting to D3D11VA for multi-GPU FSE mode");
+            d3d9ex->Release();
+            return false;
+        }
+    }
+
     D3DCAPS9 deviceCaps;
     d3d9ex->GetDeviceCaps(adapterIndex, D3DDEVTYPE_HAL, &deviceCaps);
 
@@ -770,43 +802,6 @@ bool DXVA2Renderer::initialize(PDECODER_PARAMETERS params)
         return false;
     }
 
-    if (!initializeDeviceQuirks()) {
-        return false;
-    }
-
-    // If we have a WDDM 2.0 or later display driver and we're not running in
-    // full-screen exclusive mode (or we're on a multi-GPU system in FSE),
-    // prefer the D3D11VA renderer.
-    //
-    // D3D11VA is better in this case because it can enable tearing in non-FSE
-    // modes when the user has V-Sync disabled. In non-FSE V-Sync cases, D3D11VA
-    // provides lower display latency on systems that support Independent Flip
-    // in windowed mode. When using D3D9, DWM will not promote us to IFlip unless
-    // we're full-screen (exclusive or not).
-    //
-    // We prefer D3D11VA in FSE multi-GPU cases due to a plethora of issues with
-    // D3D9Ex PresentEx()/D3DPRESENT_DONOTWAIT on hybrid graphics systems (See
-    // issues #235, #240, #386, and #951 on GitHub). Clearly this codepath is not
-    // well tested by Microsoft or GPU vendors, so stick to the more common
-    // D3D11-based renderer which is much more likely to behave.
-    //
-    // NB: The reason we only do this for WDDM 2.0 and later is because older
-    // AMD drivers (such as those for the HD 5570) render garbage when using
-    // the D3D11VA renderer.
-    if (m_DecoderSelectionPass == 0 &&
-            (m_DeviceQuirks & DXVA2_QUIRK_WDDM_20_PLUS)) {
-        if (!((SDL_GetWindowFlags(params->window) & SDL_WINDOW_FULLSCREEN_DESKTOP) == SDL_WINDOW_FULLSCREEN)) {
-            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                        "Defaulting to D3D11VA for non-FSE mode");
-            return false;
-        }
-        else if (m_DeviceQuirks & DXVA2_QUIRK_MULTI_GPU) {
-            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                        "Defaulting to D3D11VA for multi-GPU FSE mode");
-            return false;
-        }
-    }
-
     if (!initializeDecoder()) {
         return false;
     }

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -27,7 +27,7 @@ private:
     bool initializeRenderer();
     bool initializeDevice(SDL_Window* window, bool enableVsync);
     bool isDecoderBlacklisted();
-    bool initializeDeviceQuirks();
+    bool initializeQuirksForAdapter(IDirect3D9Ex* d3d9ex, int adapterIndex);
     void renderOverlay(Overlay::OverlayType type);
 
 #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(56, 68, 0)