Avoid trying VDPAU when Vulkan is preferred

The VDPAU renderer causes interoperability issues with Vulkan.

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -338,8 +338,10 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
         // Older versions of the Gallium VAAPI driver have a nasty memory leak that
         // causes memory to be leaked for each submitted frame. I believe this is
         // resolved in the libva2 drivers (VAAPI 1.x). We will try to use VDPAU
-        // instead for old VAAPI versions or drivers affected by the RFI latency bug.
+        // instead for old VAAPI versions or drivers affected by the RFI latency bug
-        if ((major == 0 || m_HasRfiLatencyBug) && vendorStr.contains("Gallium", Qt::CaseInsensitive)) {
+        // as long as we're not streaming HDR (which is unsupported by VDPAU).
+        if ((major == 0 || (m_HasRfiLatencyBug && !(m_VideoFormat & VIDEO_FORMAT_MASK_10BIT))) &&
+                vendorStr.contains("Gallium", Qt::CaseInsensitive)) {
             // Fail and let VDPAU pick this up
             SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                         "Deprioritizing VAAPI on Gallium driver. Set FORCE_VAAPI=1 to override.");

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -24,8 +24,9 @@ const VdpRGBAFormat VDPAURenderer::k_OutputFormats10Bit[] = {
     VDP_RGBA_FORMAT_R10G10B10A2
 };
 
-VDPAURenderer::VDPAURenderer()
+VDPAURenderer::VDPAURenderer(int decoderSelectionPass)
-    : m_HwContext(nullptr),
+    : m_DecoderSelectionPass(decoderSelectionPass),
+      m_HwContext(nullptr),
       m_PresentationQueueTarget(0),
       m_PresentationQueue(0),
       m_VideoMixer(0),
@@ -94,6 +95,23 @@ bool VDPAURenderer::initialize(PDECODER_PARAMETERS params)
     #endif
     };
 
+    // Avoid initializing VDPAU on this window on the first selection pass if:
+    // a) We know we want HDR compatibility
+    // b) The user wants to prefer Vulkan
+    //
+    // Using VDPAU may lead to side-effects that break our attempts to create
+    // a Vulkan swapchain on this window later.
+    if (m_DecoderSelectionPass == 0) {
+        if (params->videoFormat & VIDEO_FORMAT_MASK_10BIT) {
+            return false;
+        }
+        else if (qgetenv("PREFER_VULKAN") == "1") {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Deprioritizing Vulkan-incompatible VDPAU renderer due to PREFER_VULKAN=1");
+            return false;
+        }
+    }
+
     SDL_VERSION(&info.version);
 
     if (!SDL_GetWindowWMInfo(params->window, &info)) {

app/streaming/video/ffmpeg-renderers/vdpau.h

@@ -11,7 +11,7 @@ extern "C" {
 class VDPAURenderer : public IFFmpegRenderer
 {
 public:
-    VDPAURenderer();
+    VDPAURenderer(int decoderSelectionPass);
     virtual ~VDPAURenderer() override;
     virtual bool initialize(PDECODER_PARAMETERS params) override;
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
@@ -25,6 +25,7 @@ public:
 private:
     void renderOverlay(VdpOutputSurface destination, Overlay::OverlayType type);
 
+    int m_DecoderSelectionPass;
     uint32_t m_VideoWidth, m_VideoHeight;
     uint32_t m_DisplayWidth, m_DisplayHeight;
     AVBufferRef* m_HwContext;

app/streaming/video/ffmpeg.cpp

@@ -830,7 +830,7 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
 #endif
 #ifdef HAVE_LIBVDPAU
         case AV_HWDEVICE_TYPE_VDPAU:
-            return new VDPAURenderer();
+            return new VDPAURenderer(pass);
 #endif
 #ifdef HAVE_DRM
         case AV_HWDEVICE_TYPE_DRM:
@@ -872,7 +872,7 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
 #endif
 #ifdef HAVE_LIBVDPAU
         case AV_HWDEVICE_TYPE_VDPAU:
-            return new VDPAURenderer();
+            return new VDPAURenderer(pass);
 #endif
         default:
             return nullptr;