Avoid retrying renderer init if we know the error was not transient

app/streaming/video/ffmpeg-renderers/cuda.cpp

@@ -22,6 +22,7 @@ bool CUDARenderer::initialize(PDECODER_PARAMETERS)
 
     err = av_hwdevice_ctx_create(&m_HwContext, AV_HWDEVICE_TYPE_CUDA, nullptr, nullptr, 0);
     if (err != 0) {
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "av_hwdevice_ctx_create(CUDA) failed: %d",
                      err);

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -55,9 +55,6 @@ typedef struct _OVERLAY_VERTEX
         SDL_LOG_CATEGORY_APPLICATION, \
         "EGLRenderer: " __VA_ARGS__)
 
-SDL_Window* EGLRenderer::s_LastFailedWindow = nullptr;
-int EGLRenderer::s_LastFailedVideoFormat = 0;
-
 EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
     :
         IFFmpegRenderer(RendererType::EGL),
@@ -407,6 +404,7 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
     // https://hg.libsdl.org/SDL/rev/84618d571795
     if (!SDL_VERSION_ATLEAST(2, 0, 10)) {
         EGL_LOG(Error, "Not supported until SDL 2.0.10");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -418,15 +416,6 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
         return false;
     }
 
-    // HACK: Work around bug where renderer will repeatedly fail with:
-    // SDL_CreateRenderer() failed: Could not create GLES window surface
-    // Don't retry if we've already failed to create a renderer for this
-    // window *unless* the format has changed from 10-bit to 8-bit.
-    if (m_Window == s_LastFailedWindow) {
-        EGL_LOG(Error, "SDL_CreateRenderer() already failed on this window!");
-        return false;
-    }
-
     // This hint will ensure we use EGL to retrieve our GL context,
     // even on X11 where that is not the default. EGL is required
     // to avoid a crash in Mesa.
@@ -451,6 +440,7 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
     }
     if (renderIndex == maxRenderers) {
         EGL_LOG(Error, "Could not find a suitable SDL_Renderer");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -480,8 +470,7 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
 
     // Now we finally bail if we failed during SDL_CreateRenderer() above.
     if (!m_DummyRenderer) {
-        s_LastFailedWindow = m_Window;
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
-        s_LastFailedVideoFormat = params->videoFormat;
         return false;
     }
 
@@ -489,15 +478,18 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
     SDL_VERSION(&info.version);
     if (!SDL_GetWindowWMInfo(params->window, &info)) {
         EGL_LOG(Error, "SDL_GetWindowWMInfo() failed: %s", SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
     if (!(m_Context = SDL_GL_CreateContext(params->window))) {
         EGL_LOG(Error, "Cannot create OpenGL context: %s", SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
     if (SDL_GL_MakeCurrent(params->window, m_Context)) {
         EGL_LOG(Error, "Cannot use created EGL context: %s", SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/eglvid.h

@@ -74,9 +74,4 @@ private:
     int m_OldContextMinorVersion;
 
     SDL_Renderer *m_DummyRenderer;
-
-    // HACK: Work around bug where renderer will repeatedly fail with:
-    // SDL_CreateRenderer() failed: Could not create GLES window surface
-    static SDL_Window* s_LastFailedWindow;
-    static int s_LastFailedVideoFormat;
 };

app/streaming/video/ffmpeg-renderers/mmal.cpp

@@ -152,6 +152,7 @@ bool MmalRenderer::initialize(PDECODER_PARAMETERS params)
     MMAL_STATUS_T status;
 
     if (!isMmalOverlaySupported()) {
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -164,6 +165,7 @@ bool MmalRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "mmal_component_create() failed: %x (%s)",
                      status, mmal_status_to_string(status));
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/plvk.cpp

@@ -167,6 +167,7 @@ bool PlVkRenderer::chooseVulkanDevice(PDECODER_PARAMETERS params, bool hdrOutput
     if (physicalDeviceCount == 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "No Vulkan devices found!");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -367,6 +368,7 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_Vulkan_GetInstanceExtensions() #1 failed: %s",
                      SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -375,6 +377,7 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_Vulkan_GetInstanceExtensions() #2 failed: %s",
                      SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -390,6 +393,7 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
     if (m_PlVkInstance == nullptr) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "pl_vk_inst_create() failed");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -407,6 +411,7 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_Vulkan_CreateSurface() failed: %s",
                      SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -162,7 +162,8 @@ public:
         // Only return this reason code if the software or driver does not support
         // the specified decoding/rendering API. If the FFmpeg decoder code sees
         // this value, it will assume trying the same renderer again for any other
-        // codec will be useless and skip it.
+        // codec will be useless and skip it. This should never be set if the error
+        // could potentially be transient.
         NoSoftwareSupport,
     };
 
@@ -292,6 +293,38 @@ public:
         return m_Type;
     }
 
+    const char *getRendererName() {
+        switch (m_Type) {
+        default:
+        case RendererType::Unknown:
+            return "Unknown";
+        case RendererType::Vulkan:
+            return "Vulkan (libplacebo)";
+        case RendererType::CUDA:
+            return "CUDA";
+        case RendererType::D3D11VA:
+            return "D3D11VA";
+        case RendererType::DRM:
+            return "DRM";
+        case RendererType::DXVA2:
+            return "DXVA2 (D3D9)";
+        case RendererType::EGL:
+            return "EGL/GLES";
+        case RendererType::MMAL:
+            return "MMAL";
+        case RendererType::SDL:
+            return "SDL";
+        case RendererType::VAAPI:
+            return "VAAPI";
+        case RendererType::VDPAU:
+            return "VDPAU";
+        case RendererType::VTSampleLayer:
+            return "VideoToolbox (AVSampleBufferDisplayLayer)";
+        case RendererType::VTMetal:
+            return "VideoToolbox (Metal)";
+        }
+    }
+
     // IOverlayRenderer
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override {
         // Nothing

app/streaming/video/ffmpeg-renderers/sdlvid.cpp

@@ -186,7 +186,6 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_CreateRenderer() failed: %s",
                      SDL_GetError());
-        return false;
     }
 
     // SDL_CreateRenderer() can end up having to recreate our window (SDL_RecreateWindow())
@@ -206,6 +205,11 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_FlushEvent(SDL_WINDOWEVENT);
     }
 
+    if (!m_Renderer) {
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
+        return false;
+    }
+
 #ifdef Q_OS_WIN32
     // For some reason, using Direct3D9Ex breaks this with multi-monitor setups.
     // When focus is lost, the window is minimized then immediately restored without

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -369,6 +369,7 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to initialize VAAPI: %d",
                      status);
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -387,6 +388,7 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
         // Fail and let our VDPAU renderer pick this up
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                     "Avoiding VDPAU wrapper for VAAPI decoding");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -435,6 +437,7 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to initialize VAAPI context: %d",
                      err);
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -104,18 +104,21 @@ bool VDPAURenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_GetWindowWMInfo() failed: %s",
                      SDL_GetError());
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
     if (info.subsystem == SDL_SYSWM_WAYLAND) {
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                     "VDPAU is not supported on Wayland");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
     else if (info.subsystem != SDL_SYSWM_X11) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "VDPAU is not supported on the current subsystem: %d",
                      info.subsystem);
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
     else if (qgetenv("VDPAU_XWAYLAND") != "1" && WMUtils::isRunningWayland()) {
@@ -125,6 +128,7 @@ bool VDPAURenderer::initialize(PDECODER_PARAMETERS params)
         // https://gitlab.freedesktop.org/vdpau/libvdpau/-/issues/2
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                     "VDPAU is disabled on XWayland. Set VDPAU_XWAYLAND=1 to try your luck.");
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 
@@ -188,6 +192,7 @@ bool VDPAURenderer::initialize(PDECODER_PARAMETERS params)
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to create VDPAU context: %d",
                      err);
+        m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm

@@ -388,6 +388,7 @@ public:
             SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                         "av_hwdevice_ctx_create() failed for VT decoder: %d",
                         err);
+            m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
             return false;
         }
 

app/streaming/video/ffmpeg-renderers/vt_metal.mm

@@ -699,6 +699,7 @@ public:
 
         id<MTLDevice> device = getMetalDevice();
         if (!device) {
+            m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
             return false;
         }
 
@@ -719,6 +720,7 @@ public:
             SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                         "av_hwdevice_ctx_create() failed for VT decoder: %d",
                         err);
+            m_InitFailureReason = InitFailureReason::NoSoftwareSupport;
             return false;
         }
 

app/streaming/video/ffmpeg.cpp

@@ -311,6 +311,31 @@ void FFmpegVideoDecoder::reset()
     }
 }
 
+bool FFmpegVideoDecoder::initializeRendererInternal(IFFmpegRenderer* renderer, PDECODER_PARAMETERS params)
+{
+    if (renderer->getRendererType() != IFFmpegRenderer::RendererType::Unknown &&
+            m_FailedRenderers.find(renderer->getRendererType()) != m_FailedRenderers.end()) {
+        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                    "Skipping '%s' due to prior failure",
+                    renderer->getRendererName());
+        return false;
+    }
+
+    if (!renderer->initialize(params)) {
+        if (renderer->getInitFailureReason() == IFFmpegRenderer::InitFailureReason::NoSoftwareSupport) {
+            m_FailedRenderers.insert(renderer->getRendererType());
+
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "'%s' failed to initialize. It will not be tried again.",
+                        renderer->getRendererName());
+        }
+
+        return false;
+    }
+
+    return true;
+}
+
 bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool useAlternateFrontend)
 {
     if (useAlternateFrontend) {
@@ -320,7 +345,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
             // rendering HDR with Vulkan if possible since it's more fully featured than DRM.
             if (m_BackendRenderer->getRendererType() != IFFmpegRenderer::RendererType::Vulkan) {
                 m_FrontendRenderer = new PlVkRenderer(false, m_BackendRenderer);
-                if (m_FrontendRenderer->initialize(params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
+                if (initializeRendererInternal(m_FrontendRenderer, params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
                     return true;
                 }
                 delete m_FrontendRenderer;
@@ -335,7 +360,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
             // currently have protocols to actually get that metadata to the display).
             if (m_BackendRenderer->canExportDrmPrime()) {
                 m_FrontendRenderer = new DrmRenderer(AV_HWDEVICE_TYPE_NONE, m_BackendRenderer);
-                if (m_FrontendRenderer->initialize(params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
+                if (initializeRendererInternal(m_FrontendRenderer, params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
                     return true;
                 }
                 delete m_FrontendRenderer;
@@ -346,7 +371,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
 #if defined(HAVE_LIBPLACEBO_VULKAN) && defined(VULKAN_IS_SLOW)
             if (m_BackendRenderer->getRendererType() != IFFmpegRenderer::RendererType::Vulkan) {
                 m_FrontendRenderer = new PlVkRenderer(false, m_BackendRenderer);
-                if (m_FrontendRenderer->initialize(params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
+                if (initializeRendererInternal(m_FrontendRenderer, params) && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_HDR_SUPPORT)) {
                     return true;
                 }
                 delete m_FrontendRenderer;
@@ -360,7 +385,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
             if (qgetenv("PREFER_VULKAN") == "1") {
                 if (m_BackendRenderer->getRendererType() != IFFmpegRenderer::RendererType::Vulkan) {
                     m_FrontendRenderer = new PlVkRenderer(false, m_BackendRenderer);
-                    if (m_FrontendRenderer->initialize(params)) {
+                    if (initializeRendererInternal(m_FrontendRenderer, params)) {
                         return true;
                     }
                     delete m_FrontendRenderer;
@@ -373,7 +398,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
 #if defined(HAVE_EGL) && !defined(GL_IS_SLOW)
         if (m_BackendRenderer->canExportEGL()) {
             m_FrontendRenderer = new EGLRenderer(m_BackendRenderer);
-            if (m_FrontendRenderer->initialize(params)) {
+            if (initializeRendererInternal(m_FrontendRenderer, params)) {
                 return true;
             }
             delete m_FrontendRenderer;
@@ -396,7 +421,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
 #if (defined(VULKAN_IS_SLOW) || defined(GL_IS_SLOW)) && defined(HAVE_DRM)
         // Try DrmRenderer first if we have a slow GPU
         m_FrontendRenderer = new DrmRenderer(AV_HWDEVICE_TYPE_NONE, m_BackendRenderer);
-        if (m_FrontendRenderer->initialize(params)) {
+        if (initializeRendererInternal(m_FrontendRenderer, params)) {
             return true;
         }
         delete m_FrontendRenderer;
@@ -409,7 +434,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
         // If DRM didn't work either, try EGL now.
         if (m_BackendRenderer->canExportEGL()) {
             m_FrontendRenderer = new EGLRenderer(m_BackendRenderer);
-            if (m_FrontendRenderer->initialize(params)) {
+            if (initializeRendererInternal(m_FrontendRenderer, params)) {
                 return true;
             }
             delete m_FrontendRenderer;
@@ -419,7 +444,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
 
 #if defined(HAVE_LIBPLACEBO_VULKAN) && defined(VULKAN_IS_SLOW)
         m_FrontendRenderer = new PlVkRenderer(false, m_BackendRenderer);
-        if (m_FrontendRenderer->initialize(params)) {
+        if (initializeRendererInternal(m_FrontendRenderer, params)) {
             return true;
         }
         delete m_FrontendRenderer;
@@ -427,7 +452,7 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params, bool
 #endif
 
         m_FrontendRenderer = new SdlRenderer();
-        if (!m_FrontendRenderer->initialize(params)) {
+        if (!initializeRendererInternal(m_FrontendRenderer, params)) {
             return false;
         }
     }
@@ -653,6 +678,18 @@ bool FFmpegVideoDecoder::completeInitialization(const AVCodec* decoder, enum AVP
                          "Failed to create decoder thread: %s", SDL_GetError());
             return false;
         }
+
+        if (m_FrontendRenderer->getRendererType() != m_BackendRenderer->getRendererType()) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Renderer '%s' with '%s' backend chosen",
+                        m_FrontendRenderer->getRendererName(),
+                        m_BackendRenderer->getRendererName());
+        }
+        else {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Renderer '%s' chosen",
+                        m_FrontendRenderer->getRendererName());
+        }
     }
 
     return true;
@@ -1033,7 +1070,7 @@ bool FFmpegVideoDecoder::tryInitializeRenderer(const AVCodec* decoder,
 #endif
         SDL_assert(m_BackendRenderer == nullptr);
         if ((m_BackendRenderer = createRendererFunc()) != nullptr &&
-                m_BackendRenderer->initialize((m_TestOnly || m_BackendRenderer->needsTestFrame()) ? &testFrameDecoderParams : params) &&
+                initializeRendererInternal(m_BackendRenderer, (m_TestOnly || m_BackendRenderer->needsTestFrame()) ? &testFrameDecoderParams : params) &&
                 completeInitialization(decoder, requiredFormat,
                                        (m_TestOnly || m_BackendRenderer->needsTestFrame()) ? &testFrameDecoderParams : params,
                                        m_TestOnly || m_BackendRenderer->needsTestFrame(),
@@ -1048,7 +1085,7 @@ bool FFmpegVideoDecoder::tryInitializeRenderer(const AVCodec* decoder,
                 // The test worked, so now let's initialize it for real
                 reset();
                 if ((m_BackendRenderer = createRendererFunc()) != nullptr &&
-                        m_BackendRenderer->initialize(params) &&
+                        initializeRendererInternal(m_BackendRenderer, params) &&
                         completeInitialization(decoder, requiredFormat, params, false, i == 0 /* EGL/DRM */)) {
                     return true;
                 }

app/streaming/video/ffmpeg.h

@@ -2,6 +2,7 @@
 
 #include <functional>
 #include <QQueue>
+#include <set>
 
 #include "decoder.h"
 #include "ffmpeg-renderers/renderer.h"
@@ -75,6 +76,8 @@ private:
 
     static IFFmpegRenderer* createHwAccelRenderer(const AVCodecHWConfig* hwDecodeCfg, int pass);
 
+    bool initializeRendererInternal(IFFmpegRenderer* renderer, PDECODER_PARAMETERS params);
+
     void reset();
 
     void writeBuffer(PLENTRY entry, int& offset);
@@ -99,6 +102,7 @@ private:
     VIDEO_STATS m_ActiveWndVideoStats;
     VIDEO_STATS m_LastWndVideoStats;
     VIDEO_STATS m_GlobalVideoStats;
+    std::set<IFFmpegRenderer::RendererType> m_FailedRenderers;
 
     int m_FramesIn;
     int m_FramesOut;