Add an enum type for each renderer

app/streaming/video/ffmpeg-renderers/cuda.cpp

@@ -3,7 +3,8 @@
 #include <SDL_opengl.h>
 
 CUDARenderer::CUDARenderer()
-    : m_HwContext(nullptr)
+    : IFFmpegRenderer(RendererType::CUDA),
+      m_HwContext(nullptr)
 {
 
 }

app/streaming/video/ffmpeg-renderers/d3d11va.cpp

@@ -87,7 +87,8 @@ static const std::array<const char*, D3D11VARenderer::PixelShaders::_COUNT> k_Vi
 };
 
 D3D11VARenderer::D3D11VARenderer(int decoderSelectionPass)
-    : m_DecoderSelectionPass(decoderSelectionPass),
+    : IFFmpegRenderer(RendererType::D3D11VA),
+      m_DecoderSelectionPass(decoderSelectionPass),
       m_DevicesWithFL11Support(0),
       m_DevicesWithCodecSupport(0),
       m_LastColorSpace(-1),

app/streaming/video/ffmpeg-renderers/drm.cpp

@@ -135,7 +135,8 @@ static const std::map<AVPixelFormat, uint32_t> k_AvToDrmFormatMap
 };
 
 DrmRenderer::DrmRenderer(AVHWDeviceType hwDeviceType, IFFmpegRenderer *backendRenderer)
-    : m_BackendRenderer(backendRenderer),
+    : IFFmpegRenderer(RendererType::DRM),
+      m_BackendRenderer(backendRenderer),
       m_DrmPrimeBackend(backendRenderer && backendRenderer->canExportDrmPrime()),
       m_HwDeviceType(hwDeviceType),
       m_HwContext(nullptr),

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -36,6 +36,7 @@ typedef struct _VERTEX
 } VERTEX, *PVERTEX;
 
 DXVA2Renderer::DXVA2Renderer(int decoderSelectionPass) :
+    IFFmpegRenderer(RendererType::DXVA2),
     m_DecoderSelectionPass(decoderSelectionPass),
     m_SurfacesUsed(0),
     m_Pool(nullptr),

app/streaming/video/ffmpeg-renderers/egl_extensions.cpp

@@ -1,8 +1,5 @@
-// vim: noai:ts=4:sw=4:softtabstop=4:expandtab
 #include "renderer.h"
 
-#include <SDL_egl.h>
-
 static QStringList egl_get_extensions(EGLDisplay dpy) {
     const auto EGLExtensionsStr = eglQueryString(dpy, EGL_EXTENSIONS);
     if (!EGLExtensionsStr) {

app/streaming/video/ffmpeg-renderers/eglimagefactory.h

@@ -2,9 +2,6 @@
 
 #include "renderer.h"
 
-#define SDL_USE_BUILTIN_OPENGL_DEFINITIONS 1
-#include <SDL_egl.h>
-
 #ifdef HAVE_LIBVA
 #include <va/va_drmcommon.h>
 #endif

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -10,7 +10,6 @@
 #include <Limelight.h>
 #include <unistd.h>
 
-#include <SDL_render.h>
 #include <SDL_syswm.h>
 
 // These are extensions, so some platform headers may not provide them
@@ -61,6 +60,7 @@ int EGLRenderer::s_LastFailedVideoFormat = 0;
 
 EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
     :
+        IFFmpegRenderer(RendererType::EGL),
         m_EGLImagePixelFormat(AV_PIX_FMT_NONE),
         m_EGLDisplay(EGL_NO_DISPLAY),
         m_Textures{0},

app/streaming/video/ffmpeg-renderers/genhwaccel.cpp

@@ -1,7 +1,8 @@
 #include "genhwaccel.h"
 
 GenericHwAccelRenderer::GenericHwAccelRenderer(AVHWDeviceType hwDeviceType)
-    : m_HwDeviceType(hwDeviceType),
+    : IFFmpegRenderer(RendererType::Unknown),
+      m_HwDeviceType(hwDeviceType),
       m_HwContext(nullptr)
 {
 

app/streaming/video/ffmpeg-renderers/mmal.cpp

@@ -16,7 +16,8 @@
 #include <QTextStream>
 
 MmalRenderer::MmalRenderer()
-    : m_Renderer(nullptr),
+    : IFFmpegRenderer(RendererType::MMAL),
+      m_Renderer(nullptr),
       m_InputPort(nullptr),
       m_BackgroundRenderer(nullptr),
       m_Window(nullptr),

app/streaming/video/ffmpeg-renderers/plvk.cpp

@@ -99,6 +99,7 @@ void PlVkRenderer::overlayUploadComplete(void* opaque)
 }
 
 PlVkRenderer::PlVkRenderer(bool hwaccel, IFFmpegRenderer *backendRenderer) :
+    IFFmpegRenderer(RendererType::Vulkan),
     m_Backend(backendRenderer),
     m_HwAccelBackend(hwaccel)
 {
@@ -1088,8 +1089,3 @@ AVPixelFormat PlVkRenderer::getPreferredPixelFormat(int videoFormat)
         return AV_PIX_FMT_VULKAN;
     }
 }
-
-IFFmpegRenderer::RendererType PlVkRenderer::getRendererType()
-{
-    return IFFmpegRenderer::RendererType::Vulkan;
-}

app/streaming/video/ffmpeg-renderers/plvk.h

@@ -29,7 +29,6 @@ public:
     virtual bool needsTestFrame() override;
     virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) override;
     virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) override;
-    virtual RendererType getRendererType() override;
 
 private:
     static void lockQueue(AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index);

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -124,6 +124,24 @@ private:
 
 class IFFmpegRenderer : public Overlay::IOverlayRenderer {
 public:
+    enum class RendererType {
+        Unknown,
+        Vulkan,
+        CUDA,
+        D3D11VA,
+        DRM,
+        DXVA2,
+        EGL,
+        MMAL,
+        SDL,
+        VAAPI,
+        VDPAU,
+        VTSampleLayer,
+        VTMetal,
+    };
+
+    IFFmpegRenderer(RendererType type) : m_Type(type) {}
+
     virtual bool initialize(PDECODER_PARAMETERS params) = 0;
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) = 0;
     virtual void renderFrame(AVFrame* frame) = 0;
@@ -140,10 +158,16 @@ public:
         // where trying additional hwaccels may be undesirable since it could lead
         // to incorrectly skipping working hwaccels.
         NoHardwareSupport,
+
+        // Only return this reason code if the software or driver does not support
+        // the specified decoding/rendering API. If the FFmpeg decoder code sees
+        // this value, it will assume trying the same renderer again for any other
+        // codec will be useless and skip it.
+        NoSoftwareSupport,
     };
 
     virtual InitFailureReason getInitFailureReason() {
-        return InitFailureReason::Unknown;
+        return m_InitFailureReason;
     }
 
     // Called for threaded renderers to allow them to wait prior to us latching
@@ -264,13 +288,8 @@ public:
         // preparations might include clearing the window.
     }
 
-    // Allow renderers to expose their type
+    RendererType getRendererType() {
-    enum class RendererType {
+        return m_Type;
-        Unknown,
-        Vulkan
-    };
-    virtual RendererType getRendererType() {
-        return RendererType::Unknown;
     }
 
     // IOverlayRenderer
@@ -315,4 +334,10 @@ public:
 
     virtual void unmapDrmPrimeFrame(AVDRMFrameDescriptor*) {}
 #endif
+
+protected:
+    InitFailureReason m_InitFailureReason;
+
+private:
+    RendererType m_Type;
 };

app/streaming/video/ffmpeg-renderers/sdlvid.cpp

@@ -13,7 +13,8 @@ extern "C" {
 }
 
 SdlRenderer::SdlRenderer()
-    : m_VideoFormat(0),
+    : IFFmpegRenderer(RendererType::SDL),
+      m_VideoFormat(0),
       m_Renderer(nullptr),
       m_Texture(nullptr),
       m_ColorSpace(-1),

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -19,7 +19,8 @@
 #include <fcntl.h>
 
 VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass)
-    : m_DecoderSelectionPass(decoderSelectionPass),
+    : IFFmpegRenderer(RendererType::VAAPI),
+      m_DecoderSelectionPass(decoderSelectionPass),
       m_HwContext(nullptr),
       m_BlacklistedForDirectRendering(false),
       m_RequiresExplicitPixelFormat(false),

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -25,7 +25,8 @@ const VdpRGBAFormat VDPAURenderer::k_OutputFormats10Bit[] = {
 };
 
 VDPAURenderer::VDPAURenderer(int decoderSelectionPass)
-    : m_DecoderSelectionPass(decoderSelectionPass),
+    : IFFmpegRenderer(RendererType::VDPAU),
+      m_DecoderSelectionPass(decoderSelectionPass),
       m_HwContext(nullptr),
       m_PresentationQueueTarget(0),
       m_PresentationQueue(0),

app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm

@@ -35,7 +35,8 @@ class VTRenderer : public VTBaseRenderer
 {
 public:
     VTRenderer()
-        : m_HwContext(nullptr),
+        : IFFmpegRenderer(RendererType::VTSampleLayer),
+          m_HwContext(nullptr),
           m_DisplayLayer(nullptr),
           m_FormatDesc(nullptr),
           m_ContentLightLevelInfo(nullptr),

app/streaming/video/ffmpeg-renderers/vt_metal.mm

@@ -113,7 +113,8 @@ class VTMetalRenderer : public VTBaseRenderer
 {
 public:
     VTMetalRenderer(bool hwAccel)
-        : m_HwAccel(hwAccel),
+        : IFFmpegRenderer(RendererType::VTMetal),
+          m_HwAccel(hwAccel),
           m_Window(nullptr),
           m_HwContext(nullptr),
           m_MetalLayer(nullptr),