Use the decoder's supported pix_fmts to select a suitable renderer

app/streaming/video/ffmpeg.cpp

@@ -593,27 +593,86 @@ bool FFmpegVideoDecoder::tryInitializeRenderer(AVCodec* decoder,
     return false;
 }
 
+#define TRY_PREFERRED_PIXEL_FORMAT(RENDERER_TYPE) \
+    { \
+        RENDERER_TYPE renderer; \
+        if (renderer.getPreferredPixelFormat(params->videoFormat) == decoder->pix_fmts[i]) { \
+            if (tryInitializeRenderer(decoder, params, nullptr, \
+                                      []() -> IFFmpegRenderer* { return new RENDERER_TYPE(); })) { \
+                return true; \
+            } \
+        } \
+    }
+
+#define TRY_SUPPORTED_PIXEL_FORMAT(RENDERER_TYPE) \
+    { \
+        RENDERER_TYPE renderer; \
+        if (renderer.isPixelFormatSupported(params->videoFormat, decoder->pix_fmts[i])) { \
+            if (tryInitializeRenderer(decoder, params, nullptr, \
+                                      []() -> IFFmpegRenderer* { return new RENDERER_TYPE(); })) { \
+                return true; \
+            } \
+        } \
+    }
+
+bool FFmpegVideoDecoder::tryInitializeRendererForDecoderByName(const char *decoderName,
+                                                               PDECODER_PARAMETERS params)
+{
+    AVCodec* decoder = avcodec_find_decoder_by_name(decoderName);
+    if (decoder == nullptr) {
+        return false;
+    }
+
+    if (decoder->pix_fmts == NULL) {
+        // Supported output pixel formats are unknown. We'll just try SDL and hope it can cope.
+        return tryInitializeRenderer(decoder, params, nullptr,
+                                     []() -> IFFmpegRenderer* { return new SdlRenderer(); });
+    }
+
+    // Check if any of our decoders prefer any of the pixel formats first
+    for (int i = 0; decoder->pix_fmts[i] != AV_PIX_FMT_NONE; i++) {
+#ifdef HAVE_DRM
+        TRY_PREFERRED_PIXEL_FORMAT(DrmRenderer);
+#endif
+#ifdef HAVE_MMAL
+        TRY_PREFERRED_PIXEL_FORMAT(MmalRenderer);
+#endif
+        TRY_PREFERRED_PIXEL_FORMAT(SdlRenderer);
+    }
+
+    // Nothing prefers any of them. Let's see if anyone will tolerate one.
+    for (int i = 0; decoder->pix_fmts[i] != AV_PIX_FMT_NONE; i++) {
+#ifdef HAVE_DRM
+        TRY_SUPPORTED_PIXEL_FORMAT(DrmRenderer);
+#endif
+#ifdef HAVE_MMAL
+        TRY_SUPPORTED_PIXEL_FORMAT(MmalRenderer);
+#endif
+        TRY_SUPPORTED_PIXEL_FORMAT(SdlRenderer);
+    }
+
+    // If we made it here, we couldn't find anything
+    return false;
+}
+
 bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
 {
     // Increase log level until the first frame is decoded
     av_log_set_level(AV_LOG_DEBUG);
 
     // First try decoders that the user has manually specified via environment variables.
-    // These must output surfaces in one of the formats that the SDL renderer supports,
+    // These must output surfaces in one of the formats that one of our renderers supports,
     // which is currently:
-    // - AV_PIX_FMT_YUV420P (preferred)
+    // - AV_PIX_FMT_DRM_PRIME
+    // - AV_PIX_FMT_MMAL
+    // - AV_PIX_FMT_YUV420P
     // - AV_PIX_FMT_NV12
     // - AV_PIX_FMT_NV21
-    // These formats should cover most/all decoders that output in a standard YUV format.
     {
         QString h264DecoderHint = qgetenv("H264_DECODER_HINT");
         if (!h264DecoderHint.isEmpty() && (params->videoFormat & VIDEO_FORMAT_MASK_H264)) {
             QByteArray decoderString = h264DecoderHint.toLocal8Bit();
-            AVCodec* customAvcDecoder = avcodec_find_decoder_by_name(decoderString.constData());
+            if (tryInitializeRendererForDecoderByName(decoderString.constData(), params)) {
-
-            if (customAvcDecoder != nullptr &&
-                    tryInitializeRenderer(customAvcDecoder, params, nullptr,
-                                          []() -> IFFmpegRenderer* { return new SdlRenderer(); })) {
                 SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                             "Using custom H.264 decoder (H264_DECODER_HINT): %s",
                             decoderString.constData());
@@ -630,11 +689,7 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
         QString hevcDecoderHint = qgetenv("HEVC_DECODER_HINT");
         if (!hevcDecoderHint.isEmpty() && (params->videoFormat & VIDEO_FORMAT_MASK_H265)) {
             QByteArray decoderString = hevcDecoderHint.toLocal8Bit();
-            AVCodec* customHevcDecoder = avcodec_find_decoder_by_name(decoderString.constData());
+            if (tryInitializeRendererForDecoderByName(decoderString.constData(), params)) {
-
-            if (customHevcDecoder != nullptr &&
-                    tryInitializeRenderer(customHevcDecoder, params, nullptr,
-                                          []() -> IFFmpegRenderer* { return new SdlRenderer(); })) {
                 SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                             "Using custom HEVC decoder (HEVC_DECODER_HINT): %s",
                             decoderString.constData());
@@ -687,73 +742,22 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
 
         // Continue with special non-hwaccel hardware decoders
 
-#ifdef HAVE_MMAL
-        // MMAL is the decoder for the Raspberry Pi
         if (params->videoFormat & VIDEO_FORMAT_MASK_H264) {
-            AVCodec* mmalDecoder = avcodec_find_decoder_by_name("h264_mmal");
+            QList<const char *> knownAvcCodecs = { "h264_mmal", "h264_rkmpp", "h264_nvmpi", "h264_v4l2m2m" };
-            if (mmalDecoder != nullptr &&
+            for (const char* codec : knownAvcCodecs) {
-                    tryInitializeRenderer(mmalDecoder, params, nullptr,
+                if (tryInitializeRendererForDecoderByName(codec, params)) {
-                                          []() -> IFFmpegRenderer* { return new MmalRenderer(); })) {
+                    return true;
-                return true;
+                }
             }
         }
-#endif
+        else {
-
+            QList<const char *> knownHevcCodecs = { "hevc_rkmpp", "hevc_nvmpi", "hevc_v4l2m2m" };
-#ifdef HAVE_DRM
+            for (const char* codec : knownHevcCodecs) {
-        {
+                if (tryInitializeRendererForDecoderByName(codec, params)) {
-            // RKMPP is a hardware accelerated decoder that outputs DRI PRIME buffers
+                    return true;
-            AVCodec* rkmppDecoder;
+                }
-
-            if (params->videoFormat & VIDEO_FORMAT_MASK_H264) {
-                rkmppDecoder = avcodec_find_decoder_by_name("h264_rkmpp");
-            }
-            else {
-                rkmppDecoder = avcodec_find_decoder_by_name("hevc_rkmpp");
-            }
-
-            if (rkmppDecoder != nullptr &&
-                    tryInitializeRenderer(rkmppDecoder, params, nullptr,
-                                          []() -> IFFmpegRenderer* { return new DrmRenderer(); })) {
-                return true;
             }
         }
-#endif
-
-#ifdef Q_OS_LINUX
-        {
-            AVCodec* nvmpiDecoder;
-
-            if (params->videoFormat & VIDEO_FORMAT_MASK_H264) {
-                nvmpiDecoder = avcodec_find_decoder_by_name("h264_nvmpi");
-            }
-            else {
-                nvmpiDecoder = avcodec_find_decoder_by_name("hevc_nvmpi");
-            }
-
-            if (nvmpiDecoder != nullptr &&
-                    tryInitializeRenderer(nvmpiDecoder, params, nullptr,
-                                          []() -> IFFmpegRenderer* { return new SdlRenderer(); })) {
-                return true;
-            }
-        }
-
-        {
-            AVCodec* v4l2Decoder;
-
-            if (params->videoFormat & VIDEO_FORMAT_MASK_H264) {
-                v4l2Decoder = avcodec_find_decoder_by_name("h264_v4l2m2m");
-            }
-            else {
-                v4l2Decoder = avcodec_find_decoder_by_name("hevc_v4l2m2m");
-            }
-
-            if (v4l2Decoder != nullptr &&
-                    tryInitializeRenderer(v4l2Decoder, params, nullptr,
-                                          []() -> IFFmpegRenderer* { return new SdlRenderer(); })) {
-                return true;
-            }
-        }
-#endif
 
         // Look for the first matching hwaccel hardware decoder (pass 1)
         // This picks up "second-tier" hwaccels like CUDA.

app/streaming/video/ffmpeg.h

@@ -36,6 +36,9 @@ private:
 
     bool createFrontendRenderer(PDECODER_PARAMETERS params);
 
+    bool tryInitializeRendererForDecoderByName(const char* decoderName,
+                                               PDECODER_PARAMETERS params);
+
     bool tryInitializeRenderer(AVCodec* decoder,
                                PDECODER_PARAMETERS params,
                                const AVCodecHWConfig* hwConfig,