Validate that the codec is functional before using it. Fixes streaming failure with VAAPI if hardware lacks codec support

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -75,7 +75,6 @@ bool DXVA2Renderer::prepareDecoderContext(AVCodecContext* context)
 
     context->hwaccel_context = &m_DXVAContext;
 
-    context->get_format = ffGetFormat;
     context->get_buffer2 = ffGetBuffer2;
 
     context->opaque = this;
@@ -110,21 +109,6 @@ int DXVA2Renderer::ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int)
     return 0;
 }
 
-enum AVPixelFormat DXVA2Renderer::ffGetFormat(AVCodecContext*,
-                                              const enum AVPixelFormat* pixFmts)
-{
-    const enum AVPixelFormat *p;
-
-    for (p = pixFmts; *p != -1; p++) {
-        if (*p == AV_PIX_FMT_DXVA2_VLD)
-            return *p;
-    }
-
-    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                 "Failed to find DXVA2 HW surface format");
-    return AV_PIX_FMT_NONE;
-}
-
 bool DXVA2Renderer::initializeDecoder()
 {
     HRESULT hr;

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -35,10 +35,6 @@ private:
     static
     int ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int flags);
 
-    static
-    enum AVPixelFormat ffGetFormat(AVCodecContext*,
-                                   const enum AVPixelFormat* pixFmts);
-
     int m_VideoFormat;
     int m_VideoWidth;
     int m_VideoHeight;

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -38,6 +38,10 @@ public:
     virtual void renderFrame(AVFrame* frame);
 
 private:
+    static
+    enum AVPixelFormat ffGetFormat(AVCodecContext*,
+                                   const enum AVPixelFormat* pixFmts);
+
     Window m_XWindow;
     AVBufferRef* m_HwContext;
     void* m_X11VaLibHandle;

app/streaming/video/ffmpeg.cpp

@@ -13,93 +13,35 @@
 #include "ffmpeg-renderers/vaapi.h"
 #endif
 
-bool FFmpegVideoDecoder::chooseDecoder(
+// This is gross but it allows us to use sizeof()
-        StreamingPreferences::VideoDecoderSelection vds,
+#include "ffmpeg_videosamples.cpp"
-        SDL_Window* window,
-        int videoFormat,
-        int width, int height,
-        AVCodec*& chosenDecoder,
-        const AVCodecHWConfig*& chosenHwConfig,
-        IFFmpegRenderer*& newRenderer)
-{
-    if (videoFormat & VIDEO_FORMAT_MASK_H264) {
-        chosenDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
-    }
-    else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
-        chosenDecoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
-    }
-    else {
-        Q_ASSERT(false);
-        chosenDecoder = nullptr;
-    }
-
-    if (!chosenDecoder) {
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "Unable to find decoder for format: %x",
-                     videoFormat);
-        return false;
-    }
-
-    for (int i = 0;; i++) {
-        const AVCodecHWConfig *config = avcodec_get_hw_config(chosenDecoder, i);
-        if (!config || vds == StreamingPreferences::VDS_FORCE_SOFTWARE) {
-            // No matching hardware acceleration support.
-            // This is not an error.
-            chosenHwConfig = nullptr;
-            newRenderer = new SdlRenderer();
-            if (vds != StreamingPreferences::VDS_FORCE_HARDWARE &&
-                    newRenderer->initialize(window, videoFormat, width, height)) {
-                return true;
-            }
-            else {
-                delete newRenderer;
-                newRenderer = nullptr;
-                return false;
-            }
-        }
-
-        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
-            continue;
-        }
-
-        // Look for acceleration types we support
-        switch (config->device_type) {
-#ifdef Q_OS_WIN32
-        case AV_HWDEVICE_TYPE_DXVA2:
-            newRenderer = new DXVA2Renderer();
-            break;
-#endif
-#ifdef Q_OS_DARWIN
-        case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
-            newRenderer = VTRendererFactory::createRenderer();
-            break;
-#endif
-#ifdef HAVE_LIBVA
-        case AV_HWDEVICE_TYPE_VAAPI:
-            newRenderer = new VAAPIRenderer();
-            break;
-#endif
-        default:
-            continue;
-        }
-
-        if (newRenderer->initialize(window, videoFormat, width, height)) {
-            chosenHwConfig = config;
-            return true;
-        }
-        else {
-            // Failed to initialize
-            delete newRenderer;
-            newRenderer = nullptr;
-        }
-    }
-}
 
 bool FFmpegVideoDecoder::isHardwareAccelerated()
 {
     return m_HwDecodeCfg != nullptr;
 }
 
+enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
+                                                   const enum AVPixelFormat* pixFmts)
+{
+    FFmpegVideoDecoder* decoder = (FFmpegVideoDecoder*)context->opaque;
+    const enum AVPixelFormat *p;
+
+    for (p = pixFmts; *p != -1; p++) {
+        // Only match our hardware decoding codec or SW pixel
+        // format (if not using hardware decoding). It's crucial
+        // to override the default get_format() which will try
+        // to gracefully fall back to software decode and break us.
+        if (*p == (decoder->m_HwDecodeCfg ?
+                   decoder->m_HwDecodeCfg->pix_fmt :
+                   context->pix_fmt)) {
+            return *p;
+        }
+    }
+
+    return AV_PIX_FMT_NONE;
+}
+
 FFmpegVideoDecoder::FFmpegVideoDecoder()
     : m_VideoDecoderCtx(nullptr),
       m_DecodeBuffer(1024 * 1024, 0),
@@ -114,6 +56,11 @@ FFmpegVideoDecoder::FFmpegVideoDecoder()
 }
 
 FFmpegVideoDecoder::~FFmpegVideoDecoder()
+{
+    reset();
+}
+
+void FFmpegVideoDecoder::reset()
 {
     // Drop any frames still queued to ensure
     // they are properly freed.
@@ -140,24 +87,11 @@ FFmpegVideoDecoder::~FFmpegVideoDecoder()
     avcodec_free_context(&m_VideoDecoderCtx);
 
     delete m_Renderer;
+    m_Renderer = nullptr;
 }
 
-bool FFmpegVideoDecoder::initialize(
+bool FFmpegVideoDecoder::completeInitialization(AVCodec* decoder, int videoFormat, int width, int height, bool testOnly)
-        StreamingPreferences::VideoDecoderSelection vds,
-        SDL_Window* window,
-        int videoFormat,
-        int width,
-        int height,
-        int)
 {
-    AVCodec* decoder;
-
-    if (!chooseDecoder(vds, window, videoFormat, width, height,
-                       decoder, m_HwDecodeCfg, m_Renderer)) {
-        // Error logged in chooseDecoder()
-        return false;
-    }
-
     m_VideoDecoderCtx = avcodec_alloc_context3(decoder);
     if (!m_VideoDecoderCtx) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -165,6 +99,9 @@ bool FFmpegVideoDecoder::initialize(
         return false;
     }
 
+    // Stash a pointer to this object in the context
+    m_VideoDecoderCtx->opaque = this;
+
     // Always request low delay decoding
     m_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
 
@@ -182,6 +119,7 @@ bool FFmpegVideoDecoder::initialize(
     m_VideoDecoderCtx->width = width;
     m_VideoDecoderCtx->height = height;
     m_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
+    m_VideoDecoderCtx->get_format = ffGetFormat;
 
     // Allow the renderer to attach data to this decoder
     if (!m_Renderer->prepareDecoderContext(m_VideoDecoderCtx)) {
@@ -196,9 +134,129 @@ bool FFmpegVideoDecoder::initialize(
         return false;
     }
 
+    // FFMpeg doesn't completely initialize the codec until the codec
+    // config data comes in. This would be too late for us to change
+    // our minds on the selected video codec, so we'll do a trial run
+    // now to see if things will actually work when the video stream
+    // comes in.
+    if (testOnly) {
+        if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+            m_Pkt.data = (uint8_t*)k_H264TestFrame;
+            m_Pkt.size = sizeof(k_H264TestFrame);
+        }
+        else {
+            m_Pkt.data = (uint8_t*)k_HEVCTestFrame;
+            m_Pkt.size = sizeof(k_HEVCTestFrame);
+        }
+
+        err = avcodec_send_packet(m_VideoDecoderCtx, &m_Pkt);
+        if (err < 0) {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Test decode failed: %d", err);
+            char errorstring[512];
+            av_strerror(err, errorstring, sizeof(errorstring));
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Test decode failed: %s", errorstring);
+            return false;
+        }
+    }
+
     return true;
 }
 
+bool FFmpegVideoDecoder::initialize(
+        StreamingPreferences::VideoDecoderSelection vds,
+        SDL_Window* window,
+        int videoFormat,
+        int width,
+        int height,
+        int)
+{
+    AVCodec* decoder;
+
+    if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+        decoder = avcodec_find_decoder(AV_CODEC_ID_H264);
+    }
+    else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
+        decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
+    }
+    else {
+        Q_ASSERT(false);
+        decoder = nullptr;
+    }
+
+    if (!decoder) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to find decoder for format: %x",
+                     videoFormat);
+        return false;
+    }
+
+    for (int i = 0;; i++) {
+        const AVCodecHWConfig *config = avcodec_get_hw_config(decoder, i);
+        if (!config || vds == StreamingPreferences::VDS_FORCE_SOFTWARE) {
+            // No matching hardware acceleration support.
+            // This is not an error.
+            m_HwDecodeCfg = nullptr;
+            m_Renderer = new SdlRenderer();
+            if (vds != StreamingPreferences::VDS_FORCE_HARDWARE &&
+                    m_Renderer->initialize(window, videoFormat, width, height) &&
+                    completeInitialization(decoder, videoFormat, width, height, false)) {
+                return true;
+            }
+            else {
+                reset();
+                return false;
+            }
+        }
+
+        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
+            continue;
+        }
+
+        // Look for acceleration types we support
+        switch (config->device_type) {
+#ifdef Q_OS_WIN32
+        case AV_HWDEVICE_TYPE_DXVA2:
+            m_Renderer = new DXVA2Renderer();
+            break;
+#endif
+#ifdef Q_OS_DARWIN
+        case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
+            m_Renderer = VTRendererFactory::createRenderer();
+            break;
+#endif
+#ifdef HAVE_LIBVA
+        case AV_HWDEVICE_TYPE_VAAPI:
+            m_Renderer = new VAAPIRenderer();
+            break;
+#endif
+        default:
+            continue;
+        }
+
+        m_HwDecodeCfg = config;
+        // Submit test frame to ensure this codec really works
+        if (m_Renderer->initialize(window, videoFormat, width, height) &&
+                completeInitialization(decoder, videoFormat, width, height, true)) {
+            // OK, it worked, so now let's initialize it for real
+            if (m_Renderer->initialize(window, videoFormat, width, height) &&
+                    completeInitialization(decoder, videoFormat, width, height, false)) {
+                return true;
+            }
+            else {
+                SDL_LogCritical(SDL_LOG_CATEGORY_APPLICATION,
+                                "Decoder failed to initialize after successful test");
+                reset();
+            }
+        }
+        else {
+            // Failed to initialize or test frame failed, so keep looking
+            reset();
+        }
+    }
+}
+
 int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
 {
     PLENTRY entry = du->bufferList;

app/streaming/video/ffmpeg.h

@@ -23,15 +23,13 @@ public:
     virtual void dropFrame(SDL_UserEvent* event) override;
 
 private:
-    bool
+    bool completeInitialization(AVCodec* decoder, int videoFormat, int width, int height, bool testOnly);
-    chooseDecoder(
+
-        StreamingPreferences::VideoDecoderSelection vds,
+    void reset();
-        SDL_Window* window,
+
-        int videoFormat,
+    static
-        int width, int height,
+    enum AVPixelFormat ffGetFormat(AVCodecContext* context,
-        AVCodec*& chosenDecoder,
+                                   const enum AVPixelFormat* pixFmts);
-        const AVCodecHWConfig*& chosenHwConfig,
-        IFFmpegRenderer*& newRenderer);
 
     AVPacket m_Pkt;
     AVCodecContext* m_VideoDecoderCtx;
@@ -39,4 +37,7 @@ private:
     const AVCodecHWConfig* m_HwDecodeCfg;
     IFFmpegRenderer* m_Renderer;
     SDL_atomic_t m_QueuedFrames;
+
+    static const uint8_t k_H264TestFrame[];
+    static const uint8_t k_HEVCTestFrame[];
 };

app/streaming/video/ffmpeg_videosamples.cpp

@@ -0,0 +1,19 @@
+#include "ffmpeg.h"
+
+// 720p 60 FPS H.264 with 1 reference frame
+const uint8_t FFmpegVideoDecoder::k_H264TestFrame[] = {
+    0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20, 0xac, 0x2b, 0x40, 0x28, 0x02, 0xdd, 0x80, 0xb5, 0x06, 0x06, 0x06, 0xa5, 0x00, 0x00, 0x03, 0x03, 0xe8, 0x00, 0x01, 0xd4, 0xc0, 0x8f, 0x4a, 0xa0,
+    0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0xb0,
+    0x00, 0x00, 0x00, 0x01, 0x65, 0xb8, 0x02, 0x01, 0x67, 0x25, 0x1b, 0xf4, 0xfa, 0x7d, 0x40, 0x1a, 0x78, 0xe5, 0x10, 0x52, 0xc2, 0xee, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0xc6, 0xef, 0xbb, 0x81, 0x85, 0x2d, 0x47, 0xda, 0xca, 0x4c, 0x00, 0x00, 0x03, 0x00, 0x02, 0x7b, 0xcf, 0x80, 0x00, 0x45, 0x40, 0x01, 0x8d, 0xa6, 0x00, 0x01, 0x64, 0x00, 0x0e, 0x03, 0xc8, 0x00, 0x0e, 0x10, 0x00, 0xbd, 0xc5, 0x00, 0x01, 0x11, 0x00, 0x0e, 0xa3, 0x80, 0x00, 0x38, 0xa0,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x6e, 0x2e, 0x00, 0x83, 0x7f, 0xb4, 0x8e, 0x79, 0xa5, 0xff, 0x84, 0x3f, 0x7f, 0x34, 0x3f, 0x97, 0x1b, 0xaf, 0x31, 0x5f, 0x6e, 0xaa, 0xb6, 0xac, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x78, 0x36, 0x9d, 0xa4, 0x65, 0xf9, 0x1e, 0x5b, 0x3a, 0xb0, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x41, 0x80, 0x00, 0xc5, 0xc8, 0x00, 0x00, 0xfa, 0x00, 0x03, 0x24, 0x00, 0x0e, 0x20, 0x00, 0x3f, 0x80, 0x01, 0x32, 0x00, 0x08, 0x68, 0x00, 0x3e, 0xc0, 0x03, 0x8a,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x37, 0x0b, 0x80, 0x21, 0x7f, 0xe3, 0x85, 0x1c, 0xd9, 0xff, 0xe1, 0x1b, 0x01, 0xfa, 0xc0, 0x3e, 0x11, 0x7e, 0xfe, 0x45, 0x5c, 0x43, 0x69, 0x31, 0x4b, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x02, 0x24, 0xae, 0x1a, 0xbb, 0xae, 0x75, 0x9e, 0x35, 0xae, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x03, 0x64, 0x00, 0x09, 0x98, 0x00, 0x1e, 0xc0, 0x00, 0x64, 0x80, 0x01, 0xc4, 0x00, 0x07, 0xf0, 0x00, 0x42, 0xf0, 0x00, 0x00, 0xe1, 0x98, 0x00, 0x05, 0x4b, 0x28, 0x00, 0x06, 0x04,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x14, 0xa2, 0xe0, 0x08, 0x5f, 0xe3, 0x85, 0x1c, 0xd9, 0xff, 0xe1, 0x1b, 0x01, 0xfa, 0xc0, 0x3e, 0x11, 0x7e, 0xfe, 0x45, 0x5c, 0x43, 0x69, 0x31, 0x4b, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x02, 0x24, 0xb9, 0x7d, 0xb4, 0x70, 0x4d, 0x15, 0xc5, 0x0a, 0x4e, 0x60, 0x00, 0x00, 0x03, 0x00, 0x00, 0x26, 0xa8, 0xb0, 0x00, 0x13, 0xcd, 0xcc, 0x00, 0x07, 0x48, 0x88, 0x00, 0x06, 0x29, 0x69, 0x00, 0x01, 0x16, 0xac, 0x80, 0x00, 0x9e, 0x4e, 0x80, 0x00, 0x3f, 0x84, 0x20, 0x00, 0x6f, 0x41, 0xa0, 0x00, 0x20, 0x00, 0x02, 0x16, 0xb8, 0x00, 0x08, 0x08
+};
+
+// 720p 60 FPS HEVC with 1 reference frame
+const uint8_t FFmpegVideoDecoder::k_HEVCTestFrame[] = {
+    0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xac, 0x09,
+    0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x02, 0x80, 0x80, 0x2e, 0x1f, 0x13, 0x96, 0xb4, 0xa4, 0x25, 0x92, 0xe3, 0x01, 0x6a, 0x0c, 0x0c, 0x0d, 0x48, 0x20, 0x00, 0x00, 0x03, 0x00, 0x20, 0x00, 0x00, 0x07, 0x85, 0xf1, 0xa2, 0xd0,
+    0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0xf7, 0xc0, 0xcc, 0x90,
+    0x00, 0x00, 0x00, 0x01, 0x2a, 0x01, 0xac, 0x0c, 0xf0, 0xe6, 0xf6, 0x5e, 0xff, 0xd9, 0x97, 0xeb, 0x44, 0x7d, 0x1f, 0xa4, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x03, 0x52, 0x7c, 0x4e, 0x5b, 0xfe, 0xbe, 0xc2, 0x87, 0x20, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x2b, 0x60,
+};