From d085722911ef100e8de5debfdf53fca4675d606e Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Sat, 6 Jul 2024 02:27:46 -0500
Subject: [PATCH] Don't attempt to use mismatched 8-bit formats with 10-bit
 codecs

In addition to silently truncating the samples, this also tickles
some nasty bugs in the VF2's out-of-tree OMX decoder.
---
 app/streaming/video/ffmpeg-renderers/drm.cpp  | 30 +++++++++++++------
 .../video/ffmpeg-renderers/sdlvid.cpp         | 27 ++++++++++-------
 2 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/app/streaming/video/ffmpeg-renderers/drm.cpp b/app/streaming/video/ffmpeg-renderers/drm.cpp
index fc5d7e7d..4a6f25ff 100644
--- a/app/streaming/video/ffmpeg-renderers/drm.cpp
+++ b/app/streaming/video/ffmpeg-renderers/drm.cpp
@@ -661,16 +661,28 @@ bool DrmRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFor
     else {
         // If we're going to need to map this as a software frame, check
         // against the set of formats we support in mapSoftwareFrame().
-        switch (pixelFormat) {
-        case AV_PIX_FMT_DRM_PRIME:
-        case AV_PIX_FMT_NV12:
-        case AV_PIX_FMT_NV21:
-        case AV_PIX_FMT_P010:
-        case AV_PIX_FMT_YUV420P:
-        case AV_PIX_FMT_YUVJ420P:
+        if (pixelFormat == AV_PIX_FMT_DRM_PRIME) {
+            // AV_PIX_FMT_DRM_PRIME is always supported
             return true;
-        default:
-            return false;
+        }
+        else if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
+            switch (pixelFormat) {
+            case AV_PIX_FMT_P010:
+                return true;
+            default:
+                return false;
+            }
+        }
+        else {
+            switch (pixelFormat) {
+            case AV_PIX_FMT_NV12:
+            case AV_PIX_FMT_NV21:
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUVJ420P:
+                return true;
+            default:
+                return false;
+            }
         }
     }
 }
diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp
index f4a92521..61300add 100644
--- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp
+++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp
@@ -80,20 +80,25 @@ bool SdlRenderer::isRenderThreadSupported()
     return true;
 }
 
-bool SdlRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat)
+bool SdlRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
 {
-    // Remember to keep this in sync with SdlRenderer::renderFrame()!
-    switch (pixelFormat)
-    {
-    case AV_PIX_FMT_YUV420P:
-    case AV_PIX_FMT_YUVJ420P:
-    case AV_PIX_FMT_NV12:
-    case AV_PIX_FMT_NV21:
-        return true;
-
-    default:
+    if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
+        // SDL2 doesn't support 10-bit pixel formats
         return false;
     }
+    else {
+        // Remember to keep this in sync with SdlRenderer::renderFrame()!
+        switch (pixelFormat) {
+        case AV_PIX_FMT_YUV420P:
+        case AV_PIX_FMT_YUVJ420P:
+        case AV_PIX_FMT_NV12:
+        case AV_PIX_FMT_NV21:
+            return true;
+
+        default:
+            return false;
+        }
+    }
 }
 
 bool SdlRenderer::initialize(PDECODER_PARAMETERS params)