Centralize colorspace and color range handling

app/streaming/video/ffmpeg-renderers/d3d11va.cpp

@@ -80,8 +80,8 @@ D3D11VARenderer::D3D11VARenderer(int decoderSelectionPass)
       m_SwapChain(nullptr),
       m_DeviceContext(nullptr),
       m_RenderTargetView(nullptr),
-      m_LastColorSpace(AVCOL_SPC_UNSPECIFIED),
+      m_LastColorSpace(-1),
-      m_LastColorRange(AVCOL_RANGE_UNSPECIFIED),
+      m_LastFullRange(false),
       m_AllowTearing(false),
       m_VideoGenericPixelShader(nullptr),
       m_VideoBt601LimPixelShader(nullptr),
@@ -673,11 +673,14 @@ void D3D11VARenderer::renderOverlay(Overlay::OverlayType type)
 
 void D3D11VARenderer::bindColorConversion(AVFrame* frame)
 {
+    bool fullRange = isFrameFullRange(frame);
+    int colorspace = getFrameColorspace(frame);
+
     // We have purpose-built shaders for the common Rec 601 (SDR) and Rec 2020 (HDR) cases
-    if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_SMPTE170M) {
+    if (!fullRange && colorspace == COLORSPACE_REC_601) {
         m_DeviceContext->PSSetShader(m_VideoBt601LimPixelShader, nullptr, 0);
     }
-    else if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_BT2020_NCL) {
+    else if (!fullRange && colorspace == COLORSPACE_REC_2020) {
         m_DeviceContext->PSSetShader(m_VideoBt2020LimPixelShader, nullptr, 0);
     }
     else {
@@ -685,14 +688,14 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
         m_DeviceContext->PSSetShader(m_VideoGenericPixelShader, nullptr, 0);
 
         // If nothing has changed since last frame, we're done
-        if (frame->colorspace == m_LastColorSpace && frame->color_range == m_LastColorRange) {
+        if (colorspace == m_LastColorSpace && fullRange == m_LastFullRange) {
             return;
         }
 
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
-                    "Falling back to generic video pixel shader for %d:%d",
+                    "Falling back to generic video pixel shader for %d (%s range)",
-                    frame->colorspace,
+                    colorspace,
-                    frame->color_range);
+                    fullRange ? "full" : "limited");
 
         D3D11_BUFFER_DESC constDesc = {};
         constDesc.ByteWidth = sizeof(CSC_CONST_BUF);
@@ -701,41 +704,21 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
         constDesc.CPUAccessFlags = 0;
         constDesc.MiscFlags = 0;
 
-        // This handles the case where the color range is unknown,
-        // so that we use Limited color range which is the default
-        // behavior for Moonlight.
         CSC_CONST_BUF constBuf = {};
-        bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
         const float* rawCscMatrix;
-        switch (frame->colorspace) {
+        switch (colorspace) {
-        case AVCOL_SPC_SMPTE170M:
+        case COLORSPACE_REC_601:
-        case AVCOL_SPC_BT470BG:
             rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim;
             break;
-        case AVCOL_SPC_BT709:
+        case COLORSPACE_REC_709:
             rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim;
             break;
-        case AVCOL_SPC_BT2020_NCL:
+        case COLORSPACE_REC_2020:
-        case AVCOL_SPC_BT2020_CL:
             rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim;
             break;
         default:
-            // Sunshine doesn't always populate this data correctly,
+            SDL_assert(false);
-            // so fallback to assuming they're sending what we asked for.
+            return;
-            switch (getDecoderColorspace()) {
-            case COLORSPACE_REC_601:
-                rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim;
-                break;
-            case COLORSPACE_REC_709:
-                rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim;
-                break;
-            case COLORSPACE_REC_2020:
-                rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim;
-                break;
-            default:
-                SDL_assert(false);
-                return;
-            }
         }
 
         // We need to adjust our raw CSC matrix to be column-major and with float3 vectors
@@ -769,8 +752,8 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
         }
     }
 
-    m_LastColorSpace = frame->colorspace;
+    m_LastColorSpace = colorspace;
-    m_LastColorRange = frame->color_range;
+    m_LastFullRange = fullRange;
 }
 
 void D3D11VARenderer::renderVideo(AVFrame* frame)

app/streaming/video/ffmpeg-renderers/d3d11va.h

@@ -49,8 +49,8 @@ private:
     int m_TextureAlignment;
     int m_DisplayWidth;
     int m_DisplayHeight;
-    AVColorSpace m_LastColorSpace;
+    int m_LastColorSpace;
-    AVColorRange m_LastColorRange;
+    bool m_LastFullRange;
 
     bool m_AllowTearing;
 

app/streaming/video/ffmpeg-renderers/drm.cpp

@@ -50,8 +50,8 @@ DrmRenderer::DrmRenderer(IFFmpegRenderer *backendRenderer)
       m_CrtcId(0),
       m_PlaneId(0),
       m_CurrentFbId(0),
-      m_LastColorRange(AVCOL_RANGE_UNSPECIFIED),
+      m_LastFullRange(false),
-      m_LastColorSpace(AVCOL_SPC_UNSPECIFIED),
+      m_LastColorSpace(-1),
       m_ColorEncodingProp(nullptr),
       m_ColorRangeProp(nullptr),
       m_HdrOutputMetadataProp(nullptr),
@@ -608,7 +608,12 @@ void DrmRenderer::renderFrame(AVFrame* frame)
         return;
     }
 
-    if (frame->color_range != m_LastColorRange) {
+    int colorspace = getFrameColorspace(frame);
+    bool fullRange = isFrameFullRange(frame);
+
+    // We also update the color range when the colorspace changes in order to handle initialization
+    // where the last color range value may not actual be applied to the plane.
+    if (fullRange != m_LastFullRange || colorspace != m_LastColorSpace) {
         const char* desiredValue = getDrmColorRangeValue(frame);
 
         if (m_ColorRangeProp != nullptr && desiredValue != nullptr) {
@@ -647,10 +652,10 @@ void DrmRenderer::renderFrame(AVFrame* frame)
                         "COLOR_RANGE property does not exist on output plane. Colors may be inaccurate!");
         }
 
-        m_LastColorRange = frame->color_range;
+        m_LastFullRange = fullRange;
     }
 
-    if (frame->colorspace != m_LastColorSpace) {
+    if (colorspace != m_LastColorSpace) {
         const char* desiredValue = getDrmColorEncodingValue(frame);
 
         if (m_ColorEncodingProp != nullptr && desiredValue != nullptr) {
@@ -689,7 +694,7 @@ void DrmRenderer::renderFrame(AVFrame* frame)
                         "COLOR_ENCODING property does not exist on output plane. Colors may be inaccurate!");
         }
 
-        m_LastColorSpace = frame->colorspace;
+        m_LastColorSpace = colorspace;
     }
 
     // Update the overlay
@@ -745,12 +750,12 @@ bool DrmRenderer::isDirectRenderingSupported()
 
 const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame)
 {
-    switch (frame->colorspace) {
+    switch (getFrameColorspace(frame)) {
-    case AVCOL_SPC_SMPTE170M:
+    case COLORSPACE_REC_601:
         return "ITU-R BT.601 YCbCr";
-    case AVCOL_SPC_BT709:
+    case COLORSPACE_REC_709:
         return "ITU-R BT.709 YCbCr";
-    case AVCOL_SPC_BT2020_NCL:
+    case COLORSPACE_REC_2020:
         return "ITU-R BT.2020 YCbCr";
     default:
         return NULL;
@@ -759,14 +764,7 @@ const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame)
 
 const char* DrmRenderer::getDrmColorRangeValue(AVFrame* frame)
 {
-    switch (frame->color_range) {
+    return isFrameFullRange(frame) ? "YCbCr full range" : "YCbCr limited range";
-    case AVCOL_RANGE_MPEG:
-        return "YCbCr limited range";
-    case AVCOL_RANGE_JPEG:
-        return "YCbCr full range";
-    default:
-        return NULL;
-    }
 }
 
 #ifdef HAVE_EGL

app/streaming/video/ffmpeg-renderers/drm.h

@@ -78,8 +78,8 @@ private:
     uint32_t m_CrtcId;
     uint32_t m_PlaneId;
     uint32_t m_CurrentFbId;
-    AVColorRange m_LastColorRange;
+    bool m_LastFullRange;
-    AVColorSpace m_LastColorSpace;
+    int m_LastColorSpace;
     drmModePropertyPtr m_ColorEncodingProp;
     drmModePropertyPtr m_ColorRangeProp;
     drmModePropertyPtr m_HdrOutputMetadataProp;

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -1009,17 +1009,7 @@ void DXVA2Renderer::renderFrame(AVFrame *frame)
     IDirect3DSurface9* surface = reinterpret_cast<IDirect3DSurface9*>(frame->data[3]);
     HRESULT hr;
 
-    switch (frame->color_range) {
+    m_Desc.SampleFormat.NominalRange = isFrameFullRange(frame) ? DXVA2_NominalRange_0_255 : DXVA2_NominalRange_16_235;
-    case AVCOL_RANGE_JPEG:
-        m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_0_255;
-        break;
-    case AVCOL_RANGE_MPEG:
-        m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_16_235;
-        break;
-    default:
-        m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_Unknown;
-        break;
-    }
 
     switch (frame->color_primaries) {
     case AVCOL_PRI_BT709:
@@ -1067,17 +1057,13 @@ void DXVA2Renderer::renderFrame(AVFrame *frame)
         break;
     }
 
-    switch (frame->colorspace) {
+    switch (getFrameColorspace(frame)) {
-    case AVCOL_SPC_BT709:
+    case COLORSPACE_REC_709:
         m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT709;
         break;
-    case AVCOL_SPC_BT470BG:
+    case COLORSPACE_REC_601:
-    case AVCOL_SPC_SMPTE170M:
         m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT601;
         break;
-    case AVCOL_SPC_SMPTE240M:
-        m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_SMPTE240M;
-        break;
     default:
         m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_Unknown;
         break;

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -739,10 +739,7 @@ const float *EGLRenderer::getColorOffsets(const AVFrame* frame) {
     static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f };
     static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f };
 
-    // This handles the case where the color range is unknown,
+    return isFrameFullRange(frame) ? fullOffsets : limitedOffsets;
-    // so that we use Limited color range which is the default
-    // behavior for Moonlight.
-    return (frame->color_range == AVCOL_RANGE_JPEG) ? fullOffsets : limitedOffsets;
 }
 
 const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
@@ -780,33 +777,17 @@ const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
         1.4746f, -0.5714f, 0.0f
     };
 
-    // This handles the case where the color range is unknown,
+    bool fullRange = isFrameFullRange(frame);
-    // so that we use Limited color range which is the default
+    switch (getFrameColorspace(frame)) {
-    // behavior for Moonlight.
+        case COLORSPACE_REC_601:
-    bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
-    switch (frame->colorspace) {
-        case AVCOL_SPC_SMPTE170M:
-        case AVCOL_SPC_BT470BG:
             return fullRange ? bt601Full : bt601Lim;
-        case AVCOL_SPC_BT709:
+        case COLORSPACE_REC_709:
             return fullRange ? bt709Full : bt709Lim;
-        case AVCOL_SPC_BT2020_NCL:
+        case COLORSPACE_REC_2020:
-        case AVCOL_SPC_BT2020_CL:
             return fullRange ? bt2020Full : bt2020Lim;
         default:
-            // Some backends don't populate this, so we'll assume
+            SDL_assert(false);
-            // the host gave us what we asked for by default.
+    }
-            switch (getDecoderColorspace()) {
-                case COLORSPACE_REC_601:
-                    return fullRange ? bt601Full : bt601Lim;
-                case COLORSPACE_REC_709:
-                    return fullRange ? bt709Full : bt709Lim;
-                case COLORSPACE_REC_2020:
-                    return fullRange ? bt2020Full : bt2020Lim;
-                default:
-                    SDL_assert(false);
-            }
-    };
 
     return bt601Lim;
 }

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -158,6 +158,31 @@ public:
         return COLORSPACE_REC_601;
     }
 
+    virtual int getFrameColorspace(AVFrame* frame) {
+        // Prefer the colorspace field on the AVFrame itself
+        switch (frame->colorspace) {
+        case AVCOL_SPC_SMPTE170M:
+        case AVCOL_SPC_BT470BG:
+            return COLORSPACE_REC_601;
+        case AVCOL_SPC_BT709:
+            return COLORSPACE_REC_709;
+        case AVCOL_SPC_BT2020_NCL:
+        case AVCOL_SPC_BT2020_CL:
+            return COLORSPACE_REC_2020;
+        default:
+            // If the colorspace is not populated, assume the encoder
+            // is sending the colorspace that we requested.
+            return getDecoderColorspace();
+        }
+    }
+
+    virtual bool isFrameFullRange(AVFrame* frame) {
+        // This handles the case where the color range is unknown,
+        // so that we use Limited color range which is the default
+        // behavior for Moonlight.
+        return frame->color_range == AVCOL_RANGE_JPEG;
+    }
+
     virtual bool isRenderThreadSupported() {
         // Render thread is supported by default
         return true;

app/streaming/video/ffmpeg-renderers/sdlvid.cpp

@@ -10,7 +10,7 @@ SdlRenderer::SdlRenderer()
       m_Renderer(nullptr),
       m_Texture(nullptr),
       m_SwPixelFormat(AV_PIX_FMT_NONE),
-      m_ColorSpace(AVCOL_SPC_UNSPECIFIED),
+      m_ColorSpace(-1),
       m_MapFrame(false)
 {
     SDL_zero(m_OverlayTextures);
@@ -373,7 +373,8 @@ ReadbackRetry:
     // Because the specific YUV color conversion shader is established at
     // texture creation for most SDL render backends, we need to recreate
     // the texture when the colorspace changes.
-    if (frame->colorspace != m_ColorSpace) {
+    int colorspace = getFrameColorspace(frame);
+    if (colorspace != m_ColorSpace) {
 #ifdef HAVE_CUDA
         if (m_CudaGLHelper != nullptr) {
             delete m_CudaGLHelper;
@@ -386,7 +387,7 @@ ReadbackRetry:
             m_Texture = nullptr;
         }
 
-        m_ColorSpace = frame->colorspace;
+        m_ColorSpace = colorspace;
     }
 
     if (m_Texture == nullptr) {
@@ -410,13 +411,12 @@ ReadbackRetry:
             goto Exit;
         }
 
-        switch (frame->colorspace)
+        switch (colorspace)
         {
-        case AVCOL_SPC_BT709:
+        case COLORSPACE_REC_709:
             SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709);
             break;
-        case AVCOL_SPC_BT470BG:
+        case COLORSPACE_REC_601:
-        case AVCOL_SPC_SMPTE170M:
         default:
             SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601);
             break;

app/streaming/video/ffmpeg-renderers/sdlvid.h

@@ -26,7 +26,7 @@ private:
     SDL_Renderer* m_Renderer;
     SDL_Texture* m_Texture;
     enum AVPixelFormat m_SwPixelFormat;
-    enum AVColorSpace m_ColorSpace;
+    int m_ColorSpace;
     bool m_MapFrame;
     SDL_Texture* m_OverlayTextures[Overlay::OverlayMax];
     SDL_Rect m_OverlayRects[Overlay::OverlayMax];

app/streaming/video/ffmpeg-renderers/vt.mm

@@ -40,7 +40,7 @@ public:
           m_FormatDesc(nullptr),
           m_StreamView(nullptr),
           m_DisplayLink(nullptr),
-          m_LastAvColorSpace(AVCOL_SPC_UNSPECIFIED),
+          m_LastColorSpace(-1),
           m_ColorSpace(nullptr),
           m_VsyncMutex(nullptr),
           m_VsyncPassed(nullptr)
@@ -200,27 +200,26 @@ public:
 
         // Reset m_ColorSpace if the colorspace changes. This can happen when
         // a game enters HDR mode (Rec 601 -> Rec 2020).
-        if (frame->colorspace != m_LastAvColorSpace) {
+        int colorspace = getFrameColorspace(frame);
+        if (colorspace != m_LastColorSpace) {
             if (m_ColorSpace != nullptr) {
                 CGColorSpaceRelease(m_ColorSpace);
                 m_ColorSpace = nullptr;
             }
 
-            switch (frame->colorspace) {
+            switch (colorspace) {
-            case AVCOL_SPC_BT709:
+            case COLORSPACE_REC_709:
                 m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709);
                 break;
-            case AVCOL_SPC_BT2020_NCL:
+            case COLORSPACE_REC_2020:
                 m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020);
                 break;
-            case AVCOL_SPC_SMPTE170M:
+            case COLORSPACE_REC_601:
                 m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
                 break;
-            default:
-                break;
             }
 
-            m_LastAvColorSpace = frame->colorspace;
+            m_LastColorSpace = colorspace;
         }
 
         if (m_ColorSpace != nullptr) {
@@ -522,7 +521,7 @@ private:
     NSView* m_StreamView;
     NSTextField* m_OverlayTextFields[Overlay::OverlayMax];
     CVDisplayLinkRef m_DisplayLink;
-    AVColorSpace m_LastAvColorSpace;
+    int m_LastColorSpace;
     CGColorSpaceRef m_ColorSpace;
     SDL_mutex* m_VsyncMutex;
     SDL_cond* m_VsyncPassed;