diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp index 6e6346b0..f1d90c32 100644 --- a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp +++ b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp @@ -80,8 +80,8 @@ D3D11VARenderer::D3D11VARenderer(int decoderSelectionPass) m_SwapChain(nullptr), m_DeviceContext(nullptr), m_RenderTargetView(nullptr), - m_LastColorSpace(AVCOL_SPC_UNSPECIFIED), - m_LastColorRange(AVCOL_RANGE_UNSPECIFIED), + m_LastColorSpace(-1), + m_LastFullRange(false), m_AllowTearing(false), m_VideoGenericPixelShader(nullptr), m_VideoBt601LimPixelShader(nullptr), @@ -673,11 +673,14 @@ void D3D11VARenderer::renderOverlay(Overlay::OverlayType type) void D3D11VARenderer::bindColorConversion(AVFrame* frame) { + bool fullRange = isFrameFullRange(frame); + int colorspace = getFrameColorspace(frame); + // We have purpose-built shaders for the common Rec 601 (SDR) and Rec 2020 (HDR) cases - if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_SMPTE170M) { + if (!fullRange && colorspace == COLORSPACE_REC_601) { m_DeviceContext->PSSetShader(m_VideoBt601LimPixelShader, nullptr, 0); } - else if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_BT2020_NCL) { + else if (!fullRange && colorspace == COLORSPACE_REC_2020) { m_DeviceContext->PSSetShader(m_VideoBt2020LimPixelShader, nullptr, 0); } else { @@ -685,14 +688,14 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame) m_DeviceContext->PSSetShader(m_VideoGenericPixelShader, nullptr, 0); // If nothing has changed since last frame, we're done - if (frame->colorspace == m_LastColorSpace && frame->color_range == m_LastColorRange) { + if (colorspace == m_LastColorSpace && fullRange == m_LastFullRange) { return; } SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "Falling back to generic video pixel shader for %d:%d", - frame->colorspace, - frame->color_range); + "Falling back to generic video pixel shader for %d (%s range)", + colorspace, + fullRange ? "full" : "limited"); D3D11_BUFFER_DESC constDesc = {}; constDesc.ByteWidth = sizeof(CSC_CONST_BUF); @@ -701,41 +704,21 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame) constDesc.CPUAccessFlags = 0; constDesc.MiscFlags = 0; - // This handles the case where the color range is unknown, - // so that we use Limited color range which is the default - // behavior for Moonlight. CSC_CONST_BUF constBuf = {}; - bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG); const float* rawCscMatrix; - switch (frame->colorspace) { - case AVCOL_SPC_SMPTE170M: - case AVCOL_SPC_BT470BG: + switch (colorspace) { + case COLORSPACE_REC_601: rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim; break; - case AVCOL_SPC_BT709: + case COLORSPACE_REC_709: rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim; break; - case AVCOL_SPC_BT2020_NCL: - case AVCOL_SPC_BT2020_CL: + case COLORSPACE_REC_2020: rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim; break; default: - // Sunshine doesn't always populate this data correctly, - // so fallback to assuming they're sending what we asked for. - switch (getDecoderColorspace()) { - case COLORSPACE_REC_601: - rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim; - break; - case COLORSPACE_REC_709: - rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim; - break; - case COLORSPACE_REC_2020: - rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim; - break; - default: - SDL_assert(false); - return; - } + SDL_assert(false); + return; } // We need to adjust our raw CSC matrix to be column-major and with float3 vectors @@ -769,8 +752,8 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame) } } - m_LastColorSpace = frame->colorspace; - m_LastColorRange = frame->color_range; + m_LastColorSpace = colorspace; + m_LastFullRange = fullRange; } void D3D11VARenderer::renderVideo(AVFrame* frame) diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.h b/app/streaming/video/ffmpeg-renderers/d3d11va.h index 2439f605..af9480ef 100644 --- a/app/streaming/video/ffmpeg-renderers/d3d11va.h +++ b/app/streaming/video/ffmpeg-renderers/d3d11va.h @@ -49,8 +49,8 @@ private: int m_TextureAlignment; int m_DisplayWidth; int m_DisplayHeight; - AVColorSpace m_LastColorSpace; - AVColorRange m_LastColorRange; + int m_LastColorSpace; + bool m_LastFullRange; bool m_AllowTearing; diff --git a/app/streaming/video/ffmpeg-renderers/drm.cpp b/app/streaming/video/ffmpeg-renderers/drm.cpp index 5e80dcdb..c20355b5 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.cpp +++ b/app/streaming/video/ffmpeg-renderers/drm.cpp @@ -50,8 +50,8 @@ DrmRenderer::DrmRenderer(IFFmpegRenderer *backendRenderer) m_CrtcId(0), m_PlaneId(0), m_CurrentFbId(0), - m_LastColorRange(AVCOL_RANGE_UNSPECIFIED), - m_LastColorSpace(AVCOL_SPC_UNSPECIFIED), + m_LastFullRange(false), + m_LastColorSpace(-1), m_ColorEncodingProp(nullptr), m_ColorRangeProp(nullptr), m_HdrOutputMetadataProp(nullptr), @@ -608,7 +608,12 @@ void DrmRenderer::renderFrame(AVFrame* frame) return; } - if (frame->color_range != m_LastColorRange) { + int colorspace = getFrameColorspace(frame); + bool fullRange = isFrameFullRange(frame); + + // We also update the color range when the colorspace changes in order to handle initialization + // where the last color range value may not actual be applied to the plane. + if (fullRange != m_LastFullRange || colorspace != m_LastColorSpace) { const char* desiredValue = getDrmColorRangeValue(frame); if (m_ColorRangeProp != nullptr && desiredValue != nullptr) { @@ -647,10 +652,10 @@ void DrmRenderer::renderFrame(AVFrame* frame) "COLOR_RANGE property does not exist on output plane. Colors may be inaccurate!"); } - m_LastColorRange = frame->color_range; + m_LastFullRange = fullRange; } - if (frame->colorspace != m_LastColorSpace) { + if (colorspace != m_LastColorSpace) { const char* desiredValue = getDrmColorEncodingValue(frame); if (m_ColorEncodingProp != nullptr && desiredValue != nullptr) { @@ -689,7 +694,7 @@ void DrmRenderer::renderFrame(AVFrame* frame) "COLOR_ENCODING property does not exist on output plane. Colors may be inaccurate!"); } - m_LastColorSpace = frame->colorspace; + m_LastColorSpace = colorspace; } // Update the overlay @@ -745,12 +750,12 @@ bool DrmRenderer::isDirectRenderingSupported() const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame) { - switch (frame->colorspace) { - case AVCOL_SPC_SMPTE170M: + switch (getFrameColorspace(frame)) { + case COLORSPACE_REC_601: return "ITU-R BT.601 YCbCr"; - case AVCOL_SPC_BT709: + case COLORSPACE_REC_709: return "ITU-R BT.709 YCbCr"; - case AVCOL_SPC_BT2020_NCL: + case COLORSPACE_REC_2020: return "ITU-R BT.2020 YCbCr"; default: return NULL; @@ -759,14 +764,7 @@ const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame) const char* DrmRenderer::getDrmColorRangeValue(AVFrame* frame) { - switch (frame->color_range) { - case AVCOL_RANGE_MPEG: - return "YCbCr limited range"; - case AVCOL_RANGE_JPEG: - return "YCbCr full range"; - default: - return NULL; - } + return isFrameFullRange(frame) ? "YCbCr full range" : "YCbCr limited range"; } #ifdef HAVE_EGL diff --git a/app/streaming/video/ffmpeg-renderers/drm.h b/app/streaming/video/ffmpeg-renderers/drm.h index f19f31b5..5d88f94f 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.h +++ b/app/streaming/video/ffmpeg-renderers/drm.h @@ -78,8 +78,8 @@ private: uint32_t m_CrtcId; uint32_t m_PlaneId; uint32_t m_CurrentFbId; - AVColorRange m_LastColorRange; - AVColorSpace m_LastColorSpace; + bool m_LastFullRange; + int m_LastColorSpace; drmModePropertyPtr m_ColorEncodingProp; drmModePropertyPtr m_ColorRangeProp; drmModePropertyPtr m_HdrOutputMetadataProp; diff --git a/app/streaming/video/ffmpeg-renderers/dxva2.cpp b/app/streaming/video/ffmpeg-renderers/dxva2.cpp index 89322bca..73256354 100644 --- a/app/streaming/video/ffmpeg-renderers/dxva2.cpp +++ b/app/streaming/video/ffmpeg-renderers/dxva2.cpp @@ -1009,17 +1009,7 @@ void DXVA2Renderer::renderFrame(AVFrame *frame) IDirect3DSurface9* surface = reinterpret_cast(frame->data[3]); HRESULT hr; - switch (frame->color_range) { - case AVCOL_RANGE_JPEG: - m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_0_255; - break; - case AVCOL_RANGE_MPEG: - m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_16_235; - break; - default: - m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_Unknown; - break; - } + m_Desc.SampleFormat.NominalRange = isFrameFullRange(frame) ? DXVA2_NominalRange_0_255 : DXVA2_NominalRange_16_235; switch (frame->color_primaries) { case AVCOL_PRI_BT709: @@ -1067,17 +1057,13 @@ void DXVA2Renderer::renderFrame(AVFrame *frame) break; } - switch (frame->colorspace) { - case AVCOL_SPC_BT709: + switch (getFrameColorspace(frame)) { + case COLORSPACE_REC_709: m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT709; break; - case AVCOL_SPC_BT470BG: - case AVCOL_SPC_SMPTE170M: + case COLORSPACE_REC_601: m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT601; break; - case AVCOL_SPC_SMPTE240M: - m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_SMPTE240M; - break; default: m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_Unknown; break; diff --git a/app/streaming/video/ffmpeg-renderers/eglvid.cpp b/app/streaming/video/ffmpeg-renderers/eglvid.cpp index ad0e2788..a6cf96ca 100644 --- a/app/streaming/video/ffmpeg-renderers/eglvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/eglvid.cpp @@ -739,10 +739,7 @@ const float *EGLRenderer::getColorOffsets(const AVFrame* frame) { static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f }; static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f }; - // This handles the case where the color range is unknown, - // so that we use Limited color range which is the default - // behavior for Moonlight. - return (frame->color_range == AVCOL_RANGE_JPEG) ? fullOffsets : limitedOffsets; + return isFrameFullRange(frame) ? fullOffsets : limitedOffsets; } const float *EGLRenderer::getColorMatrix(const AVFrame* frame) { @@ -780,33 +777,17 @@ const float *EGLRenderer::getColorMatrix(const AVFrame* frame) { 1.4746f, -0.5714f, 0.0f }; - // This handles the case where the color range is unknown, - // so that we use Limited color range which is the default - // behavior for Moonlight. - bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG); - switch (frame->colorspace) { - case AVCOL_SPC_SMPTE170M: - case AVCOL_SPC_BT470BG: + bool fullRange = isFrameFullRange(frame); + switch (getFrameColorspace(frame)) { + case COLORSPACE_REC_601: return fullRange ? bt601Full : bt601Lim; - case AVCOL_SPC_BT709: + case COLORSPACE_REC_709: return fullRange ? bt709Full : bt709Lim; - case AVCOL_SPC_BT2020_NCL: - case AVCOL_SPC_BT2020_CL: + case COLORSPACE_REC_2020: return fullRange ? bt2020Full : bt2020Lim; default: - // Some backends don't populate this, so we'll assume - // the host gave us what we asked for by default. - switch (getDecoderColorspace()) { - case COLORSPACE_REC_601: - return fullRange ? bt601Full : bt601Lim; - case COLORSPACE_REC_709: - return fullRange ? bt709Full : bt709Lim; - case COLORSPACE_REC_2020: - return fullRange ? bt2020Full : bt2020Lim; - default: - SDL_assert(false); - } - }; + SDL_assert(false); + } return bt601Lim; } diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h index cf2b0f51..f1d0c3c8 100644 --- a/app/streaming/video/ffmpeg-renderers/renderer.h +++ b/app/streaming/video/ffmpeg-renderers/renderer.h @@ -158,6 +158,31 @@ public: return COLORSPACE_REC_601; } + virtual int getFrameColorspace(AVFrame* frame) { + // Prefer the colorspace field on the AVFrame itself + switch (frame->colorspace) { + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_BT470BG: + return COLORSPACE_REC_601; + case AVCOL_SPC_BT709: + return COLORSPACE_REC_709; + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + return COLORSPACE_REC_2020; + default: + // If the colorspace is not populated, assume the encoder + // is sending the colorspace that we requested. + return getDecoderColorspace(); + } + } + + virtual bool isFrameFullRange(AVFrame* frame) { + // This handles the case where the color range is unknown, + // so that we use Limited color range which is the default + // behavior for Moonlight. + return frame->color_range == AVCOL_RANGE_JPEG; + } + virtual bool isRenderThreadSupported() { // Render thread is supported by default return true; diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp index 4c1fa489..2b03a187 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp @@ -10,7 +10,7 @@ SdlRenderer::SdlRenderer() m_Renderer(nullptr), m_Texture(nullptr), m_SwPixelFormat(AV_PIX_FMT_NONE), - m_ColorSpace(AVCOL_SPC_UNSPECIFIED), + m_ColorSpace(-1), m_MapFrame(false) { SDL_zero(m_OverlayTextures); @@ -373,7 +373,8 @@ ReadbackRetry: // Because the specific YUV color conversion shader is established at // texture creation for most SDL render backends, we need to recreate // the texture when the colorspace changes. - if (frame->colorspace != m_ColorSpace) { + int colorspace = getFrameColorspace(frame); + if (colorspace != m_ColorSpace) { #ifdef HAVE_CUDA if (m_CudaGLHelper != nullptr) { delete m_CudaGLHelper; @@ -386,7 +387,7 @@ ReadbackRetry: m_Texture = nullptr; } - m_ColorSpace = frame->colorspace; + m_ColorSpace = colorspace; } if (m_Texture == nullptr) { @@ -410,13 +411,12 @@ ReadbackRetry: goto Exit; } - switch (frame->colorspace) + switch (colorspace) { - case AVCOL_SPC_BT709: + case COLORSPACE_REC_709: SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); break; - case AVCOL_SPC_BT470BG: - case AVCOL_SPC_SMPTE170M: + case COLORSPACE_REC_601: default: SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601); break; diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.h b/app/streaming/video/ffmpeg-renderers/sdlvid.h index 3223793a..592f883d 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.h +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.h @@ -26,7 +26,7 @@ private: SDL_Renderer* m_Renderer; SDL_Texture* m_Texture; enum AVPixelFormat m_SwPixelFormat; - enum AVColorSpace m_ColorSpace; + int m_ColorSpace; bool m_MapFrame; SDL_Texture* m_OverlayTextures[Overlay::OverlayMax]; SDL_Rect m_OverlayRects[Overlay::OverlayMax]; diff --git a/app/streaming/video/ffmpeg-renderers/vt.mm b/app/streaming/video/ffmpeg-renderers/vt.mm index 06ced6e4..d26cfe8a 100644 --- a/app/streaming/video/ffmpeg-renderers/vt.mm +++ b/app/streaming/video/ffmpeg-renderers/vt.mm @@ -40,7 +40,7 @@ public: m_FormatDesc(nullptr), m_StreamView(nullptr), m_DisplayLink(nullptr), - m_LastAvColorSpace(AVCOL_SPC_UNSPECIFIED), + m_LastColorSpace(-1), m_ColorSpace(nullptr), m_VsyncMutex(nullptr), m_VsyncPassed(nullptr) @@ -200,27 +200,26 @@ public: // Reset m_ColorSpace if the colorspace changes. This can happen when // a game enters HDR mode (Rec 601 -> Rec 2020). - if (frame->colorspace != m_LastAvColorSpace) { + int colorspace = getFrameColorspace(frame); + if (colorspace != m_LastColorSpace) { if (m_ColorSpace != nullptr) { CGColorSpaceRelease(m_ColorSpace); m_ColorSpace = nullptr; } - switch (frame->colorspace) { - case AVCOL_SPC_BT709: + switch (colorspace) { + case COLORSPACE_REC_709: m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709); break; - case AVCOL_SPC_BT2020_NCL: + case COLORSPACE_REC_2020: m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020); break; - case AVCOL_SPC_SMPTE170M: + case COLORSPACE_REC_601: m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); break; - default: - break; } - m_LastAvColorSpace = frame->colorspace; + m_LastColorSpace = colorspace; } if (m_ColorSpace != nullptr) { @@ -522,7 +521,7 @@ private: NSView* m_StreamView; NSTextField* m_OverlayTextFields[Overlay::OverlayMax]; CVDisplayLinkRef m_DisplayLink; - AVColorSpace m_LastAvColorSpace; + int m_LastColorSpace; CGColorSpaceRef m_ColorSpace; SDL_mutex* m_VsyncMutex; SDL_cond* m_VsyncPassed;