moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2026-04-11 10:16:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Centralize colorspace and color range handling

Browse Source
This commit is contained in:
Cameron Gutman
2022-09-24 12:28:23 -05:00
parent 78b522ec1d
commit 99885f5b4b
10 changed files with 93 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
app/streaming/video/ffmpeg-renderers/d3d11va.cpp
View File

@@ -80,8 +80,8 @@ D3D11VARenderer::D3D11VARenderer(int decoderSelectionPass)
m_SwapChain(nullptr),
m_DeviceContext(nullptr),
m_RenderTargetView(nullptr),
m_LastColorSpace(AVCOL_SPC_UNSPECIFIED),
m_LastColorRange(AVCOL_RANGE_UNSPECIFIED),
m_LastColorSpace(-1),
m_LastFullRange(false),
m_AllowTearing(false),
m_VideoGenericPixelShader(nullptr),
m_VideoBt601LimPixelShader(nullptr),
@@ -673,11 +673,14 @@ void D3D11VARenderer::renderOverlay(Overlay::OverlayType type)
void D3D11VARenderer::bindColorConversion(AVFrame* frame)
{
bool fullRange = isFrameFullRange(frame);
int colorspace = getFrameColorspace(frame);
// We have purpose-built shaders for the common Rec 601 (SDR) and Rec 2020 (HDR) cases
if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_SMPTE170M) {
if (!fullRange && colorspace == COLORSPACE_REC_601) {
m_DeviceContext->PSSetShader(m_VideoBt601LimPixelShader, nullptr, 0);
}
else if (frame->color_range == AVCOL_RANGE_MPEG && frame->colorspace == AVCOL_SPC_BT2020_NCL) {
else if (!fullRange && colorspace == COLORSPACE_REC_2020) {
m_DeviceContext->PSSetShader(m_VideoBt2020LimPixelShader, nullptr, 0);
}
else {
@@ -685,14 +688,14 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
m_DeviceContext->PSSetShader(m_VideoGenericPixelShader, nullptr, 0);
// If nothing has changed since last frame, we're done
if (frame->colorspace == m_LastColorSpace && frame->color_range == m_LastColorRange) {
if (colorspace == m_LastColorSpace && fullRange == m_LastFullRange) {
return;
}
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Falling back to generic video pixel shader for %d:%d",
frame->colorspace,
frame->color_range);
"Falling back to generic video pixel shader for %d (%s range)",
colorspace,
fullRange ? "full" : "limited");
D3D11_BUFFER_DESC constDesc = {};
constDesc.ByteWidth = sizeof(CSC_CONST_BUF);
@@ -701,41 +704,21 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
constDesc.CPUAccessFlags = 0;
constDesc.MiscFlags = 0;
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
CSC_CONST_BUF constBuf = {};
bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
const float* rawCscMatrix;
switch (frame->colorspace) {
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
switch (colorspace) {
case COLORSPACE_REC_601:
rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim;
break;
case AVCOL_SPC_BT709:
case COLORSPACE_REC_709:
rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim;
break;
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
case COLORSPACE_REC_2020:
rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim;
break;
default:
// Sunshine doesn't always populate this data correctly,
// so fallback to assuming they're sending what we asked for.
switch (getDecoderColorspace()) {
case COLORSPACE_REC_601:
rawCscMatrix = fullRange ? k_CscMatrix_Bt601Full : k_CscMatrix_Bt601Lim;
break;
case COLORSPACE_REC_709:
rawCscMatrix = fullRange ? k_CscMatrix_Bt709Full : k_CscMatrix_Bt709Lim;
break;
case COLORSPACE_REC_2020:
rawCscMatrix = fullRange ? k_CscMatrix_Bt2020Full : k_CscMatrix_Bt2020Lim;
break;
default:
SDL_assert(false);
return;
}
SDL_assert(false);
return;
}
// We need to adjust our raw CSC matrix to be column-major and with float3 vectors
@@ -769,8 +752,8 @@ void D3D11VARenderer::bindColorConversion(AVFrame* frame)
}
}
m_LastColorSpace = frame->colorspace;
m_LastColorRange = frame->color_range;
m_LastColorSpace = colorspace;
m_LastFullRange = fullRange;
}
void D3D11VARenderer::renderVideo(AVFrame* frame)

4
app/streaming/video/ffmpeg-renderers/d3d11va.h
View File

@@ -49,8 +49,8 @@ private:
int m_TextureAlignment;
int m_DisplayWidth;
int m_DisplayHeight;
AVColorSpace m_LastColorSpace;
AVColorRange m_LastColorRange;
int m_LastColorSpace;
bool m_LastFullRange;
bool m_AllowTearing;

34
app/streaming/video/ffmpeg-renderers/drm.cpp
View File

@@ -50,8 +50,8 @@ DrmRenderer::DrmRenderer(IFFmpegRenderer *backendRenderer)
m_CrtcId(0),
m_PlaneId(0),
m_CurrentFbId(0),
m_LastColorRange(AVCOL_RANGE_UNSPECIFIED),
m_LastColorSpace(AVCOL_SPC_UNSPECIFIED),
m_LastFullRange(false),
m_LastColorSpace(-1),
m_ColorEncodingProp(nullptr),
m_ColorRangeProp(nullptr),
m_HdrOutputMetadataProp(nullptr),
@@ -608,7 +608,12 @@ void DrmRenderer::renderFrame(AVFrame* frame)
return;
}
if (frame->color_range != m_LastColorRange) {
int colorspace = getFrameColorspace(frame);
bool fullRange = isFrameFullRange(frame);
// We also update the color range when the colorspace changes in order to handle initialization
// where the last color range value may not actual be applied to the plane.
if (fullRange != m_LastFullRange || colorspace != m_LastColorSpace) {
const char* desiredValue = getDrmColorRangeValue(frame);
if (m_ColorRangeProp != nullptr && desiredValue != nullptr) {
@@ -647,10 +652,10 @@ void DrmRenderer::renderFrame(AVFrame* frame)
"COLOR_RANGE property does not exist on output plane. Colors may be inaccurate!");
}
m_LastColorRange = frame->color_range;
m_LastFullRange = fullRange;
}
if (frame->colorspace != m_LastColorSpace) {
if (colorspace != m_LastColorSpace) {
const char* desiredValue = getDrmColorEncodingValue(frame);
if (m_ColorEncodingProp != nullptr && desiredValue != nullptr) {
@@ -689,7 +694,7 @@ void DrmRenderer::renderFrame(AVFrame* frame)
"COLOR_ENCODING property does not exist on output plane. Colors may be inaccurate!");
}
m_LastColorSpace = frame->colorspace;
m_LastColorSpace = colorspace;
}
// Update the overlay
@@ -745,12 +750,12 @@ bool DrmRenderer::isDirectRenderingSupported()
const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame)
{
switch (frame->colorspace) {
case AVCOL_SPC_SMPTE170M:
switch (getFrameColorspace(frame)) {
case COLORSPACE_REC_601:
return "ITU-R BT.601 YCbCr";
case AVCOL_SPC_BT709:
case COLORSPACE_REC_709:
return "ITU-R BT.709 YCbCr";
case AVCOL_SPC_BT2020_NCL:
case COLORSPACE_REC_2020:
return "ITU-R BT.2020 YCbCr";
default:
return NULL;
@@ -759,14 +764,7 @@ const char* DrmRenderer::getDrmColorEncodingValue(AVFrame* frame)
const char* DrmRenderer::getDrmColorRangeValue(AVFrame* frame)
{
switch (frame->color_range) {
case AVCOL_RANGE_MPEG:
return "YCbCr limited range";
case AVCOL_RANGE_JPEG:
return "YCbCr full range";
default:
return NULL;
}
return isFrameFullRange(frame) ? "YCbCr full range" : "YCbCr limited range";
}
#ifdef HAVE_EGL

4
app/streaming/video/ffmpeg-renderers/drm.h
View File

@@ -78,8 +78,8 @@ private:
uint32_t m_CrtcId;
uint32_t m_PlaneId;
uint32_t m_CurrentFbId;
AVColorRange m_LastColorRange;
AVColorSpace m_LastColorSpace;
bool m_LastFullRange;
int m_LastColorSpace;
drmModePropertyPtr m_ColorEncodingProp;
drmModePropertyPtr m_ColorRangeProp;
drmModePropertyPtr m_HdrOutputMetadataProp;

22
app/streaming/video/ffmpeg-renderers/dxva2.cpp
View File

@@ -1009,17 +1009,7 @@ void DXVA2Renderer::renderFrame(AVFrame *frame)
IDirect3DSurface9* surface = reinterpret_cast<IDirect3DSurface9*>(frame->data[3]);
HRESULT hr;
switch (frame->color_range) {
case AVCOL_RANGE_JPEG:
m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_0_255;
break;
case AVCOL_RANGE_MPEG:
m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_16_235;
break;
default:
m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_Unknown;
break;
}
m_Desc.SampleFormat.NominalRange = isFrameFullRange(frame) ? DXVA2_NominalRange_0_255 : DXVA2_NominalRange_16_235;
switch (frame->color_primaries) {
case AVCOL_PRI_BT709:
@@ -1067,17 +1057,13 @@ void DXVA2Renderer::renderFrame(AVFrame *frame)
break;
}
switch (frame->colorspace) {
case AVCOL_SPC_BT709:
switch (getFrameColorspace(frame)) {
case COLORSPACE_REC_709:
m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT709;
break;
case AVCOL_SPC_BT470BG:
case AVCOL_SPC_SMPTE170M:
case COLORSPACE_REC_601:
m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT601;
break;
case AVCOL_SPC_SMPTE240M:
m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_SMPTE240M;
break;
default:
m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_Unknown;
break;

35
app/streaming/video/ffmpeg-renderers/eglvid.cpp
View File

@@ -739,10 +739,7 @@ const float *EGLRenderer::getColorOffsets(const AVFrame* frame) {
static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f };
static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f };
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
return (frame->color_range == AVCOL_RANGE_JPEG) ? fullOffsets : limitedOffsets;
return isFrameFullRange(frame) ? fullOffsets : limitedOffsets;
}
const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
@@ -780,33 +777,17 @@ const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
1.4746f, -0.5714f, 0.0f
};
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
switch (frame->colorspace) {
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
bool fullRange = isFrameFullRange(frame);
switch (getFrameColorspace(frame)) {
case COLORSPACE_REC_601:
return fullRange ? bt601Full : bt601Lim;
case AVCOL_SPC_BT709:
case COLORSPACE_REC_709:
return fullRange ? bt709Full : bt709Lim;
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
case COLORSPACE_REC_2020:
return fullRange ? bt2020Full : bt2020Lim;
default:
// Some backends don't populate this, so we'll assume
// the host gave us what we asked for by default.
switch (getDecoderColorspace()) {
case COLORSPACE_REC_601:
return fullRange ? bt601Full : bt601Lim;
case COLORSPACE_REC_709:
return fullRange ? bt709Full : bt709Lim;
case COLORSPACE_REC_2020:
return fullRange ? bt2020Full : bt2020Lim;
default:
SDL_assert(false);
}
};
SDL_assert(false);
}
return bt601Lim;
}

25
app/streaming/video/ffmpeg-renderers/renderer.h
View File

@@ -158,6 +158,31 @@ public:
return COLORSPACE_REC_601;
}
virtual int getFrameColorspace(AVFrame* frame) {
// Prefer the colorspace field on the AVFrame itself
switch (frame->colorspace) {
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
return COLORSPACE_REC_601;
case AVCOL_SPC_BT709:
return COLORSPACE_REC_709;
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
return COLORSPACE_REC_2020;
default:
// If the colorspace is not populated, assume the encoder
// is sending the colorspace that we requested.
return getDecoderColorspace();
}
}
virtual bool isFrameFullRange(AVFrame* frame) {
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
return frame->color_range == AVCOL_RANGE_JPEG;
}
virtual bool isRenderThreadSupported() {
// Render thread is supported by default
return true;

14
app/streaming/video/ffmpeg-renderers/sdlvid.cpp
View File

@@ -10,7 +10,7 @@ SdlRenderer::SdlRenderer()
m_Renderer(nullptr),
m_Texture(nullptr),
m_SwPixelFormat(AV_PIX_FMT_NONE),
m_ColorSpace(AVCOL_SPC_UNSPECIFIED),
m_ColorSpace(-1),
m_MapFrame(false)
{
SDL_zero(m_OverlayTextures);
@@ -373,7 +373,8 @@ ReadbackRetry:
// Because the specific YUV color conversion shader is established at
// texture creation for most SDL render backends, we need to recreate
// the texture when the colorspace changes.
if (frame->colorspace != m_ColorSpace) {
int colorspace = getFrameColorspace(frame);
if (colorspace != m_ColorSpace) {
#ifdef HAVE_CUDA
if (m_CudaGLHelper != nullptr) {
delete m_CudaGLHelper;
@@ -386,7 +387,7 @@ ReadbackRetry:
m_Texture = nullptr;
}
m_ColorSpace = frame->colorspace;
m_ColorSpace = colorspace;
}
if (m_Texture == nullptr) {
@@ -410,13 +411,12 @@ ReadbackRetry:
goto Exit;
}
switch (frame->colorspace)
switch (colorspace)
{
case AVCOL_SPC_BT709:
case COLORSPACE_REC_709:
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709);
break;
case AVCOL_SPC_BT470BG:
case AVCOL_SPC_SMPTE170M:
case COLORSPACE_REC_601:
default:
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601);
break;

2
app/streaming/video/ffmpeg-renderers/sdlvid.h
View File

@@ -26,7 +26,7 @@ private:
SDL_Renderer* m_Renderer;
SDL_Texture* m_Texture;
enum AVPixelFormat m_SwPixelFormat;
enum AVColorSpace m_ColorSpace;
int m_ColorSpace;
bool m_MapFrame;
SDL_Texture* m_OverlayTextures[Overlay::OverlayMax];
SDL_Rect m_OverlayRects[Overlay::OverlayMax];

19
app/streaming/video/ffmpeg-renderers/vt.mm
View File

@@ -40,7 +40,7 @@ public:
m_FormatDesc(nullptr),
m_StreamView(nullptr),
m_DisplayLink(nullptr),
m_LastAvColorSpace(AVCOL_SPC_UNSPECIFIED),
m_LastColorSpace(-1),
m_ColorSpace(nullptr),
m_VsyncMutex(nullptr),
m_VsyncPassed(nullptr)
@@ -200,27 +200,26 @@ public:
// Reset m_ColorSpace if the colorspace changes. This can happen when
// a game enters HDR mode (Rec 601 -> Rec 2020).
if (frame->colorspace != m_LastAvColorSpace) {
int colorspace = getFrameColorspace(frame);
if (colorspace != m_LastColorSpace) {
if (m_ColorSpace != nullptr) {
CGColorSpaceRelease(m_ColorSpace);
m_ColorSpace = nullptr;
}
switch (frame->colorspace) {
case AVCOL_SPC_BT709:
switch (colorspace) {
case COLORSPACE_REC_709:
m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709);
break;
case AVCOL_SPC_BT2020_NCL:
case COLORSPACE_REC_2020:
m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020);
break;
case AVCOL_SPC_SMPTE170M:
case COLORSPACE_REC_601:
m_ColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
break;
default:
break;
}
m_LastAvColorSpace = frame->colorspace;
m_LastColorSpace = colorspace;
}
if (m_ColorSpace != nullptr) {
@@ -522,7 +521,7 @@ private:
NSView* m_StreamView;
NSTextField* m_OverlayTextFields[Overlay::OverlayMax];
CVDisplayLinkRef m_DisplayLink;
AVColorSpace m_LastAvColorSpace;
int m_LastColorSpace;
CGColorSpaceRef m_ColorSpace;
SDL_mutex* m_VsyncMutex;
SDL_cond* m_VsyncPassed;