moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2026-02-16 02:30:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add software fallback for YUV444 using libswscale

Browse Source
This commit is contained in:
Cameron Gutman
2024-08-19 19:20:17 -05:00
parent ea724a05a6
commit 5765c254cd
7 changed files with 157 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
app/streaming/video/ffmpeg-renderers/sdlvid.cpp
View File

@@ -7,11 +7,19 @@
#include <SDL_syswm.h>
extern "C" {
#include <libavutil/pixdesc.h>
#include <libavutil/opt.h>
}
SdlRenderer::SdlRenderer()
: m_VideoFormat(0),
m_Renderer(nullptr),
m_Texture(nullptr),
m_ColorSpace(-1),
m_NeedsYuvToRgbConversion(false),
m_SwsContext(nullptr),
m_RgbFrame(av_frame_alloc()),
m_SwFrameMapper(this)
{
SDL_zero(m_OverlayTextures);
@@ -35,6 +43,9 @@ SdlRenderer::~SdlRenderer()
}
}
av_frame_free(&m_RgbFrame);
sws_freeContext(m_SwsContext);
if (m_Texture != nullptr) {
SDL_DestroyTexture(m_Texture);
}
@@ -82,15 +93,27 @@ bool SdlRenderer::isRenderThreadSupported()
bool SdlRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
{
if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
// SDL2 doesn't support 10-bit pixel formats
return false;
}
else if (videoFormat & VIDEO_FORMAT_MASK_YUV444) {
// SDL2 doesn't support YUV444 pixel formats
return false;
if (videoFormat & (VIDEO_FORMAT_MASK_10BIT | VIDEO_FORMAT_MASK_YUV444)) {
// SDL2 can't natively handle textures with these formats, but we can perform
// conversion on the CPU using swscale then upload them as an RGB texture.
const AVPixFmtDescriptor* formatDesc = av_pix_fmt_desc_get(pixelFormat);
if (!formatDesc) {
SDL_assert(formatDesc);
return false;
}
const int expectedPixelDepth = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? 10 : 8;
const int expectedLog2ChromaW = (videoFormat & VIDEO_FORMAT_MASK_YUV444) ? 0 : 1;
const int expectedLog2ChromaH = (videoFormat & VIDEO_FORMAT_MASK_YUV444) ? 0 : 1;
return formatDesc->comp[0].depth == expectedPixelDepth &&
formatDesc->log2_chroma_w == expectedLog2ChromaW &&
formatDesc->log2_chroma_h == expectedLog2ChromaH;
}
else {
// The formats listed below are natively supported by SDL, so it can handle
// YUV to RGB conversion on the GPU using pixel shaders.
//
// Remember to keep this in sync with SdlRenderer::renderFrame()!
switch (pixelFormat) {
case AV_PIX_FMT_YUV420P:
@@ -112,8 +135,8 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params)
m_VideoFormat = params->videoFormat;
m_SwFrameMapper.setVideoFormat(m_VideoFormat);
if (params->videoFormat & (VIDEO_FORMAT_MASK_10BIT | VIDEO_FORMAT_MASK_YUV444)) {
// SDL doesn't support rendering YUV444 or 10-bit textures yet
if (params->videoFormat & VIDEO_FORMAT_MASK_10BIT) {
// SDL doesn't support rendering HDR yet
return false;
}
@@ -233,6 +256,11 @@ void SdlRenderer::renderOverlay(Overlay::OverlayType type)
}
}
void SdlRenderer::ffNoopFree(void*, uint8_t*)
{
// Nothing
}
void SdlRenderer::renderFrame(AVFrame* frame)
{
int err;
@@ -286,6 +314,7 @@ ReadbackRetry:
Uint32 sdlFormat;
// Remember to keep this in sync with SdlRenderer::isPixelFormatSupported()!
m_NeedsYuvToRgbConversion = false;
switch (frame->format)
{
case AV_PIX_FMT_YUV420P:
@@ -300,27 +329,75 @@ ReadbackRetry:
sdlFormat = SDL_PIXELFORMAT_NV21;
break;
default:
SDL_assert(false);
goto Exit;
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Performing color conversion on CPU due to lack of SDL support for format: %u",
frame->format);
sdlFormat = SDL_PIXELFORMAT_XRGB8888;
m_NeedsYuvToRgbConversion = true;
break;
}
switch (colorspace)
{
case COLORSPACE_REC_709:
SDL_assert(!isFrameFullRange(frame));
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709);
break;
case COLORSPACE_REC_601:
if (isFrameFullRange(frame)) {
// SDL's JPEG mode is Rec 601 Full Range
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_JPEG);
if (m_NeedsYuvToRgbConversion) {
m_RgbFrame->width = frame->width;
m_RgbFrame->height = frame->height;
m_RgbFrame->format = AV_PIX_FMT_BGR0;
sws_freeContext(m_SwsContext);
m_SwsContext = sws_alloc_context();
if (!m_SwsContext) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"sws_alloc_context() failed");
goto Exit;
}
else {
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601);
AVDictionary *options { nullptr };
av_dict_set_int(&options, "srcw", frame->width, 0);
av_dict_set_int(&options, "srch", frame->height, 0);
av_dict_set_int(&options, "src_format", frame->format, 0);
av_dict_set_int(&options, "dstw", m_RgbFrame->width, 0);
av_dict_set_int(&options, "dsth", m_RgbFrame->height, 0);
av_dict_set_int(&options, "dst_format", m_RgbFrame->format, 0);
av_dict_set_int(&options, "threads", std::min(SDL_GetCPUCount(), 4), 0); // Up to 4 threads
err = av_opt_set_dict(m_SwsContext, &options);
av_dict_free(&options);
if (err < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"av_opt_set_dict() failed: %s",
av_make_error_string(string, sizeof(string), err));
goto Exit;
}
err = sws_init_context(m_SwsContext, nullptr, nullptr);
if (err < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"sws_init_context() failed: %s",
av_make_error_string(string, sizeof(string), err));
goto Exit;
}
}
else {
// SDL will perform YUV conversion on the GPU
switch (colorspace)
{
case COLORSPACE_REC_709:
SDL_assert(!isFrameFullRange(frame));
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709);
break;
case COLORSPACE_REC_601:
if (isFrameFullRange(frame)) {
// SDL's JPEG mode is Rec 601 Full Range
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_JPEG);
}
else {
SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT601);
}
break;
default:
break;
}
break;
default:
break;
}
m_Texture = SDL_CreateTexture(m_Renderer,
@@ -371,7 +448,7 @@ ReadbackRetry:
frame->data[2],
frame->linesize[2]);
}
else {
else if (!m_NeedsYuvToRgbConversion) {
#if SDL_VERSION_ATLEAST(2, 0, 15)
// SDL_UpdateNVTexture is not supported on all renderer backends,
// (notably not DX9), so we must have a fallback in case it's not
@@ -430,6 +507,37 @@ ReadbackRetry:
SDL_UnlockTexture(m_Texture);
}
}
else {
// We have a pixel format that SDL doesn't natively support, so we must use
// swscale to convert the YUV frame into an RGB frame to upload to the GPU.
uint8_t* pixels;
int texturePitch;
err = SDL_LockTexture(m_Texture, nullptr, (void**)&pixels, &texturePitch);
if (err < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"SDL_LockTexture() failed: %s",
SDL_GetError());
goto Exit;
}
// Create a buffer to wrap our locked texture buffer
m_RgbFrame->buf[0] = av_buffer_create(pixels, m_RgbFrame->height * texturePitch, ffNoopFree, nullptr, 0);
m_RgbFrame->data[0] = pixels;
m_RgbFrame->linesize[0] = texturePitch;
// Perform multi-threaded color conversion into the locked texture buffer
err = sws_scale_frame(m_SwsContext, m_RgbFrame, frame);
av_buffer_unref(&m_RgbFrame->buf[0]);
SDL_UnlockTexture(m_Texture);
if (err < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"sws_scale_frame() failed: %s",
av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, err));
goto Exit;
}
}
SDL_RenderClear(m_Renderer);