Don't attempt to use mismatched 8-bit formats with 10-bit codecs

In addition to silently truncating the samples, this also tickles
some nasty bugs in the VF2's out-of-tree OMX decoder.
This commit is contained in:
Cameron Gutman 2024-07-06 02:27:46 -05:00
parent 2e29ef8d74
commit d085722911
2 changed files with 37 additions and 20 deletions

View File

@ -661,11 +661,22 @@ bool DrmRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFor
else {
// If we're going to need to map this as a software frame, check
// against the set of formats we support in mapSoftwareFrame().
if (pixelFormat == AV_PIX_FMT_DRM_PRIME) {
// AV_PIX_FMT_DRM_PRIME is always supported
return true;
}
else if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
switch (pixelFormat) {
case AV_PIX_FMT_P010:
return true;
default:
return false;
}
}
else {
switch (pixelFormat) {
case AV_PIX_FMT_DRM_PRIME:
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
case AV_PIX_FMT_P010:
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
return true;
@ -674,6 +685,7 @@ bool DrmRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFor
}
}
}
}
int DrmRenderer::getRendererAttributes()
{

View File

@ -80,11 +80,15 @@ bool SdlRenderer::isRenderThreadSupported()
return true;
}
bool SdlRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat)
bool SdlRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
{
if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
// SDL2 doesn't support 10-bit pixel formats
return false;
}
else {
// Remember to keep this in sync with SdlRenderer::renderFrame()!
switch (pixelFormat)
{
switch (pixelFormat) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
case AV_PIX_FMT_NV12:
@ -95,6 +99,7 @@ bool SdlRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat)
return false;
}
}
}
bool SdlRenderer::initialize(PDECODER_PARAMETERS params)
{