Don't attempt to use mismatched 8-bit formats with 10-bit codecs

In addition to silently truncating the samples, this also tickles
some nasty bugs in the VF2's out-of-tree OMX decoder.
This commit is contained in:
Cameron Gutman 2024-07-06 02:27:46 -05:00
parent 2e29ef8d74
commit d085722911
2 changed files with 37 additions and 20 deletions

View File

@ -661,16 +661,28 @@ bool DrmRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFor
else { else {
// If we're going to need to map this as a software frame, check // If we're going to need to map this as a software frame, check
// against the set of formats we support in mapSoftwareFrame(). // against the set of formats we support in mapSoftwareFrame().
switch (pixelFormat) { if (pixelFormat == AV_PIX_FMT_DRM_PRIME) {
case AV_PIX_FMT_DRM_PRIME: // AV_PIX_FMT_DRM_PRIME is always supported
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
case AV_PIX_FMT_P010:
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
return true; return true;
default: }
return false; else if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
switch (pixelFormat) {
case AV_PIX_FMT_P010:
return true;
default:
return false;
}
}
else {
switch (pixelFormat) {
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
return true;
default:
return false;
}
} }
} }
} }

View File

@ -80,20 +80,25 @@ bool SdlRenderer::isRenderThreadSupported()
return true; return true;
} }
bool SdlRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat) bool SdlRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
{ {
// Remember to keep this in sync with SdlRenderer::renderFrame()! if (videoFormat & VIDEO_FORMAT_MASK_10BIT) {
switch (pixelFormat) // SDL2 doesn't support 10-bit pixel formats
{
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
return true;
default:
return false; return false;
} }
else {
// Remember to keep this in sync with SdlRenderer::renderFrame()!
switch (pixelFormat) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
return true;
default:
return false;
}
}
} }
bool SdlRenderer::initialize(PDECODER_PARAMETERS params) bool SdlRenderer::initialize(PDECODER_PARAMETERS params)