Fix incorrect selection if pix_fmt is overridden before ffGetFormat()

The DRM renderer does this for the out-of-tree v4l2m2m patches to ensure
we get NV12 buffers, but it ends up clobbering our own preference for
DRM_PRIME frames in the process.
This commit is contained in:
Cameron Gutman 2024-10-03 20:28:51 -05:00
parent 3279d9c3f6
commit 9936085aee

View File

@ -180,15 +180,25 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
const enum AVPixelFormat* pixFmts)
{
FFmpegVideoDecoder* decoder = (FFmpegVideoDecoder*)context->opaque;
const enum AVPixelFormat *p;
const AVPixelFormat *p;
AVPixelFormat desiredFmt;
for (p = pixFmts; *p != -1; p++) {
if (decoder->m_HwDecodeCfg) {
desiredFmt = decoder->m_HwDecodeCfg->pix_fmt;
}
else if (decoder->m_RequiredPixelFormat != AV_PIX_FMT_NONE) {
desiredFmt = decoder->m_RequiredPixelFormat;
}
else {
desiredFmt = decoder->m_FrontendRenderer->getPreferredPixelFormat(decoder->m_VideoFormat);
}
for (p = pixFmts; *p != AV_PIX_FMT_NONE; p++) {
// Only match our hardware decoding codec or preferred SW pixel
// format (if not using hardware decoding). It's crucial
// to override the default get_format() which will try
// to gracefully fall back to software decode and break us.
if (*p == (decoder->m_HwDecodeCfg ? decoder->m_HwDecodeCfg->pix_fmt : context->pix_fmt) &&
decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
if (*p == desiredFmt && decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
return *p;
}
}
@ -196,7 +206,7 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
// Failed to match the preferred pixel formats. Try non-preferred pixel format options
// for non-hwaccel decoders if we didn't have a required pixel format to use.
if (decoder->m_HwDecodeCfg == nullptr && decoder->m_RequiredPixelFormat == AV_PIX_FMT_NONE) {
for (p = pixFmts; *p != -1; p++) {
for (p = pixFmts; *p != AV_PIX_FMT_NONE; p++) {
if (decoder->m_FrontendRenderer->isPixelFormatSupported(decoder->m_VideoFormat, *p) &&
decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
return *p;