From 34fa7167b1db3d633cf64ff542d43bdf0ef1c171 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Fri, 5 Jul 2024 23:04:10 -0500
Subject: [PATCH] Force AV_CODEC_CAP_HARDWARE set for OMX decoders

There are at least 2 out-of-tree OMX decoder implementations
that both lack AV_CODEC_CAP_HARDWARE, so they trick us into
thinking that neither of them is hardware accelerated.

Fixes false decoder warnings when linked against the patched
FFmpeg builds shipping on VisionFive 2 and LicheePi 4A SBCs.
---
 app/streaming/video/ffmpeg.cpp | 24 ++++++++++++++++++++----
 app/streaming/video/ffmpeg.h   |  4 ++++
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp
index e3c1698b..50222372 100644
--- a/app/streaming/video/ffmpeg.cpp
+++ b/app/streaming/video/ffmpeg.cpp
@@ -84,7 +84,7 @@ static const QMap<QString, int> k_NonHwaccelCodecInfo = {
 bool FFmpegVideoDecoder::isHardwareAccelerated()
 {
     return m_HwDecodeCfg != nullptr ||
-            (m_VideoDecoderCtx->codec->capabilities & AV_CODEC_CAP_HARDWARE) != 0;
+            (getAVCodecCapabilities(m_VideoDecoderCtx->codec) & AV_CODEC_CAP_HARDWARE) != 0;
 }
 
 bool FFmpegVideoDecoder::isAlwaysFullScreen()
@@ -1165,6 +1165,22 @@ bool FFmpegVideoDecoder::isDecoderIgnored(const AVCodec *decoder)
     return false;
 }
 
+int FFmpegVideoDecoder::getAVCodecCapabilities(const AVCodec *codec)
+{
+    int caps = codec->capabilities;
+
+    // There are a bunch of out-of-tree OMX decoder implementations
+    // from various SBC manufacturers that all seem to forget to set
+    // AV_CODEC_CAP_HARDWARE (probably because the upstream OMX code
+    // also doesn't set it). Avoid a false decoder warning on startup
+    // by setting it ourselves.
+    if (QString::fromUtf8(codec->name).endsWith("_omx", Qt::CaseInsensitive)) {
+        caps |= AV_CODEC_CAP_HARDWARE;
+    }
+
+    return caps;
+}
+
 bool FFmpegVideoDecoder::tryInitializeHwAccelDecoder(PDECODER_PARAMETERS params, int pass, QSet<const AVCodec*>& terminallyFailedHardwareDecoders)
 {
     const AVCodec* decoder;
@@ -1188,7 +1204,7 @@ bool FFmpegVideoDecoder::tryInitializeHwAccelDecoder(PDECODER_PARAMETERS params,
         }
 
         // Skip non-hwaccel hardware decoders
-        if (decoder->capabilities & AV_CODEC_CAP_HARDWARE) {
+        if (getAVCodecCapabilities(decoder) & AV_CODEC_CAP_HARDWARE) {
             continue;
         }
 
@@ -1322,7 +1338,7 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
             }
 
             // Skip software/hybrid decoders and normal hwaccel decoders (which were handled in the loop above)
-            if (!(decoder->capabilities & AV_CODEC_CAP_HARDWARE)) {
+            if (!(getAVCodecCapabilities(decoder) & AV_CODEC_CAP_HARDWARE)) {
                 continue;
             }
 
@@ -1372,7 +1388,7 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
             // hardware and software depending on whether an hwaccel is supplied.
             // Instead, we tell tryInitializeRendererForUnknownDecoder() not to
             // try hwaccel for this decoder.
-            if (decoder->capabilities & AV_CODEC_CAP_HARDWARE) {
+            if (getAVCodecCapabilities(decoder) & AV_CODEC_CAP_HARDWARE) {
                 continue;
             }
 
diff --git a/app/streaming/video/ffmpeg.h b/app/streaming/video/ffmpeg.h
index 6eb7a571..ecf6bf95 100644
--- a/app/streaming/video/ffmpeg.h
+++ b/app/streaming/video/ffmpeg.h
@@ -45,8 +45,12 @@ private:
 
     bool createFrontendRenderer(PDECODER_PARAMETERS params, bool useAlternateFrontend);
 
+    static
     bool isDecoderIgnored(const AVCodec* decoder);
 
+    static
+    int getAVCodecCapabilities(const AVCodec *codec);
+
     bool tryInitializeHwAccelDecoder(PDECODER_PARAMETERS params,
                                      int pass,
                                      QSet<const AVCodec*>& terminallyFailedHardwareDecoders);