Fix handling of preferred/compatible pixel formats with EGLRenderer and VAAPI/DRM backends

app/streaming/video/ffmpeg-renderers/drm.cpp

@@ -386,6 +386,10 @@ bool DrmRenderer::canExportEGL() {
     return true;
 }
 
+AVPixelFormat DrmRenderer::getEGLImagePixelFormat() {
+    return AV_PIX_FMT_NV12;
+}
+
 bool DrmRenderer::initializeEGL(EGLDisplay,
                                 const EGLExtensions &ext) {
     if (!ext.isSupported("EGL_EXT_image_dma_buf_import")) {

app/streaming/video/ffmpeg-renderers/drm.h

@@ -18,6 +18,7 @@ public:
     virtual bool isDirectRenderingSupported() override;
 #ifdef HAVE_EGL
     virtual bool canExportEGL() override;
+    virtual AVPixelFormat getEGLImagePixelFormat() override;
     virtual bool initializeEGL(EGLDisplay dpy, const EGLExtensions &ext) override;
     virtual ssize_t exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) override;
     virtual void freeEGLImages(EGLDisplay dpy, EGLImage[EGL_MAX_PLANES]) override;

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -59,7 +59,7 @@ SDL_Window* EGLRenderer::s_LastFailedWindow = nullptr;
 
 EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
     :
-        m_SwPixelFormat(AV_PIX_FMT_NONE),
+        m_EGLImagePixelFormat(AV_PIX_FMT_NONE),
         m_EGLDisplay(EGL_NO_DISPLAY),
         m_Textures{0},
         m_OverlayTextures{0},
@@ -153,16 +153,16 @@ void EGLRenderer::notifyOverlayUpdated(Overlay::OverlayType type)
     }
 }
 
-bool EGLRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat)
+bool EGLRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
 {
-    // Remember to keep this in sync with EGLRenderer::renderFrame()!
+    // Pixel format support should be determined by the backend renderer
-    switch (pixelFormat)
+    return m_Backend->isPixelFormatSupported(videoFormat, pixelFormat);
-    {
+}
-    case AV_PIX_FMT_NV12:
+
-        return true;
+AVPixelFormat EGLRenderer::getPreferredPixelFormat(int videoFormat)
-    default:
+{
-        return false;
+    // Pixel format preference should be determined by the backend renderer
-    }
+    return m_Backend->getPreferredPixelFormat(videoFormat);
 }
 
 void EGLRenderer::renderOverlay(Overlay::OverlayType type)
@@ -367,10 +367,10 @@ bool EGLRenderer::compileShaders() {
     SDL_assert(!m_ShaderProgram);
     SDL_assert(!m_OverlayShaderProgram);
 
-    SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE);
+    SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
 
     // XXX: TODO: other formats
-    SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12);
+    SDL_assert(m_EGLImagePixelFormat == AV_PIX_FMT_NV12);
 
     m_ShaderProgram = compileShader("egl.vert", "egl.frag");
     if (!m_ShaderProgram) {
@@ -721,23 +721,18 @@ void EGLRenderer::renderFrame(AVFrame* frame)
         return;
     }
 
-    if (frame->hw_frames_ctx != nullptr) {
+    // Find the native read-back format and load the shaders
-        // Find the native read-back format and load the shader
+    if (m_EGLImagePixelFormat == AV_PIX_FMT_NONE) {
-        if (m_SwPixelFormat == AV_PIX_FMT_NONE) {
+        m_EGLImagePixelFormat = m_Backend->getEGLImagePixelFormat();
-            auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
+        EGL_LOG(Info, "EGLImage pixel format: %d", m_EGLImagePixelFormat);
 
-            m_SwPixelFormat = hwFrameCtx->sw_format;
+        SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
-            SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE);
 
-            EGL_LOG(Info, "Selected read-back format: %d", m_SwPixelFormat);
-
-            // XXX: TODO: Handle other pixel formats
-            SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12);
         m_ColorSpace = frame->colorspace;
         m_ColorFull = frame->color_range == AVCOL_RANGE_JPEG;
 
         if (!specialize()) {
-                m_SwPixelFormat = AV_PIX_FMT_NONE;
+            m_EGLImagePixelFormat = AV_PIX_FMT_NONE;
             return;
         }
     }
@@ -750,11 +745,6 @@ void EGLRenderer::renderFrame(AVFrame* frame)
         glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
         m_glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]);
     }
-    } else {
-        // TODO: load texture for SW decoding ?
-        EGL_LOG(Error, "EGL rendering only supports hw frames");
-        return;
-    }
 
     glClear(GL_COLOR_BUFFER_BIT);
     glUseProgram(m_ShaderProgram);

app/streaming/video/ffmpeg-renderers/eglvid.h

@@ -14,6 +14,7 @@ public:
     virtual void renderFrame(AVFrame* frame) override;
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
     virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) override;
+    virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) override;
 
 private:
 
@@ -29,7 +30,7 @@ private:
     int m_ViewportWidth;
     int m_ViewportHeight;
 
-    int m_SwPixelFormat;
+    AVPixelFormat m_EGLImagePixelFormat;
     void *m_EGLDisplay;
     unsigned m_Textures[EGL_MAX_PLANES];
     unsigned m_OverlayTextures[Overlay::OverlayMax];

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -101,7 +101,7 @@ public:
         return true;
     }
 
-    virtual enum AVPixelFormat getPreferredPixelFormat(int videoFormat) {
+    virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) {
         if (videoFormat == VIDEO_FORMAT_H265_MAIN10) {
             // 10-bit YUV 4:2:0
             return AV_PIX_FMT_P010;
@@ -112,7 +112,7 @@ public:
         }
     }
 
-    virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) {
+    virtual bool isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat) {
         // By default, we only support the preferred pixel format
         return getPreferredPixelFormat(videoFormat) == pixelFormat;
     }
@@ -128,6 +128,10 @@ public:
         return false;
     }
 
+    virtual AVPixelFormat getEGLImagePixelFormat() {
+        return AV_PIX_FMT_NONE;
+    }
+
     virtual bool initializeEGL(EGLDisplay,
                                const EGLExtensions &) {
         return false;

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -483,6 +483,10 @@ VAAPIRenderer::canExportEGL() {
     return true;
 }
 
+AVPixelFormat VAAPIRenderer::getEGLImagePixelFormat() {
+    return AV_PIX_FMT_NV12;
+}
+
 bool
 VAAPIRenderer::initializeEGL(EGLDisplay,
                              const EGLExtensions &ext) {

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -44,6 +44,7 @@ public:
     virtual int getDecoderColorspace() override;
 #ifdef HAVE_EGL
     virtual bool canExportEGL() override;
+    virtual AVPixelFormat getEGLImagePixelFormat() override;
     virtual bool initializeEGL(EGLDisplay dpy, const EGLExtensions &ext) override;
     virtual ssize_t exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) override;
     virtual void freeEGLImages(EGLDisplay dpy, EGLImage[EGL_MAX_PLANES]) override;