Render DRM-PRIME frames as opaque images

We can't always assume they are NV12. Even if they _are_ NV12,
they may have DRM format modifiers that are incompatible with
simply assuming linear Y and UV buffers (such as tiling).

app/resources.qrc

@@ -29,6 +29,8 @@
         <file alias="ModeSeven.ttf">ModeSeven.ttf</file>
         <file alias="egl_nv12.frag">shaders/egl_nv12.frag</file>
         <file alias="egl_nv12.vert">shaders/egl_nv12.vert</file>
+        <file alias="egl_opaque.frag">shaders/egl_opaque.frag</file>
+        <file alias="egl_opaque.vert">shaders/egl_opaque.vert</file>
         <file alias="egl_overlay.frag">shaders/egl_overlay.frag</file>
         <file alias="egl_overlay.vert">shaders/egl_overlay.vert</file>
     </qresource>

app/shaders/egl_opaque.frag

@@ -0,0 +1,12 @@
+#version 300 es
+#extension GL_OES_EGL_image_external : require
+precision mediump float;
+out vec4 FragColor;
+
+in vec2 vTextCoord;
+
+uniform samplerExternalOES uTexture;
+
+void main() {
+        FragColor = texture2D(uTexture, vTextCoord);
+}

app/shaders/egl_opaque.vert

@@ -0,0 +1,10 @@
+#version 300 es
+
+layout (location = 0) in vec2 aPosition; // 2D: X,Y
+layout (location = 1) in vec2 aTexCoord;
+out vec2 vTextCoord;
+
+void main() {
+	vTextCoord = aTexCoord;
+	gl_Position = vec4(aPosition, 0, 1);
+}

app/streaming/video/ffmpeg-renderers/drm.cpp

@@ -26,6 +26,7 @@ DrmRenderer::DrmRenderer()
       m_CurrentFbId(0)
 {
 #ifdef HAVE_EGL
+    m_EGLExtDmaBuf = false;
     m_eglCreateImage = nullptr;
     m_eglCreateImageKHR = nullptr;
     m_eglDestroyImage = nullptr;
@@ -396,7 +397,8 @@ bool DrmRenderer::canExportEGL() {
 }
 
 AVPixelFormat DrmRenderer::getEGLImagePixelFormat() {
-    return AV_PIX_FMT_NV12;
+    // This tells EGLRenderer to treat the EGLImage as a single opaque texture
+    return AV_PIX_FMT_DRM_PRIME;
 }
 
 bool DrmRenderer::initializeEGL(EGLDisplay,
@@ -407,6 +409,8 @@ bool DrmRenderer::initializeEGL(EGLDisplay,
         return false;
     }
 
+    m_EGLExtDmaBuf = ext.isSupported("EGL_EXT_image_dma_buf_import_modifiers");
+
     // NB: eglCreateImage() and eglCreateImageKHR() have slightly different definitions
     m_eglCreateImage = (typeof(m_eglCreateImage))eglGetProcAddress("eglCreateImage");
     m_eglCreateImageKHR = (typeof(m_eglCreateImageKHR))eglGetProcAddress("eglCreateImageKHR");
@@ -432,54 +436,103 @@ ssize_t DrmRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
 
     SDL_assert(drmFrame->nb_objects == 1);
     SDL_assert(drmFrame->nb_layers == 1);
-    SDL_assert(drmFrame->layers[0].nb_planes == 2);
+
+    const int MAX_ATTRIB_COUNT = 30;
+    EGLAttrib attribs[MAX_ATTRIB_COUNT] = {
+        EGL_LINUX_DRM_FOURCC_EXT, (EGLAttrib)drmFrame->layers[0].format,
+        EGL_WIDTH, frame->width,
+        EGL_HEIGHT, frame->height,
+    };
+    int attribIndex = 6;
 
     for (int i = 0; i < drmFrame->layers[0].nb_planes; ++i) {
         const auto &plane = drmFrame->layers[0].planes[i];
         const auto &object = drmFrame->objects[plane.object_index];
 
-        const int EGL_ATTRIB_COUNT = 13;
+        switch (i) {
-        EGLAttrib attribs[EGL_ATTRIB_COUNT] = {
+        case 0:
-            EGL_LINUX_DRM_FOURCC_EXT, i == 0 ? DRM_FORMAT_R8 : DRM_FORMAT_GR88,
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE0_FD_EXT;
-            EGL_WIDTH, i == 0 ? frame->width : frame->width / 2,
+            attribs[attribIndex++] = object.fd;
-            EGL_HEIGHT, i == 0 ? frame->height : frame->height / 2,
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
-            EGL_DMA_BUF_PLANE0_FD_EXT, object.fd,
+            attribs[attribIndex++] = plane.offset;
-            EGL_DMA_BUF_PLANE0_OFFSET_EXT, plane.offset,
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
-            EGL_DMA_BUF_PLANE0_PITCH_EXT, plane.pitch,
+            attribs[attribIndex++] = plane.pitch;
-            EGL_NONE
+            if (m_EGLExtDmaBuf && object.format_modifier != DRM_FORMAT_MOD_INVALID) {
-        };
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT;
-
+                attribs[attribIndex++] = (EGLint)(object.format_modifier & 0xFFFFFFFF);
-        if (m_eglCreateImage) {
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT;
-            images[i] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
+                attribs[attribIndex++] = (EGLint)(object.format_modifier >> 32);
-                                         EGL_LINUX_DMA_BUF_EXT,
-                                         nullptr, attribs);
-            if (!images[i]) {
-                SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                             "eglCreateImage() Failed: %d", eglGetError());
-                goto fail;
             }
+            break;
+
+        case 1:
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE1_FD_EXT;
+            attribs[attribIndex++] = object.fd;
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
+            attribs[attribIndex++] = plane.offset;
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
+            attribs[attribIndex++] = plane.pitch;
+            if (m_EGLExtDmaBuf && object.format_modifier != DRM_FORMAT_MOD_INVALID) {
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT;
+                attribs[attribIndex++] = (EGLint)(object.format_modifier & 0xFFFFFFFF);
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT;
+                attribs[attribIndex++] = (EGLint)(object.format_modifier >> 32);
+            }
+            break;
+
+        case 2:
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE2_FD_EXT;
+            attribs[attribIndex++] = object.fd;
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
+            attribs[attribIndex++] = plane.offset;
+            attribs[attribIndex++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
+            attribs[attribIndex++] = plane.pitch;
+            if (m_EGLExtDmaBuf && object.format_modifier != DRM_FORMAT_MOD_INVALID) {
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT;
+                attribs[attribIndex++] = (EGLint)(object.format_modifier & 0xFFFFFFFF);
+                attribs[attribIndex++] = EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT;
+                attribs[attribIndex++] = (EGLint)(object.format_modifier >> 32);
+            }
+            break;
+
+        default:
+            Q_UNREACHABLE();
         }
-        else {
-            // Cast the EGLAttrib array elements to EGLint for the KHR extension
-            EGLint intAttribs[EGL_ATTRIB_COUNT];
-            for (int i = 0; i < EGL_ATTRIB_COUNT; i++) {
-                intAttribs[i] = (EGLint)attribs[i];
-            }
-
-            images[i] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
-                                            EGL_LINUX_DMA_BUF_EXT,
-                                            nullptr, intAttribs);
-            if (!images[i]) {
-                SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                             "eglCreateImageKHR() Failed: %d", eglGetError());
-                goto fail;
-            }
-        }
-
-        ++count;
     }
 
-    return count;
+    // Terminate the attribute list
+    attribs[attribIndex++] = EGL_NONE;
+    SDL_assert(attribIndex <= MAX_ATTRIB_COUNT);
+
+    // Our EGLImages are non-planar, so we only populate the first entry
+    if (m_eglCreateImage) {
+        images[0] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
+                                     EGL_LINUX_DMA_BUF_EXT,
+                                     nullptr, attribs);
+        if (!images[0]) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "eglCreateImage() Failed: %d", eglGetError());
+            goto fail;
+        }
+    }
+    else {
+        // Cast the EGLAttrib array elements to EGLint for the KHR extension
+        EGLint intAttribs[MAX_ATTRIB_COUNT];
+        for (int i = 0; i < MAX_ATTRIB_COUNT; i++) {
+            intAttribs[i] = (EGLint)attribs[i];
+        }
+
+        images[0] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
+                                        EGL_LINUX_DMA_BUF_EXT,
+                                        nullptr, intAttribs);
+        if (!images[0]) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "eglCreateImageKHR() Failed: %d", eglGetError());
+            goto fail;
+        }
+    }
+
+    return 1;
 
 fail:
     freeEGLImages(dpy, images);
@@ -487,14 +540,16 @@ fail:
 }
 
 void DrmRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
-    for (size_t i = 0; i < EGL_MAX_PLANES; ++i) {
+    if (m_eglDestroyImage) {
-        if (m_eglDestroyImage) {
+        m_eglDestroyImage(dpy, images[0]);
-            m_eglDestroyImage(dpy, images[i]);
-        }
-        else {
-            m_eglDestroyImageKHR(dpy, images[i]);
-        }
     }
+    else {
+        m_eglDestroyImageKHR(dpy, images[0]);
+    }
+
+    // Our EGLImages are non-planar
+    SDL_assert(images[1] == 0);
+    SDL_assert(images[2] == 0);
 }
 
 #endif

app/streaming/video/ffmpeg-renderers/drm.h

@@ -35,6 +35,7 @@ private:
     SDL_Rect m_OutputRect;
 
 #ifdef HAVE_EGL
+    bool m_EGLExtDmaBuf;
     PFNEGLCREATEIMAGEPROC m_eglCreateImage;
     PFNEGLDESTROYIMAGEPROC m_eglDestroyImage;
     PFNEGLCREATEIMAGEKHRPROC m_eglCreateImageKHR;

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -384,6 +384,14 @@ bool EGLRenderer::compileShaders() {
         m_ShaderProgramParams[NV12_PARAM_PLANE1] = glGetUniformLocation(m_ShaderProgram, "plane1");
         m_ShaderProgramParams[NV12_PARAM_PLANE2] = glGetUniformLocation(m_ShaderProgram, "plane2");
     }
+    else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
+        m_ShaderProgram = compileShader("egl_opaque.vert", "egl_opaque.frag");
+        if (!m_ShaderProgram) {
+            return false;
+        }
+
+        m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE] = glGetUniformLocation(m_ShaderProgram, "uTexture");
+    }
     else {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Unsupported EGL pixel format: %d",
@@ -785,13 +793,16 @@ void EGLRenderer::renderFrame(AVFrame* frame)
     glUseProgram(m_ShaderProgram);
     m_glBindVertexArrayOES(m_VAO);
 
-    // Bind parameters for the NV12 shaders
+    // Bind parameters for the shaders
     if (m_EGLImagePixelFormat == AV_PIX_FMT_NV12) {
         glUniformMatrix3fv(m_ShaderProgramParams[NV12_PARAM_YUVMAT], 1, GL_FALSE, getColorMatrix());
         glUniform3fv(m_ShaderProgramParams[NV12_PARAM_OFFSET], 1, getColorOffsets());
         glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE1], 0);
         glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE2], 1);
     }
+    else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
+        glUniform1i(m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE], 0);
+    }
 
     glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
 

app/streaming/video/ffmpeg-renderers/eglvid.h

@@ -55,6 +55,7 @@ private:
 #define NV12_PARAM_OFFSET 1
 #define NV12_PARAM_PLANE1 2
 #define NV12_PARAM_PLANE2 3
+#define OPAQUE_PARAM_TEXTURE 0
     int m_ShaderProgramParams[4];
 
 #define OVERLAY_PARAM_TEXTURE 0