Unify handling of DRM devices between DRM and VAAPI

SDL may not be able to give us a DRM FD for Vulkan windows.

app/streaming/streamutils.cpp

@@ -1,6 +1,7 @@
 #include "streamutils.h"
 
 #include <Qt>
+#include <QDir>
 
 #ifdef Q_OS_DARWIN
 #include <ApplicationServices/ApplicationServices.h>
@@ -10,6 +11,13 @@
 #include <Windows.h>
 #endif
 
+#ifdef Q_OS_UNIX
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <SDL_syswm.h>
+#endif
+
 #ifdef Q_OS_LINUX
 #include <sys/auxv.h>
 
@@ -306,3 +314,90 @@ bool StreamUtils::getNativeDesktopMode(int displayIndex, SDL_DisplayMode* mode,
     return true;
 }
 
+int StreamUtils::getDrmFdForWindow(SDL_Window* window, bool* mustClose)
+{
+    *mustClose = false;
+
+#if defined(SDL_VIDEO_DRIVER_KMSDRM) && SDL_VERSION_ATLEAST(2, 0, 15)
+    SDL_SysWMinfo info;
+    SDL_VERSION(&info.version);
+    if (!SDL_GetWindowWMInfo(window, &info)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_GetWindowWMInfo() failed: %s",
+                     SDL_GetError());
+        return -1;
+    }
+
+    if (info.subsystem == SDL_SYSWM_KMSDRM) {
+        // If SDL has an FD, share that
+        if (info.info.kmsdrm.drm_fd >= 0) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Sharing DRM FD with SDL");
+            return info.info.kmsdrm.drm_fd;
+        }
+        else {
+            char path[128];
+            snprintf(path, sizeof(path), "/dev/dri/card%u", info.info.kmsdrm.dev_index);
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Opening DRM FD from SDL by path: %s",
+                        path);
+            int fd = open(path, O_RDWR | O_CLOEXEC);
+            if (fd >= 0) {
+                *mustClose = true;
+            }
+            return fd;
+        }
+    }
+#endif
+
+    return -1;
+}
+
+int StreamUtils::getDrmFd(bool preferRenderNode)
+{
+#ifdef Q_OS_UNIX
+    const char* userDevice = SDL_getenv("DRM_DEV");
+    if (userDevice != nullptr) {
+        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                    "Opening user-specified DRM device: %s",
+                    userDevice);
+
+        return open(userDevice, O_RDWR | O_CLOEXEC);
+    }
+    else {
+        QDir driDir("/dev/dri");
+        int fd;
+
+        // We have to explicitly ask for devices to be returned
+        driDir.setFilter(QDir::Files | QDir::System);
+
+        if (preferRenderNode) {
+            // Try a render node first since we aren't using DRM for output in this codepath
+            for (QFileInfo& node : driDir.entryInfoList(QStringList("renderD*"))) {
+                QByteArray absolutePath = node.absoluteFilePath().toUtf8();
+                fd = open(absolutePath.constData(), O_RDWR | O_CLOEXEC);
+                if (fd >= 0) {
+                    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                                "Opened DRM render node: %s",
+                                absolutePath.constData());
+                    return fd;
+                }
+            }
+        }
+
+        // If that fails, try to use a primary node and hope for the best
+        for (QFileInfo& node : driDir.entryInfoList(QStringList("card*"))) {
+            QByteArray absolutePath = node.absoluteFilePath().toUtf8();
+            fd = open(absolutePath.constData(), O_RDWR | O_CLOEXEC);
+            if (fd >= 0) {
+                SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                            "Opened DRM primary node: %s",
+                            absolutePath.constData());
+                return fd;
+            }
+        }
+    }
+#endif
+
+    return -1;
+}

app/streaming/streamutils.h

@@ -36,4 +36,10 @@ public:
 
     static
     bool hasFastAes();
+
+    static
+    int getDrmFdForWindow(SDL_Window* window, bool* needsClose);
+
+    static
+    int getDrmFd(bool preferRenderNode);
 };

app/streaming/video/ffmpeg-renderers/drm.cpp

@@ -72,15 +72,6 @@ extern "C" {
 
 #include <Limelight.h>
 
-// HACK: Avoid including X11 headers which conflict with QDir
-#ifdef SDL_VIDEO_DRIVER_X11
-#undef SDL_VIDEO_DRIVER_X11
-#endif
-
-#include <SDL_syswm.h>
-
-#include <QDir>
-
 #include <map>
 
 // This map is used to lookup characteristics of a given DRM format
@@ -149,6 +140,7 @@ DrmRenderer::DrmRenderer(AVHWDeviceType hwDeviceType, IFFmpegRenderer *backendRe
       m_HwDeviceType(hwDeviceType),
       m_HwContext(nullptr),
       m_DrmFd(-1),
+      m_DrmIsMaster(false),
       m_SdlOwnsDrmFd(false),
       m_SupportsDirectRendering(false),
       m_VideoFormat(0),
@@ -265,6 +257,9 @@ bool DrmRenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary**
 
 void DrmRenderer::prepareToRender()
 {
+    // Retake DRM master if we dropped it earlier
+    drmSetMaster(m_DrmFd);
+
     // Create a dummy renderer to force SDL to complete the modesetting
     // operation that the KMSDRM backend keeps pending until the next
     // time we swap buffers. We have to do this before we enumerate
@@ -350,68 +345,26 @@ bool DrmRenderer::initialize(PDECODER_PARAMETERS params)
     m_VideoFormat = params->videoFormat;
     m_SwFrameMapper.setVideoFormat(params->videoFormat);
 
-#if SDL_VERSION_ATLEAST(2, 0, 15)
+    // Try to get the FD that we're sharing with SDL
-    SDL_SysWMinfo info;
+    m_DrmFd = StreamUtils::getDrmFdForWindow(m_Window, &m_SdlOwnsDrmFd);
+    if (m_DrmFd >= 0) {
+        // If we got a DRM FD for the window, we can render to it
+        m_DrmIsMaster = true;
 
-    SDL_VERSION(&info.version);
+        // If we just opened a new FD, let's drop master on it
-
+        // so SDL can take master for Vulkan rendering. We'll
-    if (!SDL_GetWindowWMInfo(params->window, &info)) {
+        // regrab master later if we end up direct rendering.
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+        if (!m_SdlOwnsDrmFd) {
-                     "SDL_GetWindowWMInfo() failed: %s",
+            drmDropMaster(m_DrmFd);
-                     SDL_GetError());
-        return false;
-    }
-
-    if (info.subsystem == SDL_SYSWM_KMSDRM) {
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Sharing DRM FD with SDL");
-
-        SDL_assert(info.info.kmsdrm.drm_fd >= 0);
-        m_DrmFd = info.info.kmsdrm.drm_fd;
-        m_SdlOwnsDrmFd = true;
-    }
-    else
-#endif
-    {
-        const char* userDevice = SDL_getenv("DRM_DEV");
-        if (userDevice != nullptr) {
-            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                        "Opening user-specified DRM device: %s",
-                        userDevice);
-
-            m_DrmFd = open(userDevice, O_RDWR | O_CLOEXEC);
         }
-        else {
+    }
-            QDir driDir("/dev/dri");
+    else {
+        // Try to open any DRM render node
+        m_DrmFd = StreamUtils::getDrmFd(true);
 
-            // We have to explicitly ask for devices to be returned
+        // Drop master in case we somehow got a primary node
-            driDir.setFilter(QDir::Files | QDir::System);
+        if (m_DrmFd >= 0) {
-
+            drmDropMaster(m_DrmFd);
-            // Try a render node first since we aren't using DRM for output in this codepath
-            for (QFileInfo& node : driDir.entryInfoList(QStringList("renderD*"))) {
-                QByteArray absolutePath = node.absoluteFilePath().toUtf8();
-                m_DrmFd = open(absolutePath.constData(), O_RDWR | O_CLOEXEC);
-                if (m_DrmFd >= 0) {
-                    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                                "Opened DRM render node: %s",
-                                absolutePath.constData());
-                    break;
-                }
-            }
-
-            // If that fails, try to use a primary node and hope for the best
-            if (m_DrmFd < 0) {
-                for (QFileInfo& node : driDir.entryInfoList(QStringList("card*"))) {
-                    QByteArray absolutePath = node.absoluteFilePath().toUtf8();
-                    m_DrmFd = open(absolutePath.constData(), O_RDWR | O_CLOEXEC);
-                    if (m_DrmFd >= 0) {
-                        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                                    "Opened DRM primary node: %s",
-                                    absolutePath.constData());
-                        break;
-                    }
-                }
-            }
         }
     }
 
@@ -489,7 +442,7 @@ bool DrmRenderer::initialize(PDECODER_PARAMETERS params)
     // If we're not sharing the DRM FD with SDL, that means we don't
     // have DRM master, so we can't call drmModeSetPlane(). We can
     // use EGLRenderer or SDLRenderer to render in this situation.
-    if (!m_SdlOwnsDrmFd) {
+    if (!m_DrmIsMaster) {
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                     "Direct rendering via DRM is disabled");
         return DIRECT_RENDERING_INIT_FAILED;

app/streaming/video/ffmpeg-renderers/drm.h

@@ -86,6 +86,7 @@ private:
     AVHWDeviceType m_HwDeviceType;
     AVBufferRef* m_HwContext;
     int m_DrmFd;
+    bool m_DrmIsMaster;
     bool m_SdlOwnsDrmFd;
     bool m_SupportsDirectRendering;
     int m_VideoFormat;

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -123,16 +123,33 @@ VAAPIRenderer::openDisplay(SDL_Window* window)
     }
 #if defined(SDL_VIDEO_DRIVER_KMSDRM) && defined(HAVE_LIBVA_DRM) && SDL_VERSION_ATLEAST(2, 0, 15)
     else if (info.subsystem == SDL_SYSWM_KMSDRM) {
-        SDL_assert(info.info.kmsdrm.drm_fd >= 0);
-
         // It's possible to enter this function several times as we're probing VA drivers.
         // Make sure to only duplicate the DRM FD the first time through.
         if (m_DrmFd < 0) {
+            // Try to get the FD that we're sharing with SDL
+            bool mustCloseFd = false;
+            int fd = StreamUtils::getDrmFdForWindow(window, &mustCloseFd);
+            if (fd < 0) {
+                // Try to open any DRM render node
+                fd = StreamUtils::getDrmFd(true);
+                if (fd < 0) {
+                    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                                 "Failed to open DRM render node: %d",
+                                 errno);
+                    return nullptr;
+                }
+            }
+
             // If the KMSDRM FD is not a render node FD, open the render node for libva to use.
             // Since libva 2.20, using a primary node will fail in vaGetDriverNames().
-            if (drmGetNodeTypeFromFd(info.info.kmsdrm.drm_fd) != DRM_NODE_RENDER) {
+            if (drmGetNodeTypeFromFd(fd) != DRM_NODE_RENDER) {
-                char* renderNodePath = drmGetRenderDeviceNameFromFd(info.info.kmsdrm.drm_fd);
+                char* renderNodePath = drmGetRenderDeviceNameFromFd(fd);
                 if (renderNodePath) {
+                    // Don't need the primary node FD anymore
+                    if (mustCloseFd) {
+                        close(fd);
+                    }
+
                     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                                 "Opening render node for VAAPI: %s",
                                 renderNodePath);
@@ -148,13 +165,13 @@ VAAPIRenderer::openDisplay(SDL_Window* window)
                 else {
                     SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                                 "Failed to get render node path. Using the SDL FD directly.");
-                    m_DrmFd = dup(info.info.kmsdrm.drm_fd);
+                    m_DrmFd = mustCloseFd ? fd : dup(fd);
                 }
             }
             else {
                 SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                             "KMSDRM FD is already a render node. Using the SDL FD directly.");
-                m_DrmFd = dup(info.info.kmsdrm.drm_fd);
+                m_DrmFd = mustCloseFd ? fd : dup(fd);
             }
         }