WIP Wayland rendering code for VAAPI

app/app.pro

@@ -51,6 +51,12 @@ unix:!macx {
         CONFIG += ffmpeg
 
         packagesExist(libva) {
+            packagesExist(libva-x11) {
+                CONFIG += libva-x11
+            }
+            packagesExist(libva-wayland) {
+                CONFIG += libva-wayland
+            }
             CONFIG += libva
         }
 
@@ -116,10 +122,23 @@ ffmpeg {
 libva {
     message(VAAPI renderer selected)
 
+    PKGCONFIG += libva
     DEFINES += HAVE_LIBVA
     SOURCES += streaming/video/ffmpeg-renderers/vaapi.cpp
     HEADERS += streaming/video/ffmpeg-renderers/vaapi.h
 }
+libva-x11 {
+    message(VAAPI X11 support enabled)
+
+    PKGCONFIG += libva-x11
+    DEFINES += HAVE_LIBVA_X11
+}
+libva-wayland {
+    message(VAAPI Wayland support enabled)
+
+    PKGCONFIG += libva-wayland
+    DEFINES += HAVE_LIBVA_WAYLAND
+}
 libvdpau {
     message(VDPAU renderer selected)
 

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -6,9 +6,7 @@
 #include <SDL_syswm.h>
 
 VAAPIRenderer::VAAPIRenderer()
-    : m_HwContext(nullptr),
+    : m_HwContext(nullptr)
-      m_X11VaLibHandle(nullptr),
-      m_vaPutSurface(nullptr)
 {
 
 }
@@ -16,14 +14,30 @@ VAAPIRenderer::VAAPIRenderer()
 VAAPIRenderer::~VAAPIRenderer()
 {
     if (m_HwContext != nullptr) {
-        av_buffer_unref(&m_HwContext);
+        AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
-    }
+        AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
 
-    if (m_X11VaLibHandle != nullptr) {
+        // Hold onto this VADisplay since we'll need it to uninitialize VAAPI
-        dlclose(m_X11VaLibHandle);
+        VADisplay display = vaDeviceContext->display;
+
+        av_buffer_unref(&m_HwContext);
+
+        if (display) {
+            vaTerminate(display);
+        }
     }
 }
 
+void VAAPIRenderer::vaapiLogError(void*, const char *message)
+{
+    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "VAAPI: %s", message);
+}
+
+void VAAPIRenderer::vaapiLogInfo(void*, const char *message)
+{
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "VAAPI: %s", message);
+}
+
 bool
 VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
 {
@@ -38,31 +52,53 @@ VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
     SDL_VERSION(&info.version);
 
     if (!SDL_GetWindowWMInfo(window, &info)) {
-        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                    "SDL_GetWindowWMInfo() failed: %s",
+                     "SDL_GetWindowWMInfo() failed: %s",
-                    SDL_GetError());
+                     SDL_GetError());
         return false;
     }
 
-    SDL_assert(info.subsystem == SDL_SYSWM_X11);
+    m_HwContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
+    if (!m_HwContext) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                    "Failed to allocate VAAPI context");
+        return false;
+    }
 
+    AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
+    AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
+
+    m_WindowSystem = info.subsystem;
     if (info.subsystem == SDL_SYSWM_X11) {
+#ifdef HAVE_LIBVA_X11
         m_XWindow = info.info.x11.window;
-
+        vaDeviceContext->display = vaGetDisplay(info.info.x11.display);
-        m_X11VaLibHandle = dlopen("libva-x11.so", RTLD_LAZY);
+        if (!vaDeviceContext->display) {
-        if (!m_X11VaLibHandle) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                         "Unable to open X11 display for VAAPI");
-                        "dlopen(libva.so) failed: %s",
-                        dlerror());
             return false;
         }
-
+#else
-        m_vaPutSurface = (vaPutSurface_t)dlsym(m_X11VaLibHandle, "vaPutSurface");
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Moonlight not compiled with VAAPI X11 support!");
+        return false;
+#endif
     }
     else if (info.subsystem == SDL_SYSWM_WAYLAND) {
+#ifdef HAVE_LIBVA_WAYLAND
+        m_WaylandSurface = info.info.wl.surface;
+        m_WaylandDisplay = info.info.wl.display;
+        vaDeviceContext->display = vaGetDisplayWl(info.info.wl.display);
+        if (!vaDeviceContext->display) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "Unable to open Wayland display for VAAPI");
+            return false;
+        }
+#else
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "VAAPI backend does not currently support Wayland");
+                     "Moonlight not compiled with VAAPI Wayland support!");
         return false;
+#endif
     }
     else {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -71,18 +107,34 @@ VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
         return false;
     }
 
-    err = av_hwdevice_ctx_create(&m_HwContext,
+    vaSetErrorCallback(vaDeviceContext->display, &VAAPIRenderer::vaapiLogError, nullptr);
-                                 AV_HWDEVICE_TYPE_VAAPI,
+    vaSetInfoCallback(vaDeviceContext->display, &VAAPIRenderer::vaapiLogInfo, nullptr);
-                                 nullptr, nullptr, 0);
+
+    int major, minor;
+    VAStatus status;
+    status = vaInitialize(vaDeviceContext->display, &major, &minor);
+    if (status != VA_STATUS_SUCCESS) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to initialize VAAPI: %d",
+                     status);
+        return false;
+    }
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Initialized VAAPI %d.%d",
+                major, minor);
+
+    // This will populate the driver_quirks
+    err = av_hwdevice_ctx_init(m_HwContext);
     if (err < 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "Failed to create VAAPI context: %d",
+                     "Failed to initialize VAAPI context: %d",
                      err);
         return false;
     }
 
     // This quirk is set for the VDPAU wrapper which doesn't work with our VAAPI renderer
-    if (((AVVAAPIDeviceContext*)((AVHWDeviceContext*)(m_HwContext->data))->hwctx)->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES) {
+    if (vaDeviceContext->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES) {
         // Fail and let our VDPAU renderer pick this up
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                     "Avoiding VDPAU wrapper for VAAPI decoding");
@@ -98,7 +150,8 @@ VAAPIRenderer::prepareDecoderContext(AVCodecContext* context)
     context->hw_device_ctx = av_buffer_ref(m_HwContext);
 
     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                "Using VAAPI accelerated renderer");
+                "Using VAAPI accelerated renderer on %s",
+                m_WindowSystem == SDL_SYSWM_X11 ? "X11" : "Wayland");
 
     return true;
 }
@@ -120,14 +173,45 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
 
     StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
 
-    m_vaPutSurface(vaDeviceContext->display,
+    if (m_WindowSystem == SDL_SYSWM_X11) {
-                   surface,
+#ifdef HAVE_LIBVA_X11
-                   m_XWindow,
+        vaPutSurface(vaDeviceContext->display,
-                   0, 0,
+                     surface,
-                   m_VideoWidth, m_VideoHeight,
+                     m_XWindow,
-                   dst.x, dst.y,
+                     0, 0,
-                   dst.w, dst.h,
+                     m_VideoWidth, m_VideoHeight,
-                   NULL, 0, 0);
+                     dst.x, dst.y,
+                     dst.w, dst.h,
+                     NULL, 0, 0);
+#endif
+    }
+    else if (m_WindowSystem == SDL_SYSWM_WAYLAND) {
+#ifdef HAVE_LIBVA_WAYLAND
+        struct wl_buffer* buffer;
+        VAStatus status;
+
+        status = vaGetSurfaceBufferWl(vaDeviceContext->display,
+                                      surface,
+                                      VA_FRAME_PICTURE,
+                                      &buffer);
+        if (status == VA_STATUS_SUCCESS) {
+            wl_surface_attach(m_WaylandSurface, buffer, 0, 0);
+            wl_surface_damage(m_WaylandSurface, dst.x, dst.y, dst.w, dst.h);
+
+            wl_display_flush(m_WaylandDisplay);
+            wl_surface_commit(m_WaylandSurface);
+        }
+        else {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "vaGetSurfaceBufferWl failed(): %d",
+                         status);
+        }
+#endif
+    }
+    else {
+        // We don't accept anything else in initialize().
+        SDL_assert(false);
+    }
 
     av_frame_free(&frame);
 }

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -4,29 +4,17 @@
 
 extern "C" {
 #include <va/va.h>
+#ifdef HAVE_LIBVA_X11
 #include <va/va_x11.h>
+#endif
+#ifdef HAVE_LIBVA_WAYLAND
+#include <va/va_wayland.h>
+#endif
 #include <libavutil/hwcontext_vaapi.h>
 }
 
 class VAAPIRenderer : public IFFmpegRenderer
 {
-    typedef VAStatus (*vaPutSurface_t)(
-        VADisplay dpy,
-        VASurfaceID surface,
-        Drawable draw, /* X Drawable */
-        short srcx,
-        short srcy,
-        unsigned short srcw,
-        unsigned short srch,
-        short destx,
-        short desty,
-        unsigned short destw,
-        unsigned short desth,
-        VARectangle *cliprects,
-        unsigned int number_cliprects,
-        unsigned int flags
-    );
-
 public:
     VAAPIRenderer();
     virtual ~VAAPIRenderer();
@@ -38,10 +26,21 @@ public:
     virtual void renderFrame(AVFrame* frame);
 
 private:
-    Window m_XWindow;
+    static void vaapiLogError(void*, const char *message);
+    static void vaapiLogInfo(void*, const char *message);
+
+    int m_WindowSystem;
     AVBufferRef* m_HwContext;
-    void* m_X11VaLibHandle;
+
-    vaPutSurface_t m_vaPutSurface;
+#ifdef HAVE_LIBVA_X11
+    Window m_XWindow;
+#endif
+
+#ifdef HAVE_LIBVA_WAYLAND
+    struct wl_surface* m_WaylandSurface;
+    struct wl_display* m_WaylandDisplay;
+#endif
+
     int m_VideoWidth;
     int m_VideoHeight;
     int m_DisplayWidth;

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -97,9 +97,9 @@ bool VDPAURenderer::initialize(SDL_Window* window, int, int width, int height)
     SDL_VERSION(&info.version);
 
     if (!SDL_GetWindowWMInfo(window, &info)) {
-        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                    "SDL_GetWindowWMInfo() failed: %s",
+                     "SDL_GetWindowWMInfo() failed: %s",
-                    SDL_GetError());
+                     SDL_GetError());
         return false;
     }