Allow a renderer to opt-out of the render thread and use that for SDL on OGL

app/streaming/session.cpp

@@ -1019,25 +1019,11 @@ void Session::exec(int displayOriginX, int displayOriginY)
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                         "Quit event received");
             goto DispatchDeferredCleanup;
-        case SDL_USEREVENT: {
-            SDL_Event nextEvent;
 
+        case SDL_USEREVENT:
             SDL_assert(event.user.code == SDL_CODE_FRAME_READY);
-
+            m_VideoDecoder->renderFrameOnMainThread();
-            // Drop any earlier frames
-            while (SDL_PeepEvents(&nextEvent,
-                                  1,
-                                  SDL_GETEVENT,
-                                  SDL_USEREVENT,
-                                  SDL_USEREVENT) == 1) {
-                m_VideoDecoder->dropFrame(&event.user);
-                event = nextEvent;
-            }
-
-            // Render the last frame
-            m_VideoDecoder->renderFrame(&event.user);
             break;
-        }
 
         case SDL_WINDOWEVENT:
             // Capture mouse cursor when user actives the window by clicking on

app/streaming/video/decoder.h

@@ -40,19 +40,5 @@ public:
     virtual bool isHardwareAccelerated() = 0;
     virtual int getDecoderCapabilities() = 0;
     virtual int submitDecodeUnit(PDECODE_UNIT du) = 0;
-    virtual void renderFrame(SDL_UserEvent* event) = 0;
+    virtual void renderFrameOnMainThread() = 0;
-    virtual void dropFrame(SDL_UserEvent* event) = 0;
-
-    virtual void queueFrame(void* data1 = nullptr,
-                            void* data2 = nullptr)
-    {
-        SDL_Event event;
-
-        event.type = SDL_USEREVENT;
-        event.user.code = SDL_CODE_FRAME_READY;
-        event.user.data1 = data1;
-        event.user.data2 = data2;
-
-        SDL_PushEvent(&event);
-    }
 };

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -779,6 +779,12 @@ void DXVA2Renderer::notifyOverlayUpdated(Overlay::OverlayType type)
     }
 }
 
+bool DXVA2Renderer::isRenderThreadSupported()
+{
+    // renderFrame() may be called outside of the main thread
+    return true;
+}
+
 void DXVA2Renderer::renderFrame(AVFrame *frame)
 {
     IDirect3DSurface9* surface = reinterpret_cast<IDirect3DSurface9*>(frame->data[3]);

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -28,6 +28,7 @@ public:
     virtual int getDecoderCapabilities() override;
     virtual FramePacingConstraint getFramePacingConstraint() override;
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
+    virtual bool isRenderThreadSupported() override;
 
 private:
     bool initializeDecoder();

app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp

@@ -61,6 +61,24 @@ Pacer::~Pacer()
     }
 }
 
+void Pacer::renderOnMainThread()
+{
+    // Ignore this call for renderers that work on a dedicated render thread
+    if (m_RenderThread != nullptr) {
+        return;
+    }
+
+    m_FrameQueueLock.lock();
+
+    if (!m_RenderQueue.isEmpty()) {
+        // Releases m_FrameQueueLock
+        renderLastFrameAndUnlock();
+    }
+    else {
+        m_FrameQueueLock.unlock();
+    }
+}
+
 int Pacer::renderThread(void* context)
 {
     Pacer* me = reinterpret_cast<Pacer*>(context);
@@ -87,34 +105,61 @@ int Pacer::renderThread(void* context)
             break;
         }
 
-        // Dequeue the most recent frame for rendering and free the others.
+        // Render the latest frame and discard the others
         // NB: m_FrameQueueLock still held here!
-        AVFrame* lastFrame = nullptr;
+        me->renderLastFrameAndUnlock();
-        while (!me->m_RenderQueue.isEmpty()) {
-            if (lastFrame != nullptr) {
-                // Don't hold the frame queue lock across av_frame_free(),
-                // since it could need to talk to the GPU driver. This is safe
-                // because we're guaranteed that the queue will not shrink during
-                // this time (and so dequeue() below will always get something).
-                me->m_FrameQueueLock.unlock();
-                av_frame_free(&lastFrame);
-                me->m_VideoStats->pacerDroppedFrames++;
-                me->m_FrameQueueLock.lock();
-            }
-
-            lastFrame = me->m_RenderQueue.dequeue();
-        }
-
-        // Release the frame queue lock before rendering
-        me->m_FrameQueueLock.unlock();
-
-        // Render and free the mot current frame
-        me->renderFrame(lastFrame);
     }
 
     return 0;
 }
 
+void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame)
+{
+    dropFrameForEnqueue(m_RenderQueue);
+    m_RenderQueue.enqueue(frame);
+
+    m_FrameQueueLock.unlock();
+
+    if (m_RenderThread != nullptr) {
+        m_RenderQueueNotEmpty.wakeOne();
+    }
+    else {
+        SDL_Event event;
+
+        // For main thread rendering, we'll push an event to trigger a callback
+        event.type = SDL_USEREVENT;
+        event.user.code = SDL_CODE_FRAME_READY;
+        SDL_PushEvent(&event);
+    }
+}
+
+// Caller must hold m_FrameQueueLock
+void Pacer::renderLastFrameAndUnlock()
+{
+    // Dequeue the most recent frame for rendering and free the others.
+    AVFrame* lastFrame = nullptr;
+    while (!m_RenderQueue.isEmpty()) {
+        if (lastFrame != nullptr) {
+            // Don't hold the frame queue lock across av_frame_free(),
+            // since it could need to talk to the GPU driver. This is safe
+            // because we're guaranteed that the queue will not shrink during
+            // this time (and so dequeue() below will always get something).
+            m_FrameQueueLock.unlock();
+            av_frame_free(&lastFrame);
+            m_VideoStats->pacerDroppedFrames++;
+            m_FrameQueueLock.lock();
+        }
+
+        lastFrame = m_RenderQueue.dequeue();
+    }
+
+    // Release the frame queue lock before rendering
+    m_FrameQueueLock.unlock();
+
+    // Render and free the mot current frame
+    renderFrame(lastFrame);
+}
+
 // Called in an arbitrary thread by the IVsyncSource on V-sync
 // or an event synchronized with V-sync
 void Pacer::vsyncCallback(int timeUntilNextVsyncMillis)
@@ -172,10 +217,7 @@ void Pacer::vsyncCallback(int timeUntilNextVsyncMillis)
     }
 
     // Place the first frame on the render queue
-    dropFrameForEnqueue(m_RenderQueue);
+    enqueueFrameForRenderingAndUnlock(m_PacingQueue.dequeue());
-    m_RenderQueue.enqueue(m_PacingQueue.dequeue());
-    m_FrameQueueLock.unlock();
-    m_RenderQueueNotEmpty.wakeOne();
 }
 
 bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing)
@@ -211,7 +253,9 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing)
                     m_DisplayFps, m_MaxVideoFps);
     }
 
-    m_RenderThread = SDL_CreateThread(Pacer::renderThread, "Pacer Render Thread", this);
+    if (m_VsyncRenderer->isRenderThreadSupported()) {
+        m_RenderThread = SDL_CreateThread(Pacer::renderThread, "Pacer Render Thread", this);
+    }
 
     return true;
 }
@@ -283,16 +327,10 @@ void Pacer::submitFrame(AVFrame* frame)
     if (m_VsyncSource != nullptr) {
         dropFrameForEnqueue(m_PacingQueue);
         m_PacingQueue.enqueue(frame);
-    }
+        m_FrameQueueLock.unlock();
-    else {
-        dropFrameForEnqueue(m_RenderQueue);
-        m_RenderQueue.enqueue(frame);
-    }
-    m_FrameQueueLock.unlock();
-    if (m_VsyncSource != nullptr) {
         m_PacingQueueNotEmpty.wakeOne();
     }
     else {
-        m_RenderQueueNotEmpty.wakeOne();
+        enqueueFrameForRenderingAndUnlock(frame);
     }
 }

app/streaming/video/ffmpeg-renderers/pacer/pacer.h

@@ -26,9 +26,15 @@ public:
 
     void vsyncCallback(int timeUntilNextVsyncMillis);
 
+    void renderOnMainThread();
+
 private:
     static int renderThread(void* context);
 
+    void enqueueFrameForRenderingAndUnlock(AVFrame* frame);
+
+    void renderLastFrameAndUnlock();
+
     void renderFrame(AVFrame* frame);
 
     void dropFrameForEnqueue(QQueue<AVFrame*>& queue);

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -27,6 +27,7 @@ public:
     virtual bool needsTestFrame() = 0;
     virtual int getDecoderCapabilities() = 0;
     virtual FramePacingConstraint getFramePacingConstraint() = 0;
+    virtual bool isRenderThreadSupported() = 0;
 
     // IOverlayRenderer
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override {

app/streaming/video/ffmpeg-renderers/sdlvid.cpp

@@ -62,7 +62,7 @@ bool SdlRenderer::prepareDecoderContext(AVCodecContext*)
     /* Nothing to do */
 
     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                "Using SDL software renderer");
+                "Using SDL renderer");
 
     return true;
 }
@@ -126,6 +126,24 @@ void SdlRenderer::notifyOverlayUpdated(Overlay::OverlayType type)
     }
 }
 
+bool SdlRenderer::isRenderThreadSupported()
+{
+    SDL_RendererInfo info;
+    SDL_GetRendererInfo(m_Renderer, &info);
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "SDL renderer backend: %s",
+                info.name);
+
+    if (info.name != QString("direct3d")) {
+        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                    "SDL renderer backend requires main thread rendering");
+        return false;
+    }
+
+    return true;
+}
+
 bool SdlRenderer::initialize(SDL_Window* window,
                              int,
                              int width,

app/streaming/video/ffmpeg-renderers/sdlvid.h

@@ -20,6 +20,7 @@ public:
     virtual int getDecoderCapabilities() override;
     virtual FramePacingConstraint getFramePacingConstraint() override;
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
+    virtual bool isRenderThreadSupported() override;
 
 private:
     void renderOverlay(Overlay::OverlayType type);

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -182,6 +182,12 @@ IFFmpegRenderer::FramePacingConstraint VAAPIRenderer::getFramePacingConstraint()
     return PACING_ANY;
 }
 
+bool VAAPIRenderer::isRenderThreadSupported()
+{
+    // renderFrame() may be called outside of the main thread
+    return true;
+}
+
 void
 VAAPIRenderer::renderFrame(AVFrame* frame)
 {

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -41,6 +41,7 @@ public:
     virtual bool needsTestFrame();
     virtual int getDecoderCapabilities();
     virtual FramePacingConstraint getFramePacingConstraint();
+    virtual bool isRenderThreadSupported();
 
 private:
     int m_WindowSystem;

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -251,6 +251,12 @@ IFFmpegRenderer::FramePacingConstraint VDPAURenderer::getFramePacingConstraint()
     return PACING_ANY;
 }
 
+bool VDPAURenderer::isRenderThreadSupported()
+{
+    // renderFrame() may be called outside of the main thread
+    return true;
+}
+
 void VDPAURenderer::renderFrame(AVFrame* frame)
 {
     VdpStatus status;

app/streaming/video/ffmpeg-renderers/vdpau.h

@@ -24,6 +24,7 @@ public:
     virtual bool needsTestFrame();
     virtual int getDecoderCapabilities();
     virtual FramePacingConstraint getFramePacingConstraint();
+    virtual bool isRenderThreadSupported();
 
 private:
     uint32_t m_VideoWidth, m_VideoHeight;

app/streaming/video/ffmpeg-renderers/vt.mm

@@ -296,6 +296,12 @@ public:
         return PACING_FORCE_ON;
     }
 
+    virtual bool isRenderThreadSupported() override
+    {
+        // renderFrame() may be called outside of the main thread
+        return true;
+    }
+
 private:
     AVBufferRef* m_HwContext;
     AVSampleBufferDisplayLayer* m_DisplayLayer;

app/streaming/video/ffmpeg.cpp

@@ -630,14 +630,8 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
     return DR_OK;
 }
 
-// Called on main thread
+void FFmpegVideoDecoder::renderFrameOnMainThread()
-void FFmpegVideoDecoder::renderFrame(SDL_UserEvent*)
 {
-    SDL_assert(false);
+    m_Pacer->renderOnMainThread();
 }
 
-// Called on main thread
-void FFmpegVideoDecoder::dropFrame(SDL_UserEvent*)
-{
-    SDL_assert(false);
-}

app/streaming/video/ffmpeg.h

@@ -23,8 +23,7 @@ public:
     virtual bool isHardwareAccelerated() override;
     virtual int getDecoderCapabilities() override;
     virtual int submitDecodeUnit(PDECODE_UNIT du) override;
-    virtual void renderFrame(SDL_UserEvent* event) override;
+    virtual void renderFrameOnMainThread() override;
-    virtual void dropFrame(SDL_UserEvent* event) override;
 
     virtual IFFmpegRenderer* getRenderer();
 

app/streaming/video/slvid.h

@@ -22,8 +22,7 @@ public:
     virtual int submitDecodeUnit(PDECODE_UNIT du);
 
     // Unused since rendering is done directly from the decode thread
-    virtual void renderFrame(SDL_UserEvent*) {}
+    virtual void renderFrameOnMainThread() {}
-    virtual void dropFrame(SDL_UserEvent*) {}
 
 private:
     CSLVideoContext* m_VideoContext;