Move DisplayLinkVsyncSource back into VTRenderer to reduce latency

app/app.pro

@@ -261,12 +261,10 @@ macx {
     message(VideoToolbox renderer selected)
 
     SOURCES += \
-        streaming/video/ffmpeg-renderers/vt.mm \
+        streaming/video/ffmpeg-renderers/vt.mm
-        streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm
 
     HEADERS += \
-        streaming/video/ffmpeg-renderers/vt.h \
+        streaming/video/ffmpeg-renderers/vt.h
-        streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h
 }
 soundio {
     message(libsoundio audio renderer selected)

app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include "pacer.h"
-
-class DisplayLinkVsyncSourceFactory
-{
-public:
-    static
-    IVsyncSource* createVsyncSource(Pacer* pacer);
-};
-

app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm

@@ -1,132 +0,0 @@
-#include "displaylinkvsyncsource.h"
-
-#include <SDL_syswm.h>
-
-#include <CoreVideo/CoreVideo.h>
-
-#import <Cocoa/Cocoa.h>
-
-class DisplayLinkVsyncSource : public IVsyncSource
-{
-public:
-    DisplayLinkVsyncSource(Pacer* pacer)
-        : m_Pacer(pacer),
-          m_DisplayLink(nullptr)
-    {
-
-    }
-
-    virtual ~DisplayLinkVsyncSource() override
-    {
-        if (m_DisplayLink != nullptr) {
-            CVDisplayLinkStop(m_DisplayLink);
-            CVDisplayLinkRelease(m_DisplayLink);
-        }
-    }
-
-    static
-    CGDirectDisplayID
-    getDisplayID(NSScreen* screen)
-    {
-        NSNumber* screenNumber = [screen deviceDescription][@"NSScreenNumber"];
-        return [screenNumber unsignedIntValue];
-    }
-
-    static
-    CVReturn
-    displayLinkOutputCallback(
-        CVDisplayLinkRef displayLink,
-        const CVTimeStamp* /* now */,
-        const CVTimeStamp* /* vsyncTime */,
-        CVOptionFlags,
-        CVOptionFlags*,
-        void *displayLinkContext)
-    {
-        auto me = reinterpret_cast<DisplayLinkVsyncSource*>(displayLinkContext);
-
-        SDL_assert(displayLink == me->m_DisplayLink);
-
-        // In my testing on macOS 10.13, this callback is invoked about 24 ms
-        // prior to the specified v-sync time (now - vsyncTime). Since this is
-        // greater than the standard v-sync interval (16 ms = 60 FPS), we will
-        // draw using the current host time, rather than the actual v-sync target
-        // time. Because the CVDisplayLink is in sync with the actual v-sync
-        // interval, even if many ms prior, we can safely use the current host time
-        // and get a consistent callback for each v-sync. This reduces video latency
-        // by at least 1 frame vs. rendering with the actual vsyncTime.
-        me->m_Pacer->vsyncCallback(500 / me->m_DisplayFps);
-
-        return kCVReturnSuccess;
-    }
-
-    virtual bool initialize(SDL_Window* window, int displayFps) override
-    {
-        SDL_SysWMinfo info;
-
-        SDL_VERSION(&info.version);
-
-        if (!SDL_GetWindowWMInfo(window, &info)) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "SDL_GetWindowWMInfo() failed: %s",
-                         SDL_GetError());
-            return false;
-        }
-
-        SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
-
-        m_DisplayFps = displayFps;
-
-        NSScreen* screen = [info.info.cocoa.window screen];
-        CVReturn status;
-        if (screen == nullptr) {
-            // Window not visible on any display, so use a
-            // CVDisplayLink that can work with all active displays.
-            // When we become visible, we'll recreate ourselves
-            // and associate with the new screen.
-            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
-                        "NSWindow is not visible on any display");
-            status = CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink);
-        }
-        else {
-            CGDirectDisplayID displayId;
-            displayId = getDisplayID(screen);
-            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                        "NSWindow on display: %x",
-                        displayId);
-            status = CVDisplayLinkCreateWithCGDisplay(displayId, &m_DisplayLink);
-        }
-        if (status != kCVReturnSuccess) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "Failed to create CVDisplayLink: %d",
-                         status);
-            return false;
-        }
-
-        status = CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, this);
-        if (status != kCVReturnSuccess) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "CVDisplayLinkSetOutputCallback() failed: %d",
-                         status);
-            return false;
-        }
-
-        status = CVDisplayLinkStart(m_DisplayLink);
-        if (status != kCVReturnSuccess) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "CVDisplayLinkStart() failed: %d",
-                         status);
-            return false;
-        }
-
-        return true;
-    }
-
-private:
-    Pacer* m_Pacer;
-    CVDisplayLinkRef m_DisplayLink;
-    int m_DisplayFps;
-};
-
-IVsyncSource* DisplayLinkVsyncSourceFactory::createVsyncSource(Pacer* pacer) {
-    return new DisplayLinkVsyncSource(pacer);
-}

app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp

@@ -3,10 +3,6 @@
 
 #include "nullthreadedvsyncsource.h"
 
-#ifdef Q_OS_DARWIN
-#include "displaylinkvsyncsource.h"
-#endif
-
 #ifdef Q_OS_WIN32
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
@@ -230,9 +226,7 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing)
                     "Frame pacing active: target %d Hz with %d FPS stream",
                     m_DisplayFps, m_MaxVideoFps);
 
-    #if defined(Q_OS_DARWIN)
+    #if defined(Q_OS_WIN32)
-        m_VsyncSource = DisplayLinkVsyncSourceFactory::createVsyncSource(this);
-    #elif defined(Q_OS_WIN32)
         // Don't use D3DKMTWaitForVerticalBlankEvent() on Windows 7, because
         // it blocks during other concurrent DX operations (like actually rendering).
         if (IsWindows8OrGreater()) {

app/streaming/video/ffmpeg-renderers/vt.mm

@@ -22,13 +22,24 @@ public:
         : m_HwContext(nullptr),
           m_DisplayLayer(nullptr),
           m_FormatDesc(nullptr),
-          m_StreamView(nullptr)
+          m_StreamView(nullptr),
+          m_DisplayLink(nullptr)
     {
         SDL_zero(m_OverlayTextFields);
     }
 
     virtual ~VTRenderer() override
     {
+        if (m_DisplayLink != nullptr) {
+            // Wake up the renderer in case it is waiting for v-sync
+            SDL_LockMutex(m_VsyncMutex);
+            SDL_CondSignal(m_VsyncPassed);
+            SDL_UnlockMutex(m_VsyncMutex);
+
+            CVDisplayLinkStop(m_DisplayLink);
+            CVDisplayLinkRelease(m_DisplayLink);
+        }
+
         if (m_HwContext != nullptr) {
             av_buffer_unref(&m_HwContext);
         }
@@ -48,6 +59,85 @@ public:
         }
     }
 
+    static
+    CGDirectDisplayID
+    getDisplayID(NSScreen* screen)
+    {
+        NSNumber* screenNumber = [screen deviceDescription][@"NSScreenNumber"];
+        return [screenNumber unsignedIntValue];
+    }
+
+    static
+    CVReturn
+    displayLinkOutputCallback(
+        CVDisplayLinkRef displayLink,
+        const CVTimeStamp* /* now */,
+        const CVTimeStamp* /* vsyncTime */,
+        CVOptionFlags,
+        CVOptionFlags*,
+        void *displayLinkContext)
+    {
+        auto me = reinterpret_cast<VTRenderer*>(displayLinkContext);
+
+        SDL_assert(displayLink == me->m_DisplayLink);
+
+        SDL_LockMutex(me->m_VsyncMutex);
+        SDL_CondSignal(me->m_VsyncPassed);
+        SDL_UnlockMutex(me->m_VsyncMutex);
+
+        return kCVReturnSuccess;
+    }
+
+    bool initializeVsyncCallback(SDL_SysWMinfo* info)
+    {
+        NSScreen* screen = [info->info.cocoa.window screen];
+        CVReturn status;
+        if (screen == nullptr) {
+            // Window not visible on any display, so use a
+            // CVDisplayLink that can work with all active displays.
+            // When we become visible, we'll recreate ourselves
+            // and associate with the new screen.
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "NSWindow is not visible on any display");
+            status = CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink);
+        }
+        else {
+            CGDirectDisplayID displayId;
+            displayId = getDisplayID(screen);
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "NSWindow on display: %x",
+                        displayId);
+            status = CVDisplayLinkCreateWithCGDisplay(displayId, &m_DisplayLink);
+        }
+        if (status != kCVReturnSuccess) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "Failed to create CVDisplayLink: %d",
+                         status);
+            return false;
+        }
+
+        status = CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, this);
+        if (status != kCVReturnSuccess) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "CVDisplayLinkSetOutputCallback() failed: %d",
+                         status);
+            return false;
+        }
+
+        status = CVDisplayLinkStart(m_DisplayLink);
+        if (status != kCVReturnSuccess) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "CVDisplayLinkStart() failed: %d",
+                         status);
+            return false;
+        }
+
+        m_VsyncMutex = SDL_CreateMutex();
+        m_VsyncPassed = SDL_CreateCond();
+
+        return true;
+    }
+
     // Caller frees frame after we return
     virtual void renderFrame(AVFrame* frame) override
     {
@@ -104,6 +194,13 @@ public:
         [m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
 
         CFRelease(sampleBuffer);
+
+        if (m_DisplayLink != nullptr) {
+            // Vsync is enabled, so wait for a swap before returning
+            SDL_LockMutex(m_VsyncMutex);
+            SDL_CondWait(m_VsyncPassed, m_VsyncMutex);
+            SDL_UnlockMutex(m_VsyncMutex);
+        }
     }
 
     virtual bool initialize(PDECODER_PARAMETERS params) override
@@ -191,6 +288,12 @@ public:
             return false;
         }
 
+        if (params->enableVsync) {
+            if (!initializeVsyncCallback(&info)) {
+                return false;
+            }
+        }
+
         return true;
     }
 
@@ -278,20 +381,15 @@ public:
         return true;
     }
 
-    virtual IFFmpegRenderer::FramePacingConstraint getFramePacingConstraint() override
-    {
-        // This renderer is inherently tied to V-sync due how we're
-        // rendering with AVSampleBufferDisplay layer. Running without
-        // the V-Sync source leads to massive stuttering.
-        return PACING_FORCE_ON;
-    }
-
 private:
     AVBufferRef* m_HwContext;
     AVSampleBufferDisplayLayer* m_DisplayLayer;
     CMVideoFormatDescriptionRef m_FormatDesc;
     NSView* m_StreamView;
     NSTextField* m_OverlayTextFields[Overlay::OverlayMax];
+    CVDisplayLinkRef m_DisplayLink;
+    SDL_mutex* m_VsyncMutex;
+    SDL_cond* m_VsyncPassed;
 };
 
 IFFmpegRenderer* VTRendererFactory::createRenderer() {