Allow Pacer to wait for frames up until a few ms before v-sync for better smoothness and lower latency

app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.cpp

@@ -34,13 +34,15 @@ DisplayLinkVsyncSource::displayLinkOutputCallback(
     // interval, even if many ms prior, we can safely use the current host time
     // and get a consistent callback for each v-sync. This reduces video latency
     // by at least 1 frame vs. rendering with the actual vsyncTime.
-    me->m_Pacer->vsyncCallback();
+    me->m_Pacer->vsyncCallback(500 / m_DisplayFps);
 
     return kCVReturnSuccess;
 }
 
-bool DisplayLinkVsyncSource::initialize(SDL_Window*)
+bool DisplayLinkVsyncSource::initialize(SDL_Window*, int displayFps)
 {
+    m_DisplayFps = displayFps;
+
     CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink);
     CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, this);
     CVDisplayLinkStart(m_DisplayLink);

app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h

@@ -11,7 +11,7 @@ public:
 
     virtual ~DisplayLinkVsyncSource();
 
-    virtual bool initialize(SDL_Window* window);
+    virtual bool initialize(SDL_Window* window, int displayFps);
 
 private:
     static
@@ -26,5 +26,6 @@ private:
 
     Pacer* m_Pacer;
     CVDisplayLinkRef m_DisplayLink;
+    int m_DisplayFps;
 };
 

app/streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.cpp

@@ -24,8 +24,10 @@ DxVsyncSource::~DxVsyncSource()
     }
 }
 
-bool DxVsyncSource::initialize(SDL_Window* window)
+bool DxVsyncSource::initialize(SDL_Window* window, int displayFps)
 {
+    m_DisplayFps = displayFps;
+
     m_Gdi32Handle = LoadLibraryA("gdi32.dll");
     if (m_Gdi32Handle == nullptr) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -144,8 +146,6 @@ int DxVsyncSource::vsyncThread(void* context)
         waitForVblankEventParams.hDevice = 0;
         waitForVblankEventParams.VidPnSourceId = openAdapterParams.VidPnSourceId;
 
-        me->m_Pacer->vsyncCallback();
-
         status = me->m_D3DKMTWaitForVerticalBlankEvent(&waitForVblankEventParams);
         if (status != STATUS_SUCCESS) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -154,6 +154,8 @@ int DxVsyncSource::vsyncThread(void* context)
             SDL_Delay(10);
             continue;
         }
+
+        me->m_Pacer->vsyncCallback(1000 / me->m_DisplayFps);
     }
 
     if (openAdapterParams.hAdapter != 0) {

app/streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.h

@@ -35,7 +35,7 @@ public:
 
     virtual ~DxVsyncSource();
 
-    virtual bool initialize(SDL_Window* window);
+    virtual bool initialize(SDL_Window* window, int displayFps);
 
 private:
     static int vsyncThread(void* context);
@@ -45,6 +45,7 @@ private:
     SDL_atomic_t m_Stopping;
     HMODULE m_Gdi32Handle;
     HWND m_Window;
+    int m_DisplayFps;
 
     PFND3DKMTOPENADAPTERFROMHDC m_D3DKMTOpenAdapterFromHdc;
     PFND3DKMTCLOSEADAPTER m_D3DKMTCloseAdapter;

app/streaming/video/ffmpeg-renderers/pacer/nullthreadedvsyncsource.cpp

@@ -15,8 +15,9 @@ NullThreadedVsyncSource::~NullThreadedVsyncSource()
     }
 }
 
-bool NullThreadedVsyncSource::initialize(SDL_Window*)
+bool NullThreadedVsyncSource::initialize(SDL_Window*, int displayFps)
 {
+    m_DisplayFps = displayFps;
     m_Thread = SDL_CreateThread(vsyncThread, "Null Vsync Thread", this);
     if (m_Thread == nullptr) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -35,7 +36,7 @@ int NullThreadedVsyncSource::vsyncThread(void* context)
     SDL_SetThreadPriority(SDL_THREAD_PRIORITY_HIGH);
 
     while (SDL_AtomicGet(&me->m_Stopping) == 0) {
-        me->m_Pacer->vsyncCallback();
+        me->m_Pacer->vsyncCallback(1000 / me->m_DisplayFps);
     }
 
     return 0;

app/streaming/video/ffmpeg-renderers/pacer/nullthreadedvsyncsource.h

@@ -9,7 +9,7 @@ public:
 
     virtual ~NullThreadedVsyncSource();
 
-    virtual bool initialize(SDL_Window* window);
+    virtual bool initialize(SDL_Window* window, int displayFps);
 
 private:
     static int vsyncThread(void* context);
@@ -17,4 +17,5 @@ private:
     Pacer* m_Pacer;
     SDL_Thread* m_Thread;
     SDL_atomic_t m_Stopping;
+    int m_DisplayFps;
 };

app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp

@@ -1,5 +1,7 @@
 #include "pacer.h"
 
+#include "nullthreadedvsyncsource.h"
+
 #ifdef Q_OS_DARWIN
 #include "displaylinkvsyncsource.h"
 #endif
@@ -10,6 +12,13 @@
 
 #define FRAME_HISTORY_ENTRIES 8
 
+// We may be woken up slightly late so don't go all the way
+// up to the next V-sync since we may accidentally step into
+// the next V-sync period. It also takes some amount of time
+// to do the render itself, so we can't render right before
+// V-sync happens.
+#define TIMER_SLACK_MS 3
+
 Pacer::Pacer(IFFmpegRenderer* renderer) :
     m_FrameQueueLock(0),
     m_VsyncSource(nullptr),
@@ -34,11 +43,15 @@ Pacer::~Pacer()
 
 // Called in an arbitrary thread by the IVsyncSource on V-sync
 // or an event synchronized with V-sync
-void Pacer::vsyncCallback()
+void Pacer::vsyncCallback(int timeUntilNextVsyncMillis)
 {
     // Make sure initialize() has been called
     SDL_assert(m_MaxVideoFps != 0);
 
+    SDL_assert(timeUntilNextVsyncMillis >= TIMER_SLACK_MS);
+
+    Uint32 vsyncCallbackStartTime = SDL_GetTicks();
+
     SDL_AtomicLock(&m_FrameQueueLock);
 
     // If the queue length history entries are large, be strict
@@ -71,13 +84,27 @@ void Pacer::vsyncCallback()
         av_frame_free(&frame);
     }
 
+
     if (m_FrameQueue.isEmpty()) {
         SDL_AtomicUnlock(&m_FrameQueueLock);
 
+        while (!SDL_TICKS_PASSED(SDL_GetTicks(),
+                                 vsyncCallbackStartTime + timeUntilNextVsyncMillis - TIMER_SLACK_MS)) {
+            SDL_Delay(1);
+
+            SDL_AtomicLock(&m_FrameQueueLock);
+            if (!m_FrameQueue.isEmpty()) {
+                // Don't release the lock
+                goto RenderNextFrame;
+            }
+            SDL_AtomicUnlock(&m_FrameQueueLock);
+        }
+
         // Nothing to render at this time
         return;
     }
 
+RenderNextFrame:
     // Grab the first frame
     AVFrame* frame = m_FrameQueue.dequeue();
     SDL_AtomicUnlock(&m_FrameQueueLock);
@@ -122,7 +149,7 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps)
     // immediately like they used to.
 #endif
 
-    if (m_VsyncSource != nullptr && !m_VsyncSource->initialize(window)) {
+    if (m_VsyncSource != nullptr && !m_VsyncSource->initialize(window, m_DisplayFps)) {
         return false;
     }
 

app/streaming/video/ffmpeg-renderers/pacer/pacer.h

@@ -7,7 +7,7 @@
 class IVsyncSource {
 public:
     virtual ~IVsyncSource() {}
-    virtual bool initialize(SDL_Window* window) = 0;
+    virtual bool initialize(SDL_Window* window, int displayFps) = 0;
 };
 
 class Pacer
@@ -21,7 +21,7 @@ public:
 
     bool initialize(SDL_Window* window, int maxVideoFps);
 
-    void vsyncCallback();
+    void vsyncCallback(int timeUntilNextVsyncMillis);
 
     bool isUsingFrameQueue();