From f929cffce7743cf70fb895e002d6e17315d863c2 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Wed, 15 Aug 2018 21:10:35 -0700
Subject: [PATCH] Let Pacer know what the display Hz and stream FPS are so it
 can make better drop decisions

---
 .../video/ffmpeg-renderers/pacer.cpp          | 65 +++++++++++++++----
 app/streaming/video/ffmpeg-renderers/pacer.h  |  5 ++
 app/streaming/video/ffmpeg-renderers/vt.mm    |  4 +-
 3 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/app/streaming/video/ffmpeg-renderers/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer.cpp
index 1c09a930..607bf41c 100644
--- a/app/streaming/video/ffmpeg-renderers/pacer.cpp
+++ b/app/streaming/video/ffmpeg-renderers/pacer.cpp
@@ -3,7 +3,9 @@
 #define FRAME_HISTORY_ENTRIES 8
 
 Pacer::Pacer() :
-    m_FrameQueueLock(0)
+    m_FrameQueueLock(0),
+    m_MaxVideoFps(0),
+    m_DisplayFps(0)
 {
 
 }
@@ -13,28 +15,60 @@ Pacer::~Pacer()
     drain();
 }
 
+void Pacer::initialize(SDL_Window* window, int maxVideoFps)
+{
+    m_MaxVideoFps = maxVideoFps;
+
+    int displayIndex = SDL_GetWindowDisplayIndex(window);
+    if (displayIndex < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to get current display: %s",
+                     SDL_GetError());
+
+        // Assume display 0 if it fails
+        displayIndex = 0;
+    }
+
+    SDL_DisplayMode mode;
+    if (SDL_GetCurrentDisplayMode(displayIndex, &mode) == 0) {
+        // May be zero if undefined
+        m_DisplayFps = mode.refresh_rate;
+    }
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Frame pacing: target %d Hz with %d FPS stream",
+                m_DisplayFps, m_MaxVideoFps);
+}
+
 AVFrame* Pacer::getFrameAtVsync()
 {
-    SDL_AtomicLock(&m_FrameQueueLock);
+    // Make sure initialize() has been called
+    SDL_assert(m_MaxVideoFps != 0);
 
-    int frameDropTarget;
+    SDL_AtomicLock(&m_FrameQueueLock);
 
     // If the queue length history entries are large, be strict
     // about dropping excess frames.
-    frameDropTarget = 1;
-    for (int i = 0; i < m_FrameQueueHistory.count(); i++) {
-        if (m_FrameQueueHistory[i] <= 1) {
-            // Be lenient as long as the queue length
-            // resolves before the end of frame history
-            frameDropTarget = 3;
+    int frameDropTarget = 1;
+
+    // If we may get more frames per second than we can display, use
+    // frame history to drop frames only if consistently above the
+    // one queued frame mark.
+    if (m_MaxVideoFps >= m_DisplayFps) {
+        for (int i = 0; i < m_FrameQueueHistory.count(); i++) {
+            if (m_FrameQueueHistory[i] <= 1) {
+                // Be lenient as long as the queue length
+                // resolves before the end of frame history
+                frameDropTarget = 3;
+            }
         }
-    }
 
-    if (m_FrameQueueHistory.count() == FRAME_HISTORY_ENTRIES) {
-        m_FrameQueueHistory.dequeue();
-    }
+        if (m_FrameQueueHistory.count() == FRAME_HISTORY_ENTRIES) {
+            m_FrameQueueHistory.dequeue();
+        }
 
-    m_FrameQueueHistory.enqueue(m_FrameQueue.count());
+        m_FrameQueueHistory.enqueue(m_FrameQueue.count());
+    }
 
     // Catch up if we're several frames ahead
     while (m_FrameQueue.count() > frameDropTarget) {
@@ -56,6 +90,9 @@ AVFrame* Pacer::getFrameAtVsync()
 
 void Pacer::submitFrame(AVFrame* frame)
 {
+    // Make sure initialize() has been called
+    SDL_assert(m_MaxVideoFps != 0);
+
     SDL_AtomicLock(&m_FrameQueueLock);
     m_FrameQueue.enqueue(frame);
     SDL_AtomicUnlock(&m_FrameQueueLock);
diff --git a/app/streaming/video/ffmpeg-renderers/pacer.h b/app/streaming/video/ffmpeg-renderers/pacer.h
index 9afe7ef3..1578198f 100644
--- a/app/streaming/video/ffmpeg-renderers/pacer.h
+++ b/app/streaming/video/ffmpeg-renderers/pacer.h
@@ -15,10 +15,15 @@ public:
 
     void submitFrame(AVFrame* frame);
 
+    void initialize(SDL_Window* window, int maxVideoFps);
+
     void drain();
 
 private:
     QQueue<AVFrame*> m_FrameQueue;
     QQueue<int> m_FrameQueueHistory;
     SDL_SpinLock m_FrameQueueLock;
+
+    int m_MaxVideoFps;
+    int m_DisplayFps;
 };
diff --git a/app/streaming/video/ffmpeg-renderers/vt.mm b/app/streaming/video/ffmpeg-renderers/vt.mm
index b3a07305..68483a3c 100644
--- a/app/streaming/video/ffmpeg-renderers/vt.mm
+++ b/app/streaming/video/ffmpeg-renderers/vt.mm
@@ -131,10 +131,12 @@ public:
                             int videoFormat,
                             int,
                             int,
-                            int) override
+                            int maxFps) override
     {
         int err;
 
+        m_Pacer.initialize(window, maxFps);
+
         if (videoFormat & VIDEO_FORMAT_MASK_H264) {
             // Prior to 10.13, we'll just assume everything has
             // H.264 support and fail open to allow VT decode.