From c2b12868bbe62d724011987b126eadbb7aa55f91 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sat, 11 May 2019 18:33:12 -0700 Subject: [PATCH] Move DisplayLinkVsyncSource back into VTRenderer to reduce latency --- app/app.pro | 6 +- .../pacer/displaylinkvsyncsource.h | 11 -- .../pacer/displaylinkvsyncsource.mm | 132 ------------------ .../video/ffmpeg-renderers/pacer/pacer.cpp | 8 +- app/streaming/video/ffmpeg-renderers/vt.mm | 116 +++++++++++++-- 5 files changed, 110 insertions(+), 163 deletions(-) delete mode 100644 app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h delete mode 100644 app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm diff --git a/app/app.pro b/app/app.pro index 5671bcad..09b2128a 100644 --- a/app/app.pro +++ b/app/app.pro @@ -261,12 +261,10 @@ macx { message(VideoToolbox renderer selected) SOURCES += \ - streaming/video/ffmpeg-renderers/vt.mm \ - streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm + streaming/video/ffmpeg-renderers/vt.mm HEADERS += \ - streaming/video/ffmpeg-renderers/vt.h \ - streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h + streaming/video/ffmpeg-renderers/vt.h } soundio { message(libsoundio audio renderer selected) diff --git a/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h b/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h deleted file mode 100644 index b82d3bc6..00000000 --- a/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "pacer.h" - -class DisplayLinkVsyncSourceFactory -{ -public: - static - IVsyncSource* createVsyncSource(Pacer* pacer); -}; - diff --git a/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm b/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm deleted file mode 100644 index 234d685d..00000000 --- a/app/streaming/video/ffmpeg-renderers/pacer/displaylinkvsyncsource.mm +++ /dev/null @@ -1,132 +0,0 @@ -#include "displaylinkvsyncsource.h" - -#include - -#include - -#import - -class DisplayLinkVsyncSource : public IVsyncSource -{ -public: - DisplayLinkVsyncSource(Pacer* pacer) - : m_Pacer(pacer), - m_DisplayLink(nullptr) - { - - } - - virtual ~DisplayLinkVsyncSource() override - { - if (m_DisplayLink != nullptr) { - CVDisplayLinkStop(m_DisplayLink); - CVDisplayLinkRelease(m_DisplayLink); - } - } - - static - CGDirectDisplayID - getDisplayID(NSScreen* screen) - { - NSNumber* screenNumber = [screen deviceDescription][@"NSScreenNumber"]; - return [screenNumber unsignedIntValue]; - } - - static - CVReturn - displayLinkOutputCallback( - CVDisplayLinkRef displayLink, - const CVTimeStamp* /* now */, - const CVTimeStamp* /* vsyncTime */, - CVOptionFlags, - CVOptionFlags*, - void *displayLinkContext) - { - auto me = reinterpret_cast(displayLinkContext); - - SDL_assert(displayLink == me->m_DisplayLink); - - // In my testing on macOS 10.13, this callback is invoked about 24 ms - // prior to the specified v-sync time (now - vsyncTime). Since this is - // greater than the standard v-sync interval (16 ms = 60 FPS), we will - // draw using the current host time, rather than the actual v-sync target - // time. Because the CVDisplayLink is in sync with the actual v-sync - // interval, even if many ms prior, we can safely use the current host time - // and get a consistent callback for each v-sync. This reduces video latency - // by at least 1 frame vs. rendering with the actual vsyncTime. - me->m_Pacer->vsyncCallback(500 / me->m_DisplayFps); - - return kCVReturnSuccess; - } - - virtual bool initialize(SDL_Window* window, int displayFps) override - { - SDL_SysWMinfo info; - - SDL_VERSION(&info.version); - - if (!SDL_GetWindowWMInfo(window, &info)) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "SDL_GetWindowWMInfo() failed: %s", - SDL_GetError()); - return false; - } - - SDL_assert(info.subsystem == SDL_SYSWM_COCOA); - - m_DisplayFps = displayFps; - - NSScreen* screen = [info.info.cocoa.window screen]; - CVReturn status; - if (screen == nullptr) { - // Window not visible on any display, so use a - // CVDisplayLink that can work with all active displays. - // When we become visible, we'll recreate ourselves - // and associate with the new screen. - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "NSWindow is not visible on any display"); - status = CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink); - } - else { - CGDirectDisplayID displayId; - displayId = getDisplayID(screen); - SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, - "NSWindow on display: %x", - displayId); - status = CVDisplayLinkCreateWithCGDisplay(displayId, &m_DisplayLink); - } - if (status != kCVReturnSuccess) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Failed to create CVDisplayLink: %d", - status); - return false; - } - - status = CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, this); - if (status != kCVReturnSuccess) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "CVDisplayLinkSetOutputCallback() failed: %d", - status); - return false; - } - - status = CVDisplayLinkStart(m_DisplayLink); - if (status != kCVReturnSuccess) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "CVDisplayLinkStart() failed: %d", - status); - return false; - } - - return true; - } - -private: - Pacer* m_Pacer; - CVDisplayLinkRef m_DisplayLink; - int m_DisplayFps; -}; - -IVsyncSource* DisplayLinkVsyncSourceFactory::createVsyncSource(Pacer* pacer) { - return new DisplayLinkVsyncSource(pacer); -} diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp index 3ab4befa..1ff8d49c 100644 --- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp +++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp @@ -3,10 +3,6 @@ #include "nullthreadedvsyncsource.h" -#ifdef Q_OS_DARWIN -#include "displaylinkvsyncsource.h" -#endif - #ifdef Q_OS_WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -230,9 +226,7 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing) "Frame pacing active: target %d Hz with %d FPS stream", m_DisplayFps, m_MaxVideoFps); - #if defined(Q_OS_DARWIN) - m_VsyncSource = DisplayLinkVsyncSourceFactory::createVsyncSource(this); - #elif defined(Q_OS_WIN32) + #if defined(Q_OS_WIN32) // Don't use D3DKMTWaitForVerticalBlankEvent() on Windows 7, because // it blocks during other concurrent DX operations (like actually rendering). if (IsWindows8OrGreater()) { diff --git a/app/streaming/video/ffmpeg-renderers/vt.mm b/app/streaming/video/ffmpeg-renderers/vt.mm index d155f455..7f3501dc 100644 --- a/app/streaming/video/ffmpeg-renderers/vt.mm +++ b/app/streaming/video/ffmpeg-renderers/vt.mm @@ -22,13 +22,24 @@ public: : m_HwContext(nullptr), m_DisplayLayer(nullptr), m_FormatDesc(nullptr), - m_StreamView(nullptr) + m_StreamView(nullptr), + m_DisplayLink(nullptr) { SDL_zero(m_OverlayTextFields); } virtual ~VTRenderer() override { + if (m_DisplayLink != nullptr) { + // Wake up the renderer in case it is waiting for v-sync + SDL_LockMutex(m_VsyncMutex); + SDL_CondSignal(m_VsyncPassed); + SDL_UnlockMutex(m_VsyncMutex); + + CVDisplayLinkStop(m_DisplayLink); + CVDisplayLinkRelease(m_DisplayLink); + } + if (m_HwContext != nullptr) { av_buffer_unref(&m_HwContext); } @@ -48,6 +59,85 @@ public: } } + static + CGDirectDisplayID + getDisplayID(NSScreen* screen) + { + NSNumber* screenNumber = [screen deviceDescription][@"NSScreenNumber"]; + return [screenNumber unsignedIntValue]; + } + + static + CVReturn + displayLinkOutputCallback( + CVDisplayLinkRef displayLink, + const CVTimeStamp* /* now */, + const CVTimeStamp* /* vsyncTime */, + CVOptionFlags, + CVOptionFlags*, + void *displayLinkContext) + { + auto me = reinterpret_cast(displayLinkContext); + + SDL_assert(displayLink == me->m_DisplayLink); + + SDL_LockMutex(me->m_VsyncMutex); + SDL_CondSignal(me->m_VsyncPassed); + SDL_UnlockMutex(me->m_VsyncMutex); + + return kCVReturnSuccess; + } + + bool initializeVsyncCallback(SDL_SysWMinfo* info) + { + NSScreen* screen = [info->info.cocoa.window screen]; + CVReturn status; + if (screen == nullptr) { + // Window not visible on any display, so use a + // CVDisplayLink that can work with all active displays. + // When we become visible, we'll recreate ourselves + // and associate with the new screen. + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "NSWindow is not visible on any display"); + status = CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink); + } + else { + CGDirectDisplayID displayId; + displayId = getDisplayID(screen); + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "NSWindow on display: %x", + displayId); + status = CVDisplayLinkCreateWithCGDisplay(displayId, &m_DisplayLink); + } + if (status != kCVReturnSuccess) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to create CVDisplayLink: %d", + status); + return false; + } + + status = CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, this); + if (status != kCVReturnSuccess) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CVDisplayLinkSetOutputCallback() failed: %d", + status); + return false; + } + + status = CVDisplayLinkStart(m_DisplayLink); + if (status != kCVReturnSuccess) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CVDisplayLinkStart() failed: %d", + status); + return false; + } + + m_VsyncMutex = SDL_CreateMutex(); + m_VsyncPassed = SDL_CreateCond(); + + return true; + } + // Caller frees frame after we return virtual void renderFrame(AVFrame* frame) override { @@ -104,6 +194,13 @@ public: [m_DisplayLayer enqueueSampleBuffer:sampleBuffer]; CFRelease(sampleBuffer); + + if (m_DisplayLink != nullptr) { + // Vsync is enabled, so wait for a swap before returning + SDL_LockMutex(m_VsyncMutex); + SDL_CondWait(m_VsyncPassed, m_VsyncMutex); + SDL_UnlockMutex(m_VsyncMutex); + } } virtual bool initialize(PDECODER_PARAMETERS params) override @@ -191,6 +288,12 @@ public: return false; } + if (params->enableVsync) { + if (!initializeVsyncCallback(&info)) { + return false; + } + } + return true; } @@ -278,20 +381,15 @@ public: return true; } - virtual IFFmpegRenderer::FramePacingConstraint getFramePacingConstraint() override - { - // This renderer is inherently tied to V-sync due how we're - // rendering with AVSampleBufferDisplay layer. Running without - // the V-Sync source leads to massive stuttering. - return PACING_FORCE_ON; - } - private: AVBufferRef* m_HwContext; AVSampleBufferDisplayLayer* m_DisplayLayer; CMVideoFormatDescriptionRef m_FormatDesc; NSView* m_StreamView; NSTextField* m_OverlayTextFields[Overlay::OverlayMax]; + CVDisplayLinkRef m_DisplayLink; + SDL_mutex* m_VsyncMutex; + SDL_cond* m_VsyncPassed; }; IFFmpegRenderer* VTRendererFactory::createRenderer() {