From d997a0d0c878afa9eeaf7c81b5e315f02e52749e Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 23 Jun 2026 21:09:20 -0500 Subject: [PATCH] Dynamically adjust libplacebo swapchain depth on macOS Direct-to-display mode on some Macs (M4 Mac Mini) can retain our drawables, which cuts the frame rate in half if we're in double-buffered mode. We don't want to use triple-buffered mode all the time because it increases latency when used if not required, so detect delayed presentation and enable it on the fly if needed. --- app/streaming/video/ffmpeg-renderers/plvk.cpp | 92 ++++++++++++++----- app/streaming/video/ffmpeg-renderers/plvk.h | 15 +++ 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/app/streaming/video/ffmpeg-renderers/plvk.cpp b/app/streaming/video/ffmpeg-renderers/plvk.cpp index fddede67..8ada74a3 100644 --- a/app/streaming/video/ffmpeg-renderers/plvk.cpp +++ b/app/streaming/video/ffmpeg-renderers/plvk.cpp @@ -421,6 +421,7 @@ bool PlVkRenderer::isExtensionSupportedByPhysicalDevice(VkPhysicalDevice device, bool PlVkRenderer::initialize(PDECODER_PARAMETERS params) { m_Window = params->window; + m_MaxVideoFps = params->frameRate; unsigned int instanceExtensionCount = 0; if (!SDL_Vulkan_GetInstanceExtensions(params->window, &instanceExtensionCount, nullptr)) { @@ -488,18 +489,17 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params) return false; } - VkPresentModeKHR presentMode; if (params->enableVsync) { // FIFO mode improves frame pacing compared with Mailbox, especially for // platforms like X11 that lack a VSyncSource implementation for Pacer. - presentMode = VK_PRESENT_MODE_FIFO_KHR; + m_VkPresentMode = VK_PRESENT_MODE_FIFO_KHR; } else { // We want immediate mode for V-Sync disabled if possible if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_IMMEDIATE_KHR)) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using Immediate present mode with V-Sync disabled"); - presentMode = VK_PRESENT_MODE_IMMEDIATE_KHR; + m_VkPresentMode = VK_PRESENT_MODE_IMMEDIATE_KHR; } else { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, @@ -509,41 +509,26 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params) if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_FIFO_RELAXED_KHR)) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using FIFO Relaxed present mode with V-Sync disabled"); - presentMode = VK_PRESENT_MODE_FIFO_RELAXED_KHR; + m_VkPresentMode = VK_PRESENT_MODE_FIFO_RELAXED_KHR; } // Mailbox at least provides non-blocking behavior else if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_MAILBOX_KHR)) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using Mailbox present mode with V-Sync disabled"); - presentMode = VK_PRESENT_MODE_MAILBOX_KHR; + m_VkPresentMode = VK_PRESENT_MODE_MAILBOX_KHR; } // FIFO is always supported else { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using FIFO present mode with V-Sync disabled"); - presentMode = VK_PRESENT_MODE_FIFO_KHR; + m_VkPresentMode = VK_PRESENT_MODE_FIFO_KHR; } } } - pl_vulkan_swapchain_params vkSwapchainParams = {}; - vkSwapchainParams.surface = m_VkSurface; - vkSwapchainParams.present_mode = presentMode; - vkSwapchainParams.swapchain_depth = 1; // No queued frames -#if PL_API_VER >= 338 - vkSwapchainParams.disable_10bit_sdr = true; // Some drivers don't dither 10-bit SDR output correctly -#endif - - { - // Don't let Qt take DRM master from us during pl_vulkan_create_swapchain() - DrmMasterLocker locker; - - m_Swapchain = pl_vulkan_create_swapchain(m_Vulkan, &vkSwapchainParams); - if (m_Swapchain == nullptr) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "pl_vulkan_create_swapchain() failed"); - return false; - } + // Start with a swapchain that is double-buffered for lowest display latency + if (!createSwapchain(1)) { + return false; } m_Renderer = pl_renderer_create(m_Log, m_Vulkan->gpu); @@ -622,6 +607,35 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params) return true; } + +bool PlVkRenderer::createSwapchain(int depth) +{ + pl_swapchain_destroy(&m_Swapchain); + + pl_vulkan_swapchain_params vkSwapchainParams = {}; + vkSwapchainParams.surface = m_VkSurface; + vkSwapchainParams.present_mode = m_VkPresentMode; + vkSwapchainParams.swapchain_depth = depth; +#if PL_API_VER >= 338 + vkSwapchainParams.disable_10bit_sdr = true; // Some drivers don't dither 10-bit SDR output correctly +#endif + + { + // Don't let Qt take DRM master from us during pl_vulkan_create_swapchain() + DrmMasterLocker locker; + + m_Swapchain = pl_vulkan_create_swapchain(m_Vulkan, &vkSwapchainParams); + if (m_Swapchain == nullptr) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "pl_vulkan_create_swapchain() failed"); + return false; + } + } + + m_SwapchainDepth = depth; + return true; +} + bool PlVkRenderer::prepareDecoderContext(AVCodecContext *context, AVDictionary **) { if (m_HwDeviceCtx) { @@ -991,6 +1005,10 @@ void PlVkRenderer::renderFrame(AVFrame *frame) targetFrame.crop.x1 = dst.x + dst.w; targetFrame.crop.y1 = dst.y + dst.h; +#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH + Uint32 renderStartTime = SDL_GetTicks(); +#endif + // Render the video image and overlays into the swapchain buffer targetFrame.num_overlays = (int)overlays.size(); targetFrame.overlays = overlays.data(); @@ -1013,6 +1031,32 @@ void PlVkRenderer::renderFrame(AVFrame *frame) goto UnmapExit; } +#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH + // Trigger a switch to triple-buffered mode if our frame presentation time + // exceeds 110% of the frame interval for half a second of frames. + if (SDL_GetTicks() - renderStartTime > (1100U / m_MaxVideoFps)) { + m_DelayedPresents++; + } + else if (m_DelayedPresents > 0) { + m_DelayedPresents--; + } + + if (m_DelayedPresents == m_MaxVideoFps / 2 && m_SwapchainDepth < 2) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Switching to triple-buffered swapchain after delayed presentations"); + if (!createSwapchain(2)) { + // Recreate the renderer + SDL_Event event; + event.type = SDL_RENDER_DEVICE_RESET; + SDL_PushEvent(&event); + goto UnmapExit; + } + + // Restore the swapchain's colorspace from the previous swapchain frame + pl_swapchain_colorspace_hint(m_Swapchain, &targetFrame.color); + } +#endif + #ifdef Q_OS_WIN32 // On Windows, we swap buffers here instead of waitToRender() // to avoid some performance problems on Nvidia GPUs. diff --git a/app/streaming/video/ffmpeg-renderers/plvk.h b/app/streaming/video/ffmpeg-renderers/plvk.h index b8fca0bf..1ea282f5 100644 --- a/app/streaming/video/ffmpeg-renderers/plvk.h +++ b/app/streaming/video/ffmpeg-renderers/plvk.h @@ -23,6 +23,11 @@ private: pl_vulkan m_Vulkan; /* CVMetalTextureCacheRef */ void* m_TextureCache = nullptr; }; + +// Work around direct-to-display mode sometimes (but not always!) blocking us +// from getting a new drawable while the current one is getting scanned out. +#define PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH 1 + #endif class PlVkRenderer : public IFFmpegRenderer { @@ -49,6 +54,7 @@ private: static void unlockQueue(AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index); static void overlayUploadComplete(void* opaque); + bool createSwapchain(int depth); bool mapAvFrameToPlacebo(const AVFrame *frame, pl_frame* mappedFrame); void unmapAvFrameFromPlacebo(const AVFrame *frame, pl_frame* mappedFrame); bool populateQueues(int videoFormat); @@ -68,13 +74,22 @@ private: std::unique_ptr m_MetalTextureFactory; #endif +#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH + int m_DelayedPresents = 0; +#endif + // SDL state SDL_Window* m_Window = nullptr; + // Stream state + int m_MaxVideoFps; + // The libplacebo rendering state pl_log m_Log = nullptr; pl_vk_inst m_PlVkInstance = nullptr; VkSurfaceKHR m_VkSurface = VK_NULL_HANDLE; + int m_SwapchainDepth = 0; + VkPresentModeKHR m_VkPresentMode; pl_vulkan m_Vulkan = nullptr; pl_swapchain m_Swapchain = nullptr; pl_renderer m_Renderer = nullptr;