diff --git a/app/app.pro b/app/app.pro index 12067522..164eacd5 100644 --- a/app/app.pro +++ b/app/app.pro @@ -350,6 +350,10 @@ libplacebo { streaming/video/ffmpeg-renderers/plvk_c.c HEADERS += \ streaming/video/ffmpeg-renderers/plvk.h + + macx { + SOURCES += streaming/video/ffmpeg-renderers/plvk_objc.mm + } } config_EGL { message(EGL renderer selected) diff --git a/app/main.cpp b/app/main.cpp index b7d57853..532e5d03 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -635,8 +635,13 @@ int main(int argc, char *argv[]) qputenv("QSG_RENDER_LOOP", "basic"); #endif -#if defined(Q_OS_DARWIN) && defined(QT_DEBUG) - // Enable Metal valiation for debug builds +#if defined(Q_OS_DARWIN) && defined(QT_DEBUG) && !defined(HAVE_LIBPLACEBO_VULKAN) + // Enable Metal valiation for debug builds without libplacebo + // + // The current MoltenVK driver as of Vulkan SDK 1.4.350 triggers Metal debug layer + // violations on frame and overlay uploads like: + // _validateReplaceRegion:252: failed assertion `Replace Region Validation + // bytesPerRow(4803) must be a multiple of MTLPixelFormatBGRA8Unorm pixel bytes(4). qputenv("MTL_DEBUG_LAYER", "1"); qputenv("MTL_SHADER_VALIDATION", "1"); #endif diff --git a/app/streaming/streamutils.cpp b/app/streaming/streamutils.cpp index b5372e34..9bd32f50 100644 --- a/app/streaming/streamutils.cpp +++ b/app/streaming/streamutils.cpp @@ -74,11 +74,12 @@ static int __riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, Uint32 StreamUtils::getPlatformWindowFlags() { -#if defined(Q_OS_DARWIN) - return SDL_WINDOW_METAL; -#elif defined(HAVE_LIBPLACEBO_VULKAN) +#if defined(HAVE_LIBPLACEBO_VULKAN) // We'll fall back to GL if Vulkan fails return SDL_WINDOW_VULKAN; +#elif defined(Q_OS_DARWIN) + // Vulkan needs to supersede Metal, otherwise the Vulkan library won't be loaded + return SDL_WINDOW_METAL; #else return 0; #endif diff --git a/app/streaming/video/ffmpeg-renderers/plvk.cpp b/app/streaming/video/ffmpeg-renderers/plvk.cpp index f5c8d508..cbef1f91 100644 --- a/app/streaming/video/ffmpeg-renderers/plvk.cpp +++ b/app/streaming/video/ffmpeg-renderers/plvk.cpp @@ -177,6 +177,9 @@ PlVkRenderer::~PlVkRenderer() pl_renderer_destroy(&m_Renderer); pl_swapchain_destroy(&m_Swapchain); +#ifdef Q_OS_DARWIN + m_MetalTextureFactory.reset(); +#endif pl_vulkan_destroy(&m_Vulkan); // This surface was created by SDL, so there's no libplacebo API to destroy it @@ -358,7 +361,10 @@ bool PlVkRenderer::tryInitializeDevice(VkPhysicalDevice device, VkPhysicalDevice vkParams.device = device; if (m_HwDeviceType == AV_HWDEVICE_TYPE_VULKAN) { -#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(60, 26, 100) +#if defined(Q_OS_DARWIN) + vkParams.opt_extensions = nullptr; + vkParams.num_opt_extensions = 0; +#elif LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(60, 26, 100) vkParams.opt_extensions = av_vk_get_optional_device_extensions(&vkParams.num_opt_extensions); #else vkParams.opt_extensions = k_OptionalDeviceExtensions; @@ -602,6 +608,10 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params) } } +#ifdef Q_OS_DARWIN + m_MetalTextureFactory = std::make_unique(m_Vulkan); +#endif + return true; } @@ -625,13 +635,23 @@ bool PlVkRenderer::prepareDecoderContext(AVCodecContext *context, AVDictionary * bool PlVkRenderer::mapAvFrameToPlacebo(const AVFrame *frame, pl_frame* mappedFrame) { - pl_avframe_params mapParams = {}; - mapParams.frame = frame; - mapParams.tex = m_Textures; - if (!pl_map_avframe_ex(m_Vulkan->gpu, mappedFrame, &mapParams)) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "pl_map_avframe_ex() failed"); - return false; +#ifdef Q_OS_DARWIN + if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + if (!m_MetalTextureFactory->mapVideoToolboxToPlacebo(frame, mappedFrame)) { + return false; + } + } + else +#endif + { + pl_avframe_params mapParams = {}; + mapParams.frame = frame; + mapParams.tex = m_Textures; + if (!pl_map_avframe_ex(m_Vulkan->gpu, mappedFrame, &mapParams)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "pl_map_avframe_ex() failed"); + return false; + } } // libplacebo assumes a minimum luminance value of 0 means the actual value was unknown. @@ -653,6 +673,19 @@ bool PlVkRenderer::mapAvFrameToPlacebo(const AVFrame *frame, pl_frame* mappedFra return true; } +void PlVkRenderer::unmapAvFrameFromPlacebo(const AVFrame *frame, pl_frame* mappedFrame) +{ +#ifdef Q_OS_DARWIN + if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + m_MetalTextureFactory->unmapVideoToolboxFromPlacebo(mappedFrame); + } + else +#endif + { + pl_unmap_avframe(m_Vulkan->gpu, mappedFrame); + } +} + bool PlVkRenderer::populateQueues(int videoFormat) { auto vkDeviceContext = (AVVulkanDeviceContext*)((AVHWDeviceContext *)m_HwDeviceCtx->data)->hwctx; @@ -971,7 +1004,7 @@ UnmapExit: pl_tex_destroy(m_Vulkan->gpu, &texture); } - pl_unmap_avframe(m_Vulkan->gpu, &mappedFrame); + unmapAvFrameFromPlacebo(frame, &mappedFrame); } bool PlVkRenderer::testRenderFrame(AVFrame *frame) @@ -995,7 +1028,7 @@ bool PlVkRenderer::testRenderFrame(AVFrame *frame) return false; } - pl_unmap_avframe(m_Vulkan->gpu, &mappedFrame); + unmapAvFrameFromPlacebo(frame, &mappedFrame); return true; } diff --git a/app/streaming/video/ffmpeg-renderers/plvk.h b/app/streaming/video/ffmpeg-renderers/plvk.h index b0d24713..b8fca0bf 100644 --- a/app/streaming/video/ffmpeg-renderers/plvk.h +++ b/app/streaming/video/ffmpeg-renderers/plvk.h @@ -10,6 +10,21 @@ #include #include +#ifdef Q_OS_DARWIN +class MetalVulkanTextureFactory { +public: + MetalVulkanTextureFactory(pl_vulkan vulkan); + ~MetalVulkanTextureFactory(); + + bool mapVideoToolboxToPlacebo(const AVFrame *frame, pl_frame* mappedFrame); + void unmapVideoToolboxFromPlacebo(pl_frame* mappedFrame); + +private: + pl_vulkan m_Vulkan; + /* CVMetalTextureCacheRef */ void* m_TextureCache = nullptr; +}; +#endif + class PlVkRenderer : public IFFmpegRenderer { public: PlVkRenderer(AVHWDeviceType hwDeviceType = AV_HWDEVICE_TYPE_NONE, IFFmpegRenderer *backendRenderer = nullptr); @@ -35,6 +50,7 @@ private: static void overlayUploadComplete(void* opaque); bool mapAvFrameToPlacebo(const AVFrame *frame, pl_frame* mappedFrame); + void unmapAvFrameFromPlacebo(const AVFrame *frame, pl_frame* mappedFrame); bool populateQueues(int videoFormat); bool chooseVulkanDevice(PDECODER_PARAMETERS params, bool hdrOutputRequired); bool tryInitializeDevice(VkPhysicalDevice device, VkPhysicalDeviceProperties* deviceProps, @@ -48,6 +64,10 @@ private: IFFmpegRenderer* m_Backend; AVHWDeviceType m_HwDeviceType; +#ifdef Q_OS_DARWIN + std::unique_ptr m_MetalTextureFactory; +#endif + // SDL state SDL_Window* m_Window = nullptr; diff --git a/app/streaming/video/ffmpeg-renderers/plvk_objc.mm b/app/streaming/video/ffmpeg-renderers/plvk_objc.mm new file mode 100644 index 00000000..22aee7c2 --- /dev/null +++ b/app/streaming/video/ffmpeg-renderers/plvk_objc.mm @@ -0,0 +1,159 @@ +#include "plvk.h" + +#define PL_LIBAV_IMPLEMENTATION 0 +#include + +#import +#import + +#include +#include + +#include +#include + +static id getVulkanMetalDevice(pl_vulkan vulkan) +{ + VkExportMetalDeviceInfoEXT metalDeviceInfo = {}; + metalDeviceInfo.sType = VK_STRUCTURE_TYPE_EXPORT_METAL_DEVICE_INFO_EXT; + + VkExportMetalObjectsInfoEXT exportInfo = {}; + exportInfo.sType = VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECTS_INFO_EXT; + exportInfo.pNext = &metalDeviceInfo; + + auto vkExportMetalObjectsEXT = (PFN_vkExportMetalObjectsEXT)(vulkan->get_proc_addr(vulkan->instance, "vkExportMetalObjectsEXT")); + if (vkExportMetalObjectsEXT) { + vkExportMetalObjectsEXT(vulkan->device, &exportInfo); + return metalDeviceInfo.mtlDevice; + } + + return nullptr; +} + +MetalVulkanTextureFactory::MetalVulkanTextureFactory(pl_vulkan vulkan) : + m_Vulkan(vulkan) +{ + id device = getVulkanMetalDevice(vulkan); + if (!device) { + return; + } + + CFStringRef keys[1] = { kCVMetalTextureUsage }; + NSUInteger values[1] = { MTLTextureUsageShaderRead }; + auto cacheAttributes = CFDictionaryCreate(kCFAllocatorDefault, (const void**)keys, (const void**)values, 1, nullptr, nullptr); + int err = CVMetalTextureCacheCreate(kCFAllocatorDefault, cacheAttributes, device, nullptr, (CVMetalTextureCacheRef*)&m_TextureCache); + CFRelease(cacheAttributes); + + if (err != kCVReturnSuccess) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CVMetalTextureCacheCreate() failed: %d", + err); + } +} + +MetalVulkanTextureFactory::~MetalVulkanTextureFactory() +{ + if (m_TextureCache) { + CFRelease((CVMetalTextureCacheRef)m_TextureCache); + } +} + +bool MetalVulkanTextureFactory::mapVideoToolboxToPlacebo(const AVFrame *frame, pl_frame* mappedFrame) +{ + SDL_assert(frame->format == AV_PIX_FMT_VIDEOTOOLBOX); + + if (!m_TextureCache) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unable to map VT frames without Metal texture cache"); + return false; + } + + CVPixelBufferRef pixBuf = (CVPixelBufferRef)(frame->data[3]); + size_t planes = CVPixelBufferGetPlaneCount(pixBuf); + + // Map the AVFrame metadata into the pl_frame + pl_frame_from_avframe(mappedFrame, frame); + + // Create Metal textures for the planes of the CVPixelBuffer + mappedFrame->num_planes = 0; + for (size_t i = 0; i < planes; i++) { + MTLPixelFormat fmt; + + pl_tex_params texParams = {}; + texParams.w = CVPixelBufferGetWidthOfPlane(pixBuf, i); + texParams.h = CVPixelBufferGetHeightOfPlane(pixBuf, i); + texParams.sampleable = true; + + switch (CVPixelBufferGetPixelFormatType(pixBuf)) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: + fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm; + texParams.format = pl_find_named_fmt(m_Vulkan->gpu, (i == 0) ? "r8" : "rg8"); + break; + + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: + fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm; + texParams.format = pl_find_named_fmt(m_Vulkan->gpu, (i == 0) ? "r16" : "rg16"); + break; + + default: + unmapVideoToolboxFromPlacebo(mappedFrame); + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unknown pixel format: %x", + CVPixelBufferGetPixelFormatType(pixBuf)); + return false; + } + + CVMetalTextureRef cvTexture; + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, (CVMetalTextureCacheRef)m_TextureCache, + pixBuf, nullptr, fmt, texParams.w, texParams.h, i, + &cvTexture); + if (err != kCVReturnSuccess) { + unmapVideoToolboxFromPlacebo(mappedFrame); + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CVMetalTextureCacheCreateTextureFromImage() failed: %d", + err); + return false; + } + + if (i == 0) { + mappedFrame->planes[i].components = 1; + mappedFrame->planes[i].component_mapping[0] = PL_CHANNEL_Y; + } + else { + mappedFrame->planes[i].components = 2; + mappedFrame->planes[i].component_mapping[0] = PL_CHANNEL_CB; + mappedFrame->planes[i].component_mapping[1] = PL_CHANNEL_CR; + } + + texParams.import_handle = PL_HANDLE_MTL_TEX; + texParams.shared_mem.handle.handle = (__bridge void *)CVMetalTextureGetTexture(cvTexture); + texParams.user_data = cvTexture; + mappedFrame->planes[i].texture = pl_tex_create(m_Vulkan->gpu, &texParams); + if (!mappedFrame->planes[i].texture) { + CFRelease(cvTexture); + unmapVideoToolboxFromPlacebo(mappedFrame); + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "pl_tex_create() failed"); + return false; + } + + mappedFrame->num_planes++; + } + + return true; +} + +void MetalVulkanTextureFactory::unmapVideoToolboxFromPlacebo(pl_frame* mappedFrame) +{ + for (int i = 0; i < mappedFrame->num_planes; i++) { + auto cvTexture = (CVMetalTextureRef)mappedFrame->planes[i].texture->params.user_data; + pl_tex_destroy(m_Vulkan->gpu, &mappedFrame->planes[i].texture); + CFRelease(cvTexture); + } +}