diff --git a/app/streaming/video/ffmpeg-renderers/vt_metal.mm b/app/streaming/video/ffmpeg-renderers/vt_metal.mm index 12eb7df2..d1061026 100644 --- a/app/streaming/video/ffmpeg-renderers/vt_metal.mm +++ b/app/streaming/video/ffmpeg-renderers/vt_metal.mm @@ -398,6 +398,81 @@ public: return m_SwMappingTextures[planeIndex]; } + bool createTexturesFromFrame(AVFrame* frame, std::array& cvMetalTextures) + { + SDL_assert(frame->format == AV_PIX_FMT_VIDEOTOOLBOX); + + CVPixelBufferRef pixBuf = reinterpret_cast(frame->data[3]); + size_t planes = getFramePlaneCount(frame); + + // Create Metal textures for the planes of the CVPixelBuffer + for (size_t i = 0; i < planes; i++) { + MTLPixelFormat fmt; + + switch (CVPixelBufferGetPixelFormatType(pixBuf)) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: + fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm; + break; + + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: + fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm; + break; + + default: + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unknown pixel format: %x", + CVPixelBufferGetPixelFormatType(pixBuf)); + return false; + } + + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_TextureCache, pixBuf, nullptr, fmt, + CVPixelBufferGetWidthOfPlane(pixBuf, i), + CVPixelBufferGetHeightOfPlane(pixBuf, i), + i, + &cvMetalTextures[i]); + if (err != kCVReturnSuccess) { + for (size_t j = 0; j < i; j++) { + CFRelease(cvMetalTextures[j]); + } + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CVMetalTextureCacheCreateTextureFromImage() failed: %d", + err); + return false; + } + } + + return true; + } + + bool testRenderFrame(AVFrame *frame) override + { @autoreleasepool { + if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + std::array cvMetalTextures; + size_t planes = getFramePlaneCount(frame); + SDL_assert(planes <= MAX_VIDEO_PLANES); + + // Test that we can actually create Metal textures from the CVPixelBufferRef + if (!createTexturesFromFrame(frame, cvMetalTextures)) { + return false; + } + + for (size_t i = 0; i < planes; i++) { + CFRelease(cvMetalTextures[i]); + } + } + else { + // Mapping software frames should always work + } + + return true; + }} + // Caller frees frame after we return virtual void renderFrameIntoDrawable(AVFrame* frame, id drawable) { @autoreleasepool { @@ -406,45 +481,8 @@ public: SDL_assert(planes <= MAX_VIDEO_PLANES); if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { - CVPixelBufferRef pixBuf = reinterpret_cast(frame->data[3]); - - // Create Metal textures for the planes of the CVPixelBuffer - for (size_t i = 0; i < planes; i++) { - MTLPixelFormat fmt; - - switch (CVPixelBufferGetPixelFormatType(pixBuf)) { - case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: - case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: - fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm; - break; - - case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: - case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: - case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: - case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: - fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm; - break; - - default: - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Unknown pixel format: %x", - CVPixelBufferGetPixelFormatType(pixBuf)); - return; - } - - CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_TextureCache, pixBuf, nullptr, fmt, - CVPixelBufferGetWidthOfPlane(pixBuf, i), - CVPixelBufferGetHeightOfPlane(pixBuf, i), - i, - &cvMetalTextures[i]); - if (err != kCVReturnSuccess) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "CVMetalTextureCacheCreateTextureFromImage() failed: %d", - err); - return; - } + if (!createTexturesFromFrame(frame, cvMetalTextures)) { + return; } }