From 200cab9d17ef63048a0d9f7a184bed604376b25b Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 20 Oct 2025 22:28:27 -0500 Subject: [PATCH] Attach EDR metadata to Metal layer This improves the accuracy of HDR streaming and enables HDR->SDR tonemapping. --- app/streaming/video/ffmpeg-renderers/vt.h | 9 ++- .../ffmpeg-renderers/vt_avsamplelayer.mm | 69 ----------------- .../video/ffmpeg-renderers/vt_base.mm | 76 +++++++++++++++++++ .../video/ffmpeg-renderers/vt_metal.mm | 18 ++++- 4 files changed, 98 insertions(+), 74 deletions(-) diff --git a/app/streaming/video/ffmpeg-renderers/vt.h b/app/streaming/video/ffmpeg-renderers/vt.h index 84a8e4bc..b608901c 100644 --- a/app/streaming/video/ffmpeg-renderers/vt.h +++ b/app/streaming/video/ffmpeg-renderers/vt.h @@ -6,8 +6,15 @@ #import class VTBaseRenderer : public IFFmpegRenderer { public: - VTBaseRenderer(IFFmpegRenderer::RendererType type) : IFFmpegRenderer(type) {} + VTBaseRenderer(IFFmpegRenderer::RendererType type); + virtual ~VTBaseRenderer(); bool checkDecoderCapabilities(id device, PDECODER_PARAMETERS params); + void setHdrMode(bool enabled) override; + +protected: + bool m_HdrMetadataChanged; // Manual reset + CFDataRef m_MasteringDisplayColorVolume; + CFDataRef m_ContentLightLevelInfo; }; #endif diff --git a/app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm b/app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm index bd552810..067aeb99 100644 --- a/app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm +++ b/app/streaming/video/ffmpeg-renderers/vt_avsamplelayer.mm @@ -39,8 +39,6 @@ public: m_HwContext(nullptr), m_DisplayLayer(nullptr), m_FormatDesc(nullptr), - m_ContentLightLevelInfo(nullptr), - m_MasteringDisplayColorVolume(nullptr), m_StreamView(nullptr), m_DisplayLink(nullptr), m_LastColorSpace(-1), @@ -90,14 +88,6 @@ public: CGColorSpaceRelease(m_ColorSpace); } - if (m_MasteringDisplayColorVolume != nullptr) { - CFRelease(m_MasteringDisplayColorVolume); - } - - if (m_ContentLightLevelInfo != nullptr) { - CFRelease(m_ContentLightLevelInfo); - } - for (int i = 0; i < Overlay::OverlayMax; i++) { if (m_OverlayTextFields[i] != nullptr) { [m_OverlayTextFields[i] removeFromSuperview]; @@ -204,63 +194,6 @@ public: } } - virtual void setHdrMode(bool enabled) override - { - // Free existing HDR metadata - if (m_MasteringDisplayColorVolume != nullptr) { - CFRelease(m_MasteringDisplayColorVolume); - m_MasteringDisplayColorVolume = nullptr; - } - if (m_ContentLightLevelInfo != nullptr) { - CFRelease(m_ContentLightLevelInfo); - m_ContentLightLevelInfo = nullptr; - } - - // Store new HDR metadata if available - SS_HDR_METADATA hdrMetadata; - if (enabled && LiGetHdrMetadata(&hdrMetadata)) { - if (hdrMetadata.displayPrimaries[0].x != 0 && hdrMetadata.maxDisplayLuminance != 0) { - // This data is all in big-endian - struct { - vector_ushort2 primaries[3]; - vector_ushort2 white_point; - uint32_t luminance_max; - uint32_t luminance_min; - } __attribute__((packed, aligned(4))) mdcv; - - // mdcv is in GBR order while SS_HDR_METADATA is in RGB order - mdcv.primaries[0].x = __builtin_bswap16(hdrMetadata.displayPrimaries[1].x); - mdcv.primaries[0].y = __builtin_bswap16(hdrMetadata.displayPrimaries[1].y); - mdcv.primaries[1].x = __builtin_bswap16(hdrMetadata.displayPrimaries[2].x); - mdcv.primaries[1].y = __builtin_bswap16(hdrMetadata.displayPrimaries[2].y); - mdcv.primaries[2].x = __builtin_bswap16(hdrMetadata.displayPrimaries[0].x); - mdcv.primaries[2].y = __builtin_bswap16(hdrMetadata.displayPrimaries[0].y); - - mdcv.white_point.x = __builtin_bswap16(hdrMetadata.whitePoint.x); - mdcv.white_point.y = __builtin_bswap16(hdrMetadata.whitePoint.y); - - // These luminance values are in 10000ths of a nit - mdcv.luminance_max = __builtin_bswap32((uint32_t)hdrMetadata.maxDisplayLuminance * 10000); - mdcv.luminance_min = __builtin_bswap32(hdrMetadata.minDisplayLuminance); - - m_MasteringDisplayColorVolume = CFDataCreate(nullptr, (const UInt8*)&mdcv, sizeof(mdcv)); - } - - if (hdrMetadata.maxContentLightLevel != 0 && hdrMetadata.maxFrameAverageLightLevel != 0) { - // This data is all in big-endian - struct { - uint16_t max_content_light_level; - uint16_t max_frame_average_light_level; - } __attribute__((packed, aligned(2))) cll; - - cll.max_content_light_level = __builtin_bswap16(hdrMetadata.maxContentLightLevel); - cll.max_frame_average_light_level = __builtin_bswap16(hdrMetadata.maxFrameAverageLightLevel); - - m_ContentLightLevelInfo = CFDataCreate(nullptr, (const UInt8*)&cll, sizeof(cll)); - } - } - } - // Caller frees frame after we return virtual void renderFrame(AVFrame* frame) override { @autoreleasepool { @@ -573,8 +506,6 @@ private: AVBufferRef* m_HwContext; AVSampleBufferDisplayLayer* m_DisplayLayer; CMVideoFormatDescriptionRef m_FormatDesc; - CFDataRef m_ContentLightLevelInfo; - CFDataRef m_MasteringDisplayColorVolume; NSView* m_StreamView; dispatch_block_t m_OverlayUpdateBlocks[Overlay::OverlayMax]; NSTextField* m_OverlayTextFields[Overlay::OverlayMax]; diff --git a/app/streaming/video/ffmpeg-renderers/vt_base.mm b/app/streaming/video/ffmpeg-renderers/vt_base.mm index 57a06351..cdf97f6f 100644 --- a/app/streaming/video/ffmpeg-renderers/vt_base.mm +++ b/app/streaming/video/ffmpeg-renderers/vt_base.mm @@ -9,6 +9,24 @@ #import #import +VTBaseRenderer::VTBaseRenderer(IFFmpegRenderer::RendererType type) : + IFFmpegRenderer(type), + m_HdrMetadataChanged(false), + m_MasteringDisplayColorVolume(nullptr), + m_ContentLightLevelInfo(nullptr) { + +} + +VTBaseRenderer::~VTBaseRenderer() { + if (m_MasteringDisplayColorVolume != nullptr) { + CFRelease(m_MasteringDisplayColorVolume); + } + + if (m_ContentLightLevelInfo != nullptr) { + CFRelease(m_ContentLightLevelInfo); + } +} + bool VTBaseRenderer::checkDecoderCapabilities(id device, PDECODER_PARAMETERS params) { if (params->videoFormat & VIDEO_FORMAT_MASK_H264) { if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) { @@ -76,3 +94,61 @@ bool VTBaseRenderer::checkDecoderCapabilities(id device, PDECODER_PAR return true; } + +void VTBaseRenderer::setHdrMode(bool enabled) { + // Free existing HDR metadata + if (m_MasteringDisplayColorVolume != nullptr) { + CFRelease(m_MasteringDisplayColorVolume); + m_MasteringDisplayColorVolume = nullptr; + } + if (m_ContentLightLevelInfo != nullptr) { + CFRelease(m_ContentLightLevelInfo); + m_ContentLightLevelInfo = nullptr; + } + + // Store new HDR metadata if available + SS_HDR_METADATA hdrMetadata; + if (enabled && LiGetHdrMetadata(&hdrMetadata)) { + if (hdrMetadata.displayPrimaries[0].x != 0 && hdrMetadata.maxDisplayLuminance != 0) { + // This data is all in big-endian + struct { + vector_ushort2 primaries[3]; + vector_ushort2 white_point; + uint32_t luminance_max; + uint32_t luminance_min; + } __attribute__((packed, aligned(4))) mdcv; + + // mdcv is in GBR order while SS_HDR_METADATA is in RGB order + mdcv.primaries[0].x = __builtin_bswap16(hdrMetadata.displayPrimaries[1].x); + mdcv.primaries[0].y = __builtin_bswap16(hdrMetadata.displayPrimaries[1].y); + mdcv.primaries[1].x = __builtin_bswap16(hdrMetadata.displayPrimaries[2].x); + mdcv.primaries[1].y = __builtin_bswap16(hdrMetadata.displayPrimaries[2].y); + mdcv.primaries[2].x = __builtin_bswap16(hdrMetadata.displayPrimaries[0].x); + mdcv.primaries[2].y = __builtin_bswap16(hdrMetadata.displayPrimaries[0].y); + + mdcv.white_point.x = __builtin_bswap16(hdrMetadata.whitePoint.x); + mdcv.white_point.y = __builtin_bswap16(hdrMetadata.whitePoint.y); + + // These luminance values are in 10000ths of a nit + mdcv.luminance_max = __builtin_bswap32((uint32_t)hdrMetadata.maxDisplayLuminance * 10000); + mdcv.luminance_min = __builtin_bswap32(hdrMetadata.minDisplayLuminance); + + m_MasteringDisplayColorVolume = CFDataCreate(nullptr, (const UInt8*)&mdcv, sizeof(mdcv)); + } + + if (hdrMetadata.maxContentLightLevel != 0 && hdrMetadata.maxFrameAverageLightLevel != 0) { + // This data is all in big-endian + struct { + uint16_t max_content_light_level; + uint16_t max_frame_average_light_level; + } __attribute__((packed, aligned(2))) cll; + + cll.max_content_light_level = __builtin_bswap16(hdrMetadata.maxContentLightLevel); + cll.max_frame_average_light_level = __builtin_bswap16(hdrMetadata.maxFrameAverageLightLevel); + + m_ContentLightLevelInfo = CFDataCreate(nullptr, (const UInt8*)&cll, sizeof(cll)); + } + } + + m_HdrMetadataChanged = true; +} diff --git a/app/streaming/video/ffmpeg-renderers/vt_metal.mm b/app/streaming/video/ffmpeg-renderers/vt_metal.mm index 58d39e3d..8fcb9df8 100644 --- a/app/streaming/video/ffmpeg-renderers/vt_metal.mm +++ b/app/streaming/video/ffmpeg-renderers/vt_metal.mm @@ -326,7 +326,7 @@ public: { int colorspace = getFrameColorspace(frame); bool fullRange = isFrameFullRange(frame); - if (colorspace != m_LastColorSpace || fullRange != m_LastFullRange) { + if (colorspace != m_LastColorSpace || fullRange != m_LastFullRange || m_HdrMetadataChanged) { CGColorSpaceRef newColorSpace; ParamBuffer paramBuffer; @@ -340,14 +340,13 @@ public: paramBuffer.cscParams = (fullRange ? k_CscParams_Bt709Full : k_CscParams_Bt709Lim); break; case COLORSPACE_REC_2020: - // https://developer.apple.com/documentation/metal/hdr_content/using_color_spaces_to_display_hdr_content + m_MetalLayer.pixelFormat = MTLPixelFormatBGR10A2Unorm; if (frame->color_trc == AVCOL_TRC_SMPTE2084) { + // https://developer.apple.com/documentation/metal/hdr_content/using_color_spaces_to_display_hdr_content m_MetalLayer.colorspace = newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ); - m_MetalLayer.pixelFormat = MTLPixelFormatBGR10A2Unorm; } else { m_MetalLayer.colorspace = newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020); - m_MetalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm; } paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full : k_CscParams_Bt2020Lim); break; @@ -359,6 +358,16 @@ public: break; } + // Set the EDR metadata for HDR10 to enable OS tonemapping + if (frame->color_trc == AVCOL_TRC_SMPTE2084 && m_MasteringDisplayColorVolume != nullptr) { + m_MetalLayer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:(__bridge NSData*)m_MasteringDisplayColorVolume + contentInfo:(__bridge NSData*)m_ContentLightLevelInfo + opticalOutputScale:203.0]; + } + else { + m_MetalLayer.EDRMetadata = nullptr; + } + paramBuffer.bitnessScaleFactor = getBitnessScaleFactor(frame); // The CAMetalLayer retains the CGColorSpace @@ -410,6 +419,7 @@ public: m_LastColorSpace = colorspace; m_LastFullRange = fullRange; + m_HdrMetadataChanged = false; } return true;