diff --git a/app/app.pro b/app/app.pro index 1cb65640..ca25d548 100644 --- a/app/app.pro +++ b/app/app.pro @@ -41,7 +41,7 @@ win32 { } macx { LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56 - LIBS += -framework VideoToolbox + LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreGraphics -framework CoreMedia -framework AppKit } SOURCES += \ @@ -64,7 +64,7 @@ win32 { SOURCES += streaming/renderers/dxva2.cpp } macx { - SOURCES += streaming/renderers/vt.cpp + SOURCES += streaming/renderers/vt.mm } HEADERS += \ diff --git a/app/streaming/renderers/vt.cpp b/app/streaming/renderers/vt.cpp deleted file mode 100644 index 36553580..00000000 --- a/app/streaming/renderers/vt.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "vt.h" - -#include - -VTRenderer::VTRenderer() - : m_HwContext(nullptr) -{ - -} - -VTRenderer::~VTRenderer() -{ - if (m_HwContext != nullptr) { - av_buffer_unref(&m_HwContext); - } -} - -bool VTRenderer::prepareDecoderContext(AVCodecContext* context) -{ - context->hw_device_ctx = av_buffer_ref(m_HwContext); - return true; -} - -bool VTRenderer::initialize(SDL_Window* window, - int videoFormat, - int width, - int height) -{ - int err; - - if (videoFormat & VIDEO_FORMAT_MASK_H264) { - // Prior to 10.13, we'll just assume everything has - // H.264 support and fail open to allow VT decode. -#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 - if (__builtin_available(macOS 10.13, *)) { - if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) { - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "No HW accelerated H.264 decode via VT"); - return false; - } - } - else -#endif - { - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "Assuming H.264 HW decode on < macOS 10.13"); - } - } - else if (videoFormat & VIDEO_FORMAT_MASK_H265) { -#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 - if (__builtin_available(macOS 10.13, *)) { - if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) { - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "No HW accelerated HEVC decode via VT"); - return false; - } - } - else -#endif - { - // Fail closed for HEVC if we're not on 10.13+ - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "No HEVC support on < macOS 10.13"); - return false; - } - } - - err = av_hwdevice_ctx_create(&m_HwContext, - AV_HWDEVICE_TYPE_VIDEOTOOLBOX, - nullptr, - nullptr, - 0); - if (err < 0) { - SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, - "av_hwdevice_ctx_create() failed for VT decoder: %d", - err); - return false; - } - - return false; // true to test VT -} - -void VTRenderer::renderFrame(AVFrame* frame) -{ - CVPixelBufferRef pixBuf = reinterpret_cast(frame->data[3]); -} diff --git a/app/streaming/renderers/vt.h b/app/streaming/renderers/vt.h index 4d845a84..b7cf1dd6 100644 --- a/app/streaming/renderers/vt.h +++ b/app/streaming/renderers/vt.h @@ -2,20 +2,10 @@ #include "renderer.h" -#import - -class VTRenderer : public IRenderer -{ +// A factory is required to avoid pulling in +// incompatible Objective-C headers. +class VTRendererFactory { public: - VTRenderer(); - virtual ~VTRenderer(); - virtual bool initialize(SDL_Window* window, - int videoFormat, - int width, - int height); - virtual bool prepareDecoderContext(AVCodecContext* context); - virtual void renderFrame(AVFrame* frame); - -private: - AVBufferRef* m_HwContext; + static + IRenderer* createRenderer(); }; diff --git a/app/streaming/renderers/vt.mm b/app/streaming/renderers/vt.mm new file mode 100644 index 00000000..1e204b15 --- /dev/null +++ b/app/streaming/renderers/vt.mm @@ -0,0 +1,199 @@ +// Nasty hack to avoid conflict between AVFoundation and +// libavutil both defining AVMediaType +#define AVMediaType AVMediaType_FFmpeg +#include "vt.h" +#undef AVMediaType + +#include +#include + +#import +#import +#import + +class VTRenderer : public IRenderer +{ +public: + VTRenderer() + : m_HwContext(nullptr), + m_DisplayLayer(nullptr), + m_FormatDesc(nullptr), + m_View(nullptr) + { + + } + + virtual ~VTRenderer() + { + if (m_HwContext != nullptr) { + av_buffer_unref(&m_HwContext); + } + + if (m_FormatDesc != nullptr) { + CFRelease(m_FormatDesc); + } + } + + + virtual bool initialize(SDL_Window* window, + int videoFormat, + int, + int) override + { + int err; + + if (videoFormat & VIDEO_FORMAT_MASK_H264) { + // Prior to 10.13, we'll just assume everything has + // H.264 support and fail open to allow VT decode. + #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 + if (__builtin_available(macOS 10.13, *)) { + if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "No HW accelerated H.264 decode via VT"); + return false; + } + } + else + #endif + { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Assuming H.264 HW decode on < macOS 10.13"); + } + } + else if (videoFormat & VIDEO_FORMAT_MASK_H265) { + #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 + if (__builtin_available(macOS 10.13, *)) { + if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "No HW accelerated HEVC decode via VT"); + return false; + } + } + else + #endif + { + // Fail closed for HEVC if we're not on 10.13+ + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "No HEVC support on < macOS 10.13"); + return false; + } + } + + SDL_SysWMinfo info; + + SDL_VERSION(&info.version); + + if (!SDL_GetWindowWMInfo(window, &info)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "SDL_GetWindowWMInfo() failed: %s", + SDL_GetError()); + return false; + } + + SDL_assert(info.subsystem == SDL_SYSWM_COCOA); + + // SDL adds its own content view to listen for events. + // We need to add a subview for our display layer. + NSView* contentView = info.info.cocoa.window.contentView; + m_View = [[NSView alloc] initWithFrame:contentView.bounds]; + m_View.wantsLayer = YES; + [contentView addSubview: m_View]; + + setupDisplayLayer(); + + err = av_hwdevice_ctx_create(&m_HwContext, + AV_HWDEVICE_TYPE_VIDEOTOOLBOX, + nullptr, + nullptr, + 0); + if (err < 0) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "av_hwdevice_ctx_create() failed for VT decoder: %d", + err); + return false; + } + + return true; + } + + virtual bool prepareDecoderContext(AVCodecContext* context) override + { + context->hw_device_ctx = av_buffer_ref(m_HwContext); + return true; + } + + virtual void renderFrame(AVFrame* frame) override + { + CVPixelBufferRef pixBuf = reinterpret_cast(frame->data[3]); + OSStatus status; + + if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Resetting failed AVSampleBufferDisplay layer"); + setupDisplayLayer(); + } + + // If the format has changed or doesn't exist yet, construct it with the + // pixel buffer data + if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) { + status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, + pixBuf, &m_FormatDesc); + if (status != noErr) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CMVideoFormatDescriptionCreateForImageBuffer() failed: %d", + status); + return; + } + } + + CMSampleBufferRef sampleBuffer; + status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, + pixBuf, + m_FormatDesc, + &kCMTimingInfoInvalid, + &sampleBuffer); + if (status != noErr) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "CMSampleBufferCreateReadyWithImageBuffer() failed: %d", + status); + return; + } + + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true); + CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); + + CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); + + [m_DisplayLayer enqueueSampleBuffer:sampleBuffer]; + + CFRelease(sampleBuffer); + } + +private: + void setupDisplayLayer() + { + CALayer* oldLayer = m_DisplayLayer; + + m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init]; + m_DisplayLayer.bounds = m_View.bounds; + m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_View.bounds), CGRectGetMidY(m_View.bounds)); + m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect; + + CALayer* viewLayer = m_View.layer; + if (oldLayer != nil) { + [viewLayer replaceSublayer:oldLayer with:m_DisplayLayer]; + } + else { + [viewLayer addSublayer:m_DisplayLayer]; + } + } + + AVBufferRef* m_HwContext; + AVSampleBufferDisplayLayer* m_DisplayLayer; + CMVideoFormatDescriptionRef m_FormatDesc; + NSView* m_View; +}; + +IRenderer* VTRendererFactory::createRenderer() { + return new VTRenderer(); +} diff --git a/app/streaming/video.cpp b/app/streaming/video.cpp index 047f4d12..f77c3074 100644 --- a/app/streaming/video.cpp +++ b/app/streaming/video.cpp @@ -87,7 +87,7 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds, #endif #ifdef __APPLE__ case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: - newRenderer = new VTRenderer(); + newRenderer = VTRendererFactory::createRenderer(); break; #endif default: