moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2025-07-04 00:36:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement VideoToolbox rendering

Browse Source
This commit is contained in:
Cameron Gutman 2018-07-15 18:11:57 -07:00
parent 3a4da50bbb
commit a6ac893e1e
5 changed files with 207 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/app.pro
View File

@ -41,7 +41,7 @@ win32 {
} }
macx { macx {
LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56 LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56
LIBS += -framework VideoToolbox LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreGraphics -framework CoreMedia -framework AppKit
} }
SOURCES += \ SOURCES += \
@ -64,7 +64,7 @@ win32 {
SOURCES += streaming/renderers/dxva2.cpp SOURCES += streaming/renderers/dxva2.cpp
} }
macx { macx {
SOURCES += streaming/renderers/vt.cpp SOURCES += streaming/renderers/vt.mm
} }
HEADERS += \ HEADERS += \

86
app/streaming/renderers/vt.cpp
View File

@ -1,86 +0,0 @@
#include "vt.h"
#include <Limelight.h>
VTRenderer::VTRenderer()
: m_HwContext(nullptr)
{
}
VTRenderer::~VTRenderer()
{
if (m_HwContext != nullptr) {
av_buffer_unref(&m_HwContext);
}
}
bool VTRenderer::prepareDecoderContext(AVCodecContext* context)
{
context->hw_device_ctx = av_buffer_ref(m_HwContext);
return true;
}
bool VTRenderer::initialize(SDL_Window* window,
int videoFormat,
int width,
int height)
{
int err;
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
// Prior to 10.13, we'll just assume everything has
// H.264 support and fail open to allow VT decode.
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated H.264 decode via VT");
return false;
}
}
else
#endif
{
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming H.264 HW decode on < macOS 10.13");
}
}
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated HEVC decode via VT");
return false;
}
}
else
#endif
{
// Fail closed for HEVC if we're not on 10.13+
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HEVC support on < macOS 10.13");
return false;
}
}
err = av_hwdevice_ctx_create(&m_HwContext,
AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
nullptr,
nullptr,
0);
if (err < 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"av_hwdevice_ctx_create() failed for VT decoder: %d",
err);
return false;
}
return false; // true to test VT
}
void VTRenderer::renderFrame(AVFrame* frame)
{
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
}

20
app/streaming/renderers/vt.h
View File

@ -2,20 +2,10 @@
#include "renderer.h" #include "renderer.h"
#import <VideoToolbox/VideoToolbox.h> // A factory is required to avoid pulling in
// incompatible Objective-C headers.
class VTRenderer : public IRenderer class VTRendererFactory {
{
public: public:
VTRenderer(); static
virtual ~VTRenderer(); IRenderer* createRenderer();
virtual bool initialize(SDL_Window* window,
int videoFormat,
int width,
int height);
virtual bool prepareDecoderContext(AVCodecContext* context);
virtual void renderFrame(AVFrame* frame);
private:
AVBufferRef* m_HwContext;
}; };

199
app/streaming/renderers/vt.mm Normal file
View File

@ -0,0 +1,199 @@
// Nasty hack to avoid conflict between AVFoundation and
// libavutil both defining AVMediaType
#define AVMediaType AVMediaType_FFmpeg
#include "vt.h"
#undef AVMediaType
#include <SDL_syswm.h>
#include <Limelight.h>
#import <Cocoa/Cocoa.h>
#import <VideoToolbox/VideoToolbox.h>
#import <AVFoundation/AVFoundation.h>
class VTRenderer : public IRenderer
{
public:
VTRenderer()
: m_HwContext(nullptr),
m_DisplayLayer(nullptr),
m_FormatDesc(nullptr),
m_View(nullptr)
{
}
virtual ~VTRenderer()
{
if (m_HwContext != nullptr) {
av_buffer_unref(&m_HwContext);
}
if (m_FormatDesc != nullptr) {
CFRelease(m_FormatDesc);
}
}
virtual bool initialize(SDL_Window* window,
int videoFormat,
int,
int) override
{
int err;
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
// Prior to 10.13, we'll just assume everything has
// H.264 support and fail open to allow VT decode.
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated H.264 decode via VT");
return false;
}
}
else
#endif
{
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming H.264 HW decode on < macOS 10.13");
}
}
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated HEVC decode via VT");
return false;
}
}
else
#endif
{
// Fail closed for HEVC if we're not on 10.13+
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HEVC support on < macOS 10.13");
return false;
}
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(window, &info)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"SDL_GetWindowWMInfo() failed: %s",
SDL_GetError());
return false;
}
SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
// SDL adds its own content view to listen for events.
// We need to add a subview for our display layer.
NSView* contentView = info.info.cocoa.window.contentView;
m_View = [[NSView alloc] initWithFrame:contentView.bounds];
m_View.wantsLayer = YES;
[contentView addSubview: m_View];
setupDisplayLayer();
err = av_hwdevice_ctx_create(&m_HwContext,
AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
nullptr,
nullptr,
0);
if (err < 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"av_hwdevice_ctx_create() failed for VT decoder: %d",
err);
return false;
}
return true;
}
virtual bool prepareDecoderContext(AVCodecContext* context) override
{
context->hw_device_ctx = av_buffer_ref(m_HwContext);
return true;
}
virtual void renderFrame(AVFrame* frame) override
{
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
OSStatus status;
if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Resetting failed AVSampleBufferDisplay layer");
setupDisplayLayer();
}
// If the format has changed or doesn't exist yet, construct it with the
// pixel buffer data
if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) {
status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault,
pixBuf, &m_FormatDesc);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMVideoFormatDescriptionCreateForImageBuffer() failed: %d",
status);
return;
}
}
CMSampleBufferRef sampleBuffer;
status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault,
pixBuf,
m_FormatDesc,
&kCMTimingInfoInvalid,
&sampleBuffer);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMSampleBufferCreateReadyWithImageBuffer() failed: %d",
status);
return;
}
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
[m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
private:
void setupDisplayLayer()
{
CALayer* oldLayer = m_DisplayLayer;
m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init];
m_DisplayLayer.bounds = m_View.bounds;
m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_View.bounds), CGRectGetMidY(m_View.bounds));
m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
CALayer* viewLayer = m_View.layer;
if (oldLayer != nil) {
[viewLayer replaceSublayer:oldLayer with:m_DisplayLayer];
}
else {
[viewLayer addSublayer:m_DisplayLayer];
}
}
AVBufferRef* m_HwContext;
AVSampleBufferDisplayLayer* m_DisplayLayer;
CMVideoFormatDescriptionRef m_FormatDesc;
NSView* m_View;
};
IRenderer* VTRendererFactory::createRenderer() {
return new VTRenderer();
}

2
app/streaming/video.cpp
View File

@ -87,7 +87,7 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
#endif #endif
#ifdef __APPLE__ #ifdef __APPLE__
case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
newRenderer = new VTRenderer(); newRenderer = VTRendererFactory::createRenderer();
break; break;
#endif #endif
default: default: