mirror of
https://github.com/moonlight-stream/moonlight-qt.git
synced 2025-07-04 00:36:36 +00:00
Implement VideoToolbox rendering
This commit is contained in:
parent
3a4da50bbb
commit
a6ac893e1e
@ -41,7 +41,7 @@ win32 {
|
||||
}
|
||||
macx {
|
||||
LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56
|
||||
LIBS += -framework VideoToolbox
|
||||
LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreGraphics -framework CoreMedia -framework AppKit
|
||||
}
|
||||
|
||||
SOURCES += \
|
||||
@ -64,7 +64,7 @@ win32 {
|
||||
SOURCES += streaming/renderers/dxva2.cpp
|
||||
}
|
||||
macx {
|
||||
SOURCES += streaming/renderers/vt.cpp
|
||||
SOURCES += streaming/renderers/vt.mm
|
||||
}
|
||||
|
||||
HEADERS += \
|
||||
|
@ -1,86 +0,0 @@
|
||||
#include "vt.h"
|
||||
|
||||
#include <Limelight.h>
|
||||
|
||||
VTRenderer::VTRenderer()
|
||||
: m_HwContext(nullptr)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
VTRenderer::~VTRenderer()
|
||||
{
|
||||
if (m_HwContext != nullptr) {
|
||||
av_buffer_unref(&m_HwContext);
|
||||
}
|
||||
}
|
||||
|
||||
bool VTRenderer::prepareDecoderContext(AVCodecContext* context)
|
||||
{
|
||||
context->hw_device_ctx = av_buffer_ref(m_HwContext);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool VTRenderer::initialize(SDL_Window* window,
|
||||
int videoFormat,
|
||||
int width,
|
||||
int height)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
|
||||
// Prior to 10.13, we'll just assume everything has
|
||||
// H.264 support and fail open to allow VT decode.
|
||||
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
|
||||
if (__builtin_available(macOS 10.13, *)) {
|
||||
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HW accelerated H.264 decode via VT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"Assuming H.264 HW decode on < macOS 10.13");
|
||||
}
|
||||
}
|
||||
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
|
||||
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
|
||||
if (__builtin_available(macOS 10.13, *)) {
|
||||
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HW accelerated HEVC decode via VT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
// Fail closed for HEVC if we're not on 10.13+
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HEVC support on < macOS 10.13");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
err = av_hwdevice_ctx_create(&m_HwContext,
|
||||
AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
|
||||
nullptr,
|
||||
nullptr,
|
||||
0);
|
||||
if (err < 0) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"av_hwdevice_ctx_create() failed for VT decoder: %d",
|
||||
err);
|
||||
return false;
|
||||
}
|
||||
|
||||
return false; // true to test VT
|
||||
}
|
||||
|
||||
void VTRenderer::renderFrame(AVFrame* frame)
|
||||
{
|
||||
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
|
||||
}
|
@ -2,20 +2,10 @@
|
||||
|
||||
#include "renderer.h"
|
||||
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
|
||||
class VTRenderer : public IRenderer
|
||||
{
|
||||
// A factory is required to avoid pulling in
|
||||
// incompatible Objective-C headers.
|
||||
class VTRendererFactory {
|
||||
public:
|
||||
VTRenderer();
|
||||
virtual ~VTRenderer();
|
||||
virtual bool initialize(SDL_Window* window,
|
||||
int videoFormat,
|
||||
int width,
|
||||
int height);
|
||||
virtual bool prepareDecoderContext(AVCodecContext* context);
|
||||
virtual void renderFrame(AVFrame* frame);
|
||||
|
||||
private:
|
||||
AVBufferRef* m_HwContext;
|
||||
static
|
||||
IRenderer* createRenderer();
|
||||
};
|
||||
|
199
app/streaming/renderers/vt.mm
Normal file
199
app/streaming/renderers/vt.mm
Normal file
@ -0,0 +1,199 @@
|
||||
// Nasty hack to avoid conflict between AVFoundation and
|
||||
// libavutil both defining AVMediaType
|
||||
#define AVMediaType AVMediaType_FFmpeg
|
||||
#include "vt.h"
|
||||
#undef AVMediaType
|
||||
|
||||
#include <SDL_syswm.h>
|
||||
#include <Limelight.h>
|
||||
|
||||
#import <Cocoa/Cocoa.h>
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
class VTRenderer : public IRenderer
|
||||
{
|
||||
public:
|
||||
VTRenderer()
|
||||
: m_HwContext(nullptr),
|
||||
m_DisplayLayer(nullptr),
|
||||
m_FormatDesc(nullptr),
|
||||
m_View(nullptr)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
virtual ~VTRenderer()
|
||||
{
|
||||
if (m_HwContext != nullptr) {
|
||||
av_buffer_unref(&m_HwContext);
|
||||
}
|
||||
|
||||
if (m_FormatDesc != nullptr) {
|
||||
CFRelease(m_FormatDesc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
virtual bool initialize(SDL_Window* window,
|
||||
int videoFormat,
|
||||
int,
|
||||
int) override
|
||||
{
|
||||
int err;
|
||||
|
||||
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
|
||||
// Prior to 10.13, we'll just assume everything has
|
||||
// H.264 support and fail open to allow VT decode.
|
||||
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
|
||||
if (__builtin_available(macOS 10.13, *)) {
|
||||
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HW accelerated H.264 decode via VT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"Assuming H.264 HW decode on < macOS 10.13");
|
||||
}
|
||||
}
|
||||
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
|
||||
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
|
||||
if (__builtin_available(macOS 10.13, *)) {
|
||||
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HW accelerated HEVC decode via VT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
// Fail closed for HEVC if we're not on 10.13+
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"No HEVC support on < macOS 10.13");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
SDL_SysWMinfo info;
|
||||
|
||||
SDL_VERSION(&info.version);
|
||||
|
||||
if (!SDL_GetWindowWMInfo(window, &info)) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"SDL_GetWindowWMInfo() failed: %s",
|
||||
SDL_GetError());
|
||||
return false;
|
||||
}
|
||||
|
||||
SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
|
||||
|
||||
// SDL adds its own content view to listen for events.
|
||||
// We need to add a subview for our display layer.
|
||||
NSView* contentView = info.info.cocoa.window.contentView;
|
||||
m_View = [[NSView alloc] initWithFrame:contentView.bounds];
|
||||
m_View.wantsLayer = YES;
|
||||
[contentView addSubview: m_View];
|
||||
|
||||
setupDisplayLayer();
|
||||
|
||||
err = av_hwdevice_ctx_create(&m_HwContext,
|
||||
AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
|
||||
nullptr,
|
||||
nullptr,
|
||||
0);
|
||||
if (err < 0) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"av_hwdevice_ctx_create() failed for VT decoder: %d",
|
||||
err);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
virtual bool prepareDecoderContext(AVCodecContext* context) override
|
||||
{
|
||||
context->hw_device_ctx = av_buffer_ref(m_HwContext);
|
||||
return true;
|
||||
}
|
||||
|
||||
virtual void renderFrame(AVFrame* frame) override
|
||||
{
|
||||
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
|
||||
OSStatus status;
|
||||
|
||||
if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
|
||||
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"Resetting failed AVSampleBufferDisplay layer");
|
||||
setupDisplayLayer();
|
||||
}
|
||||
|
||||
// If the format has changed or doesn't exist yet, construct it with the
|
||||
// pixel buffer data
|
||||
if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) {
|
||||
status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault,
|
||||
pixBuf, &m_FormatDesc);
|
||||
if (status != noErr) {
|
||||
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"CMVideoFormatDescriptionCreateForImageBuffer() failed: %d",
|
||||
status);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
CMSampleBufferRef sampleBuffer;
|
||||
status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault,
|
||||
pixBuf,
|
||||
m_FormatDesc,
|
||||
&kCMTimingInfoInvalid,
|
||||
&sampleBuffer);
|
||||
if (status != noErr) {
|
||||
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
|
||||
"CMSampleBufferCreateReadyWithImageBuffer() failed: %d",
|
||||
status);
|
||||
return;
|
||||
}
|
||||
|
||||
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
|
||||
CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
|
||||
|
||||
CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
|
||||
|
||||
[m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
|
||||
|
||||
CFRelease(sampleBuffer);
|
||||
}
|
||||
|
||||
private:
|
||||
void setupDisplayLayer()
|
||||
{
|
||||
CALayer* oldLayer = m_DisplayLayer;
|
||||
|
||||
m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init];
|
||||
m_DisplayLayer.bounds = m_View.bounds;
|
||||
m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_View.bounds), CGRectGetMidY(m_View.bounds));
|
||||
m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
|
||||
|
||||
CALayer* viewLayer = m_View.layer;
|
||||
if (oldLayer != nil) {
|
||||
[viewLayer replaceSublayer:oldLayer with:m_DisplayLayer];
|
||||
}
|
||||
else {
|
||||
[viewLayer addSublayer:m_DisplayLayer];
|
||||
}
|
||||
}
|
||||
|
||||
AVBufferRef* m_HwContext;
|
||||
AVSampleBufferDisplayLayer* m_DisplayLayer;
|
||||
CMVideoFormatDescriptionRef m_FormatDesc;
|
||||
NSView* m_View;
|
||||
};
|
||||
|
||||
IRenderer* VTRendererFactory::createRenderer() {
|
||||
return new VTRenderer();
|
||||
}
|
@ -87,7 +87,7 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
|
||||
#endif
|
||||
#ifdef __APPLE__
|
||||
case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
|
||||
newRenderer = new VTRenderer();
|
||||
newRenderer = VTRendererFactory::createRenderer();
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
|
Loading…
x
Reference in New Issue
Block a user