Separate FFmpeg decoder from the Session class (#4)

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -0,0 +1,452 @@
+#include <Initguid.h>
+#include "dxva2.h"
+
+#include <Limelight.h>
+
+DEFINE_GUID(DXVADDI_Intel_ModeH264_E, 0x604F8E68,0x4951,0x4C54,0x88,0xFE,0xAB,0xD2,0x5C,0x15,0xB3,0xD6);
+
+#define SAFE_COM_RELEASE(x) if (x) { (x)->Release(); }
+
+DXVA2Renderer::DXVA2Renderer() :
+    m_FrameIndex(0),
+    m_SurfacesUsed(0),
+    m_SdlRenderer(nullptr),
+    m_DecService(nullptr),
+    m_Decoder(nullptr),
+    m_Pool(nullptr),
+    m_Device(nullptr),
+    m_RenderTarget(nullptr),
+    m_ProcService(nullptr),
+    m_Processor(nullptr)
+{
+    RtlZeroMemory(m_DecSurfaces, sizeof(m_DecSurfaces));
+    RtlZeroMemory(&m_DXVAContext, sizeof(m_DXVAContext));
+}
+
+DXVA2Renderer::~DXVA2Renderer()
+{
+    SAFE_COM_RELEASE(m_DecService);
+    SAFE_COM_RELEASE(m_Decoder);
+    SAFE_COM_RELEASE(m_Device);
+    SAFE_COM_RELEASE(m_RenderTarget);
+    SAFE_COM_RELEASE(m_ProcService);
+    SAFE_COM_RELEASE(m_Processor);
+
+    for (int i = 0; i < ARRAYSIZE(m_DecSurfaces); i++) {
+        SAFE_COM_RELEASE(m_DecSurfaces[i]);
+    }
+
+    if (m_Pool != nullptr) {
+        av_buffer_pool_uninit(&m_Pool);
+    }
+
+    if (m_SdlRenderer != nullptr) {
+        SDL_DestroyRenderer(m_SdlRenderer);
+    }
+}
+
+void DXVA2Renderer::ffPoolDummyDelete(void*, uint8_t*)
+{
+    /* Do nothing */
+}
+
+AVBufferRef* DXVA2Renderer::ffPoolAlloc(void* opaque, int)
+{
+    DXVA2Renderer* me = reinterpret_cast<DXVA2Renderer*>(opaque);
+
+    if (me->m_SurfacesUsed < ARRAYSIZE(me->m_DecSurfaces)) {
+        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                    "DXVA2 decoder surface high-water mark: %d",
+                    me->m_SurfacesUsed);
+        return av_buffer_create((uint8_t*)me->m_DecSurfaces[me->m_SurfacesUsed++],
+                                sizeof(*me->m_DecSurfaces), ffPoolDummyDelete, 0, 0);
+    }
+
+    return NULL;
+}
+
+bool DXVA2Renderer::prepareDecoderContext(AVCodecContext* context)
+{
+    // m_DXVAContext.workaround and report_id already initialized elsewhere
+    m_DXVAContext.decoder = m_Decoder;
+    m_DXVAContext.cfg = &m_Config;
+    m_DXVAContext.surface = m_DecSurfaces;
+    m_DXVAContext.surface_count = ARRAYSIZE(m_DecSurfaces);
+
+    context->hwaccel_context = &m_DXVAContext;
+
+    context->get_format = ffGetFormat;
+    context->get_buffer2 = ffGetBuffer2;
+
+    context->opaque = this;
+
+    m_Pool = av_buffer_pool_init2(ARRAYSIZE(m_DecSurfaces), this, ffPoolAlloc, nullptr);
+    if (!m_Pool) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed create buffer pool");
+        return false;
+    }
+
+    return true;
+}
+
+int DXVA2Renderer::ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int)
+{
+    DXVA2Renderer* me = reinterpret_cast<DXVA2Renderer*>(context->opaque);
+
+    frame->buf[0] = av_buffer_pool_get(me->m_Pool);
+    if (!frame->buf[0]) {
+        return AVERROR(ENOMEM);
+    }
+
+    frame->data[3] = frame->buf[0]->data;
+    frame->format = AV_PIX_FMT_DXVA2_VLD;
+    frame->width = me->m_Width;
+    frame->height = me->m_Height;
+
+    return 0;
+}
+
+enum AVPixelFormat DXVA2Renderer::ffGetFormat(AVCodecContext*,
+                                              const enum AVPixelFormat* pixFmts)
+{
+    const enum AVPixelFormat *p;
+
+    for (p = pixFmts; *p != -1; p++) {
+        if (*p == AV_PIX_FMT_DXVA2_VLD)
+            return *p;
+    }
+
+    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                 "Failed to find DXVA2 HW surface format");
+    return AV_PIX_FMT_NONE;
+}
+
+bool DXVA2Renderer::initializeDecoder()
+{
+    HRESULT hr;
+
+    hr = DXVA2CreateVideoService(m_Device, IID_IDirectXVideoDecoderService,
+                                 reinterpret_cast<void**>(&m_DecService));
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "DXVA2CreateVideoService(IID_IDirectXVideoDecoderService) failed: %x",
+                     hr);
+        return false;
+    }
+
+    GUID* guids;
+    GUID chosenDeviceGuid;
+    UINT guidCount;
+    hr = m_DecService->GetDecoderDeviceGuids(&guidCount, &guids);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "GetDecoderDeviceGuids() failed: %x",
+                     hr);
+        return false;
+    }
+
+    UINT i;
+    for (i = 0; i < guidCount; i++) {
+        if (m_VideoFormat == VIDEO_FORMAT_H264) {
+            if (IsEqualGUID(guids[i], DXVA2_ModeH264_E) ||
+                    IsEqualGUID(guids[i], DXVA2_ModeH264_F)) {
+                chosenDeviceGuid = guids[i];
+                break;
+            }
+            else if (IsEqualGUID(guids[i], DXVADDI_Intel_ModeH264_E)) {
+                chosenDeviceGuid = guids[i];
+                m_DXVAContext.workaround |= FF_DXVA2_WORKAROUND_INTEL_CLEARVIDEO;
+                break;
+            }
+        }
+        else if (m_VideoFormat == VIDEO_FORMAT_H265) {
+            if (IsEqualGUID(guids[i], DXVA2_ModeHEVC_VLD_Main)) {
+                chosenDeviceGuid = guids[i];
+                break;
+            }
+        }
+        else if (m_VideoFormat == VIDEO_FORMAT_H265_MAIN10) {
+            if (IsEqualGUID(guids[i], DXVA2_ModeHEVC_VLD_Main10)) {
+                chosenDeviceGuid = guids[i];
+                break;
+            }
+        }
+    }
+
+    CoTaskMemFree(guids);
+
+    if (i == guidCount) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "No matching decoder device GUIDs");
+        return false;
+    }
+
+    DXVA2_ConfigPictureDecode* configs;
+    UINT configCount;
+    hr = m_DecService->GetDecoderConfigurations(chosenDeviceGuid, &m_Desc, nullptr, &configCount, &configs);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "GetDecoderConfigurations() failed: %x",
+                     hr);
+        return false;
+    }
+
+    for (i = 0; i < configCount; i++) {
+        if ((configs[i].ConfigBitstreamRaw == 1 || configs[i].ConfigBitstreamRaw == 2) &&
+                IsEqualGUID(configs[i].guidConfigBitstreamEncryption, DXVA2_NoEncrypt)) {
+            m_Config = configs[i];
+            break;
+        }
+    }
+
+    CoTaskMemFree(configs);
+
+    if (i == configCount) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "No matching decoder configurations");
+        return false;
+    }
+
+    int alignment;
+
+    // HEVC using DXVA requires 128B alignment
+    if (m_VideoFormat & VIDEO_FORMAT_MASK_H265) {
+        alignment = 128;
+    }
+    else {
+        alignment = 16;
+    }
+
+    hr = m_DecService->CreateSurface(FFALIGN(m_Width, alignment),
+                                     FFALIGN(m_Height, alignment),
+                                     ARRAYSIZE(m_DecSurfaces) - 1,
+                                     m_Desc.Format,
+                                     D3DPOOL_DEFAULT,
+                                     0,
+                                     DXVA2_VideoDecoderRenderTarget,
+                                     m_DecSurfaces,
+                                     nullptr);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "CreateSurface() failed: %x",
+                     hr);
+        return false;
+    }
+
+    hr = m_DecService->CreateVideoDecoder(chosenDeviceGuid, &m_Desc, &m_Config,
+                                          m_DecSurfaces, ARRAYSIZE(m_DecSurfaces),
+                                          &m_Decoder);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "CreateVideoDecoder() failed: %x",
+                     hr);
+        return false;
+    }
+
+    return true;
+}
+
+bool DXVA2Renderer::initializeRenderer()
+{
+    HRESULT hr;
+
+    hr = m_Device->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &m_RenderTarget);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "GetBackBuffer() failed: %x",
+                     hr);
+        return false;
+    }
+
+    D3DSURFACE_DESC renderTargetDesc;
+    m_RenderTarget->GetDesc(&renderTargetDesc);
+
+    hr = DXVA2CreateVideoService(m_Device, IID_IDirectXVideoProcessorService,
+                                 reinterpret_cast<void**>(&m_ProcService));
+
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "DXVA2CreateVideoService(IID_IDirectXVideoProcessorService) failed: %x",
+                     hr);
+        return false;
+    }
+
+    UINT guidCount;
+    GUID* guids;
+    hr = m_ProcService->GetVideoProcessorDeviceGuids(&m_Desc, &guidCount, &guids);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "GetVideoProcessorDeviceGuids() failed: %x",
+                     hr);
+        return false;
+    }
+
+    UINT i;
+    for (i = 0; i < guidCount; i++) {
+        DXVA2_VideoProcessorCaps caps;
+        hr = m_ProcService->GetVideoProcessorCaps(guids[i], &m_Desc, renderTargetDesc.Format, &caps);
+        if (SUCCEEDED(hr)) {
+            if (!(caps.DeviceCaps & DXVA2_VPDev_HardwareDevice)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "Device %d is not hardware: %x",
+                            i,
+                            caps.DeviceCaps);
+                continue;
+            }
+            else if (!(caps.VideoProcessorOperations & DXVA2_VideoProcess_YUV2RGB) &&
+                     !(caps.VideoProcessorOperations & DXVA2_VideoProcess_YUV2RGBExtended)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "Device %d can't convert YUV2RGB: %x",
+                            i,
+                            caps.VideoProcessorOperations);
+                continue;
+            }
+
+            m_ProcService->GetProcAmpRange(guids[i], &m_Desc, renderTargetDesc.Format, DXVA2_ProcAmp_Brightness, &m_BrightnessRange);
+            m_ProcService->GetProcAmpRange(guids[i], &m_Desc, renderTargetDesc.Format, DXVA2_ProcAmp_Contrast, &m_ContrastRange);
+            m_ProcService->GetProcAmpRange(guids[i], &m_Desc, renderTargetDesc.Format, DXVA2_ProcAmp_Hue, &m_HueRange);
+            m_ProcService->GetProcAmpRange(guids[i], &m_Desc, renderTargetDesc.Format, DXVA2_ProcAmp_Saturation, &m_SaturationRange);
+
+            hr = m_ProcService->CreateVideoProcessor(guids[i], &m_Desc, renderTargetDesc.Format, 0, &m_Processor);
+            if (FAILED(hr)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "CreateVideoProcessor() failed for GUID %d: %x",
+                            i,
+                            hr);
+                continue;
+            }
+
+            break;
+        }
+        else {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "GetVideoProcessorCaps() failed for GUID %d: %x",
+                        i,
+                        hr);
+        }
+    }
+
+    CoTaskMemFree(guids);
+
+    if (i == guidCount) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to find a usable DXVA2 processor");
+        return false;
+    }
+
+    return true;
+}
+
+bool DXVA2Renderer::initialize(SDL_Window* window, int videoFormat, int width, int height)
+{
+    m_VideoFormat = videoFormat;
+    m_Width = width;
+    m_Height = height;
+
+    // FFmpeg will be decoding on different threads than the main thread that we're
+    // currently running on right now. We must set this hint so SDL will pass
+    // D3DCREATE_MULTITHREADED to IDirect3D9::CreateDevice().
+    SDL_SetHint(SDL_HINT_RENDER_DIRECT3D_THREADSAFE, "1");
+
+    // We want VSYNC when running in full-screen. DWM will give us
+    // tear-free video when running composited (windowed) without any
+    // extra latency waiting for VSYNC.
+    m_SdlRenderer = SDL_CreateRenderer(window, -1,
+                                       SDL_RENDERER_ACCELERATED |
+                                       ((SDL_GetWindowFlags(window) & SDL_WINDOW_FULLSCREEN) ?
+                                           SDL_RENDERER_PRESENTVSYNC : 0));
+    if (!m_SdlRenderer) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateRenderer() failed: %s",
+                     SDL_GetError());
+        return false;
+    }
+
+    m_Device = SDL_RenderGetD3D9Device(m_SdlRenderer);
+
+    RtlZeroMemory(&m_Desc, sizeof(m_Desc));
+    m_Desc.SampleWidth = m_Width;
+    m_Desc.SampleHeight = m_Height;
+    m_Desc.SampleFormat.VideoChromaSubsampling = DXVA2_VideoChromaSubsampling_ProgressiveChroma;
+    m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_0_255;
+    m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_BT709;
+    m_Desc.SampleFormat.VideoLighting = DXVA2_VideoLighting_dim;
+    m_Desc.SampleFormat.VideoPrimaries = DXVA2_VideoPrimaries_BT709;
+    m_Desc.SampleFormat.VideoTransferFunction = DXVA2_VideoTransFunc_709;
+    m_Desc.SampleFormat.SampleFormat = DXVA2_SampleProgressiveFrame;
+    m_Desc.Format = (D3DFORMAT)MAKEFOURCC('N','V','1','2');
+
+    if (!initializeDecoder()) {
+        return false;
+    }
+
+    if (!initializeRenderer()) {
+        return false;
+    }
+
+    return true;
+}
+
+void DXVA2Renderer::renderFrame(AVFrame* frame)
+{
+    IDirect3DSurface9* surface = reinterpret_cast<IDirect3DSurface9*>(frame->data[3]);
+    HRESULT hr;
+
+    hr = m_Device->TestCooperativeLevel();
+    switch (hr) {
+    case D3D_OK:
+        break;
+    case D3DERR_DEVICELOST:
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "D3D device lost!");
+        return;
+    case D3DERR_DEVICENOTRESET:
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "D3D device not reset!");
+        return;
+    default:
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unknown D3D error: %x",
+                     hr);
+        return;
+    }
+
+    DXVA2_VideoSample sample = {};
+    sample.Start = m_FrameIndex;
+    sample.End = m_FrameIndex + 1;
+    sample.SrcSurface = surface;
+    sample.SrcRect.right = m_Desc.SampleWidth;
+    sample.SrcRect.bottom = m_Desc.SampleHeight;
+    sample.DstRect = sample.SrcRect;
+    sample.SampleFormat = m_Desc.SampleFormat;
+    sample.PlanarAlpha = DXVA2_Fixed32OpaqueAlpha();
+
+    DXVA2_VideoProcessBltParams bltParams = {};
+
+    bltParams.TargetFrame = m_FrameIndex++;
+    bltParams.TargetRect.right = m_Desc.SampleWidth;
+    bltParams.TargetRect.bottom = m_Desc.SampleHeight;
+    bltParams.BackgroundColor.Y = 0x1000;
+    bltParams.BackgroundColor.Cb = 0x8000;
+    bltParams.BackgroundColor.Cr = 0x8000;
+    bltParams.BackgroundColor.Alpha = 0xFFFF;
+
+    bltParams.DestFormat.SampleFormat = DXVA2_SampleProgressiveFrame;
+
+    bltParams.ProcAmpValues.Brightness = m_BrightnessRange.DefaultValue;
+    bltParams.ProcAmpValues.Contrast = m_ContrastRange.DefaultValue;
+    bltParams.ProcAmpValues.Hue = m_HueRange.DefaultValue;
+    bltParams.ProcAmpValues.Saturation = m_SaturationRange.DefaultValue;
+
+    bltParams.Alpha = DXVA2_Fixed32OpaqueAlpha();
+
+    hr = m_Processor->VideoProcessBlt(m_RenderTarget, &bltParams, &sample, 1, nullptr);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VideoProcessBlt() failed: %x",
+                     hr);
+    }
+
+    m_Device->Present(nullptr, nullptr, nullptr, nullptr);
+}

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "renderer.h"
+
+#include <d3d9.h>
+#include <dxva2api.h>
+
+extern "C" {
+#include <libavcodec/dxva2.h>
+}
+
+class DXVA2Renderer : public IFFmpegRenderer
+{
+public:
+    DXVA2Renderer();
+    virtual ~DXVA2Renderer();
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int width,
+                            int height);
+    virtual bool prepareDecoderContext(AVCodecContext* context);
+    virtual void renderFrame(AVFrame* frame);
+
+private:
+    bool initializeDecoder();
+    bool initializeRenderer();
+
+    static
+    AVBufferRef* ffPoolAlloc(void* opaque, int size);
+
+    static
+    void ffPoolDummyDelete(void*, uint8_t*);
+
+    static
+    int ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int flags);
+
+    static
+    enum AVPixelFormat ffGetFormat(AVCodecContext*,
+                                   const enum AVPixelFormat* pixFmts);
+
+    int m_VideoFormat;
+    int m_Width;
+    int m_Height;
+
+    SDL_Renderer* m_SdlRenderer;
+
+    struct dxva_context m_DXVAContext;
+    IDirect3DSurface9* m_DecSurfaces[19];
+    DXVA2_ConfigPictureDecode m_Config;
+    IDirectXVideoDecoderService* m_DecService;
+    IDirectXVideoDecoder* m_Decoder;
+    int m_SurfacesUsed;
+    AVBufferPool* m_Pool;
+
+    IDirect3DDevice9* m_Device;
+    IDirect3DSurface9* m_RenderTarget;
+    IDirectXVideoProcessorService* m_ProcService;
+    IDirectXVideoProcessor* m_Processor;
+    DXVA2_ValueRange m_BrightnessRange;
+    DXVA2_ValueRange m_ContrastRange;
+    DXVA2_ValueRange m_HueRange;
+    DXVA2_ValueRange m_SaturationRange;
+    DXVA2_VideoDesc m_Desc;
+    REFERENCE_TIME m_FrameIndex;
+};

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <SDL.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+}
+
+class IFFmpegRenderer {
+public:
+    virtual ~IFFmpegRenderer() {}
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int width,
+                            int height) = 0;
+    virtual bool prepareDecoderContext(AVCodecContext* context) = 0;
+    virtual void renderFrame(AVFrame* frame) = 0;
+};
+
+class SdlRenderer : public IFFmpegRenderer {
+public:
+    SdlRenderer();
+    virtual ~SdlRenderer();
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int width,
+                            int height);
+    virtual bool prepareDecoderContext(AVCodecContext* context);
+    virtual void renderFrame(AVFrame* frame);
+
+private:
+    SDL_Renderer* m_Renderer;
+    SDL_Texture* m_Texture;
+};

app/streaming/video/ffmpeg-renderers/sdl.cpp

@@ -0,0 +1,71 @@
+#include "renderer.h"
+
+SdlRenderer::SdlRenderer()
+    : m_Renderer(nullptr),
+      m_Texture(nullptr)
+{
+
+}
+
+SdlRenderer::~SdlRenderer()
+{
+    if (m_Texture != nullptr) {
+        SDL_DestroyTexture(m_Texture);
+    }
+
+    if (m_Renderer != nullptr) {
+        SDL_DestroyRenderer(m_Renderer);
+    }
+}
+
+bool SdlRenderer::prepareDecoderContext(AVCodecContext*)
+{
+    /* Nothing to do */
+    return true;
+}
+
+bool SdlRenderer::initialize(SDL_Window* window,
+                             int,
+                             int width,
+                             int height)
+{
+    m_Renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
+    if (!m_Renderer) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateRenderer() failed: %s",
+                     SDL_GetError());
+        return false;
+    }
+
+    // The window may be smaller than the stream size, so ensure our
+    // logical rendering surface size is equal to the stream size
+    SDL_RenderSetLogicalSize(m_Renderer, width, height);
+
+    m_Texture = SDL_CreateTexture(m_Renderer,
+                                  SDL_PIXELFORMAT_YV12,
+                                  SDL_TEXTUREACCESS_STREAMING,
+                                  width,
+                                  height);
+    if (!m_Texture) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateRenderer() failed: %s",
+                     SDL_GetError());
+        return false;
+    }
+
+    return true;
+}
+
+void SdlRenderer::renderFrame(AVFrame* frame)
+{
+    SDL_UpdateYUVTexture(m_Texture, nullptr,
+                         frame->data[0],
+                         frame->linesize[0],
+                         frame->data[1],
+                         frame->linesize[1],
+                         frame->data[2],
+                         frame->linesize[2]);
+    SDL_RenderClear(m_Renderer);
+    SDL_RenderCopy(m_Renderer, m_Texture, nullptr, nullptr);
+    SDL_RenderPresent(m_Renderer);
+}

app/streaming/video/ffmpeg-renderers/vt.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "renderer.h"
+
+// A factory is required to avoid pulling in
+// incompatible Objective-C headers.
+class VTRendererFactory {
+public:
+    static
+    IFFmpegRenderer* createRenderer();
+};

app/streaming/video/ffmpeg-renderers/vt.mm

@@ -0,0 +1,199 @@
+// Nasty hack to avoid conflict between AVFoundation and
+// libavutil both defining AVMediaType
+#define AVMediaType AVMediaType_FFmpeg
+#include "vt.h"
+#undef AVMediaType
+
+#include <SDL_syswm.h>
+#include <Limelight.h>
+
+#import <Cocoa/Cocoa.h>
+#import <VideoToolbox/VideoToolbox.h>
+#import <AVFoundation/AVFoundation.h>
+
+class VTRenderer : public IFFmpegRenderer
+{
+public:
+    VTRenderer()
+        : m_HwContext(nullptr),
+          m_DisplayLayer(nullptr),
+          m_FormatDesc(nullptr),
+          m_View(nullptr)
+    {
+
+    }
+
+    virtual ~VTRenderer()
+    {
+        if (m_HwContext != nullptr) {
+            av_buffer_unref(&m_HwContext);
+        }
+
+        if (m_FormatDesc != nullptr) {
+            CFRelease(m_FormatDesc);
+        }
+    }
+
+
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int,
+                            int) override
+    {
+        int err;
+
+        if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+            // Prior to 10.13, we'll just assume everything has
+            // H.264 support and fail open to allow VT decode.
+    #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
+            if (__builtin_available(macOS 10.13, *)) {
+                if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
+                    SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                                "No HW accelerated H.264 decode via VT");
+                    return false;
+                }
+            }
+            else
+    #endif
+            {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "Assuming H.264 HW decode on < macOS 10.13");
+            }
+        }
+        else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
+    #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
+            if (__builtin_available(macOS 10.13, *)) {
+                if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
+                    SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                                "No HW accelerated HEVC decode via VT");
+                    return false;
+                }
+            }
+            else
+    #endif
+            {
+                // Fail closed for HEVC if we're not on 10.13+
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "No HEVC support on < macOS 10.13");
+                return false;
+            }
+        }
+
+        SDL_SysWMinfo info;
+
+        SDL_VERSION(&info.version);
+
+        if (!SDL_GetWindowWMInfo(window, &info)) {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "SDL_GetWindowWMInfo() failed: %s",
+                        SDL_GetError());
+            return false;
+        }
+
+        SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
+
+        // SDL adds its own content view to listen for events.
+        // We need to add a subview for our display layer.
+        NSView* contentView = info.info.cocoa.window.contentView;
+        m_View = [[NSView alloc] initWithFrame:contentView.bounds];
+        m_View.wantsLayer = YES;
+        [contentView addSubview: m_View];
+
+        setupDisplayLayer();
+
+        err = av_hwdevice_ctx_create(&m_HwContext,
+                                     AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
+                                     nullptr,
+                                     nullptr,
+                                     0);
+        if (err < 0) {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "av_hwdevice_ctx_create() failed for VT decoder: %d",
+                        err);
+            return false;
+        }
+
+        return true;
+    }
+
+    virtual bool prepareDecoderContext(AVCodecContext* context) override
+    {
+        context->hw_device_ctx = av_buffer_ref(m_HwContext);
+        return true;
+    }
+
+    virtual void renderFrame(AVFrame* frame) override
+    {
+        CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
+        OSStatus status;
+
+        if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Resetting failed AVSampleBufferDisplay layer");
+            setupDisplayLayer();
+        }
+
+        // If the format has changed or doesn't exist yet, construct it with the
+        // pixel buffer data
+        if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) {
+            status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault,
+                                                                  pixBuf, &m_FormatDesc);
+            if (status != noErr) {
+                SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                             "CMVideoFormatDescriptionCreateForImageBuffer() failed: %d",
+                             status);
+                return;
+            }
+        }
+
+        CMSampleBufferRef sampleBuffer;
+        status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault,
+                                                          pixBuf,
+                                                          m_FormatDesc,
+                                                          &kCMTimingInfoInvalid,
+                                                          &sampleBuffer);
+        if (status != noErr) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "CMSampleBufferCreateReadyWithImageBuffer() failed: %d",
+                         status);
+            return;
+        }
+
+        CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
+        CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
+
+        CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
+
+        [m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
+
+        CFRelease(sampleBuffer);
+    }
+
+private:
+    void setupDisplayLayer()
+    {
+        CALayer* oldLayer = m_DisplayLayer;
+
+        m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init];
+        m_DisplayLayer.bounds = m_View.bounds;
+        m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_View.bounds), CGRectGetMidY(m_View.bounds));
+        m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
+
+        CALayer* viewLayer = m_View.layer;
+        if (oldLayer != nil) {
+            [viewLayer replaceSublayer:oldLayer with:m_DisplayLayer];
+        }
+        else {
+            [viewLayer addSublayer:m_DisplayLayer];
+        }
+    }
+
+    AVBufferRef* m_HwContext;
+    AVSampleBufferDisplayLayer* m_DisplayLayer;
+    CMVideoFormatDescriptionRef m_FormatDesc;
+    NSView* m_View;
+};
+
+IFFmpegRenderer* VTRendererFactory::createRenderer() {
+    return new VTRenderer();
+}