From 5787deeee14d92d013e57b67b11df66ba43f3c8b Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Sun, 15 Jul 2018 12:48:17 -0700
Subject: [PATCH] Basic VT decoding without rendering

---
 app/app.pro                    |  1 +
 app/streaming/renderers/vt.cpp | 58 ++++++++++++++++++++++++++++++++--
 app/streaming/renderers/vt.h   |  3 ++
 3 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/app/app.pro b/app/app.pro
index 0e8eed15..1cb65640 100644
--- a/app/app.pro
+++ b/app/app.pro
@@ -41,6 +41,7 @@ win32 {
 }
 macx {
     LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56
+    LIBS += -framework VideoToolbox
 }
 
 SOURCES += \
diff --git a/app/streaming/renderers/vt.cpp b/app/streaming/renderers/vt.cpp
index 9110406d..46c905ab 100644
--- a/app/streaming/renderers/vt.cpp
+++ b/app/streaming/renderers/vt.cpp
@@ -1,17 +1,23 @@
 #include "vt.h"
 
+#include <Limelight.h>
+
 VTRenderer::VTRenderer()
+    : m_HwContext(nullptr)
 {
 
 }
 
 VTRenderer::~VTRenderer()
 {
+    if (m_HwContext != nullptr) {
+        av_buffer_unref(&m_HwContext);
+    }
 }
 
-bool VTRenderer::prepareDecoderContext(AVCodecContext*)
+bool VTRenderer::prepareDecoderContext(AVCodecContext* context)
 {
-    /* Nothing to do */
+    context->hw_device_ctx = av_buffer_ref(m_HwContext);
     return true;
 }
 
@@ -20,9 +26,55 @@ bool VTRenderer::initialize(SDL_Window* window,
                             int width,
                             int height)
 {
-    return false;
+    int err;
+
+    if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+        // Prior to 10.13, we'll just assume everything has
+        // H.264 support and fail open to allow VT decode.
+        if (__builtin_available(macOS 10.13, *)) {
+            if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "No HW accelerated H.264 decode via VT");
+                return false;
+            }
+        }
+        else {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Assuming H.264 HW decode on < macOS 10.13");
+        }
+    }
+    else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
+        if (__builtin_available(macOS 10.13, *)) {
+            if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "No HW accelerated HEVC decode via VT");
+                return false;
+            }
+        }
+        else {
+            // Fail closed for HEVC if we're not on 10.13+
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "No HEVC support on < macOS 10.13");
+            return false;
+        }
+    }
+
+    err = av_hwdevice_ctx_create(&m_HwContext,
+                                 AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
+                                 nullptr,
+                                 nullptr,
+                                 0);
+    if (err < 0) {
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "av_hwdevice_ctx_create() failed for VT decoder: %d",
+                    err);
+        return false;
+    }
+
+    return false; // true to test VT
 }
 
 void VTRenderer::renderFrame(AVFrame* frame)
 {
+    CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
 }
diff --git a/app/streaming/renderers/vt.h b/app/streaming/renderers/vt.h
index 5ee7e952..4d845a84 100644
--- a/app/streaming/renderers/vt.h
+++ b/app/streaming/renderers/vt.h
@@ -2,6 +2,8 @@
 
 #include "renderer.h"
 
+#import <VideoToolbox/VideoToolbox.h>
+
 class VTRenderer : public IRenderer
 {
 public:
@@ -15,4 +17,5 @@ public:
     virtual void renderFrame(AVFrame* frame);
 
 private:
+    AVBufferRef* m_HwContext;
 };