From b7cbec0b1c6a7a491f2ae1dd6c5a889493d664be Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Sat, 25 Aug 2018 13:36:54 -0700
Subject: [PATCH] Pass RFI flags to decoder

---
 app/streaming/session.cpp                    | 44 ++++++++++++++++++++
 app/streaming/session.hpp                    |  4 ++
 app/streaming/video/ffmpeg-renderers/sdl.cpp |  5 ++-
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp
index 8ff6dffa..dbee8173 100644
--- a/app/streaming/session.cpp
+++ b/app/streaming/session.cpp
@@ -232,6 +232,43 @@ bool Session::isHardwareDecodeAvailable(StreamingPreferences::VideoDecoderSelect
     return ret;
 }
 
+int Session::getDecoderCapabilities(StreamingPreferences::VideoDecoderSelection vds,
+                                    int videoFormat, int width, int height, int frameRate)
+{
+    IVideoDecoder* decoder;
+
+    if (SDL_InitSubSystem(SDL_INIT_VIDEO) != 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_InitSubSystem(SDL_INIT_VIDEO) failed: %s",
+                     SDL_GetError());
+        return false;
+    }
+
+    SDL_Window* window = SDL_CreateWindow("", 0, 0, width, height, SDL_WINDOW_HIDDEN);
+    if (!window) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to create window for hardware decode test: %s",
+                     SDL_GetError());
+        SDL_QuitSubSystem(SDL_INIT_VIDEO);
+        return false;
+    }
+
+    if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, true, decoder)) {
+        SDL_DestroyWindow(window);
+        SDL_QuitSubSystem(SDL_INIT_VIDEO);
+        return false;
+    }
+
+    SDL_DestroyWindow(window);
+
+    int caps = decoder->getDecoderCapabilities();
+
+    delete decoder;
+    SDL_QuitSubSystem(SDL_INIT_VIDEO);
+
+    return caps;
+}
+
 Session::Session(NvComputer* computer, NvApp& app)
     : m_Computer(computer),
       m_App(app),
@@ -452,6 +489,13 @@ bool Session::validateLaunch()
         return false;
     }
 
+    // Add the capability flags from the chosen decoder/renderer
+    m_VideoCallbacks.capabilities |= getDecoderCapabilities(m_Preferences.videoDecoderSelection,
+                                                            m_StreamConfig.supportsHevc ? VIDEO_FORMAT_H265 : VIDEO_FORMAT_H264,
+                                                            m_StreamConfig.width,
+                                                            m_StreamConfig.height,
+                                                            m_StreamConfig.fps);
+
     return true;
 }
 
diff --git a/app/streaming/session.hpp b/app/streaming/session.hpp
index 631a3531..2cc623bf 100644
--- a/app/streaming/session.hpp
+++ b/app/streaming/session.hpp
@@ -25,6 +25,10 @@ public:
     bool isHardwareDecodeAvailable(StreamingPreferences::VideoDecoderSelection vds,
                                    int videoFormat, int width, int height, int frameRate);
 
+    static
+    int getDecoderCapabilities(StreamingPreferences::VideoDecoderSelection vds,
+                               int videoFormat, int width, int height, int frameRate);
+
 signals:
     void stageStarting(QString stage);
 
diff --git a/app/streaming/video/ffmpeg-renderers/sdl.cpp b/app/streaming/video/ffmpeg-renderers/sdl.cpp
index 1adcbaf5..6cef3ac0 100644
--- a/app/streaming/video/ffmpeg-renderers/sdl.cpp
+++ b/app/streaming/video/ffmpeg-renderers/sdl.cpp
@@ -38,8 +38,9 @@ bool SdlRenderer::needsTestFrame()
 
 int SdlRenderer::getDecoderCapabilities()
 {
-    // The FFmpeg CPU decoder can handle reference frame invalidation
-    return CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC;
+    // The FFmpeg CPU decoder can handle reference frame invalidation,
+    // but only for H.264.
+    return CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC;
 }
 
 bool SdlRenderer::initialize(SDL_Window* window,