Implemented software video decoding and special key combos

app/gui/AppView.qml

@@ -66,8 +66,16 @@ GridView {
             anchors.fill: parent
             onClicked: {
                 // TODO: Check if a different game is running
-                var session = appModel.createSessionForApp(index);
+                var session = appModel.createSessionForApp(index)
-                session.exec();
+
+                // Don't poll while the stream is running
+                ComputerManager.stopPollingAsync()
+
+                // Run the streaming session to completion
+                session.exec()
+
+                // Start polling again
+                ComputerManager.startPolling()
             }
         }
     }

app/streaming/input.cpp

@@ -1,6 +1,6 @@
 #include <Limelight.h>
 #include <SDL.h>
-#include "input.hpp"
+#include "streaming/session.hpp"
 
 #define VK_0 0x30
 #define VK_A 0x41
@@ -80,22 +80,29 @@ void SdlInputHandler::handleKeyEvent(SDL_KeyboardEvent* event)
         // Check for the unbind combo (Ctrl+Alt+Shift+Z)
         else if (event->keysym.sym == SDLK_z) {
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                        "Detected mouse unbind combo");
+                        "Detected mouse capture toggle combo");
-
+            SDL_SetRelativeMouseMode((SDL_bool)!SDL_GetRelativeMouseMode());
-            // TODO: disable mouse capture then recapture when
-            // we regain focus or on click.
             return;
         }
         // Check for the full-screen combo (Ctrl+Alt+Shift+X)
         else if (event->keysym.sym == SDLK_x) {
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                         "Detected full-screen toggle combo");
-
+            if (SDL_GetWindowFlags(Session::s_ActiveSession->m_Window) & SDL_WINDOW_FULLSCREEN) {
-            // TODO: toggle full-screen
+                SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, 0);
+            }
+            else {
+                SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, SDL_WINDOW_FULLSCREEN);
+            }
             return;
         }
     }
 
+    if (!SDL_GetRelativeMouseMode()) {
+        // Not capturing
+        return;
+    }
+
     // Set modifier flags
     modifiers = 0;
     if (event->keysym.mod & KMOD_CTRL) {
@@ -316,6 +323,18 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event)
 {
     int button;
 
+    // Capture the mouse again if clicked when unbound
+    if (event->button == SDL_BUTTON_LEFT &&
+            event->state == SDL_PRESSED &&
+            !SDL_GetRelativeMouseMode()) {
+        SDL_SetRelativeMouseMode(SDL_TRUE);
+        return;
+    }
+    else if (!SDL_GetRelativeMouseMode()) {
+        // Not capturing
+        return;
+    }
+
     switch (event->button)
     {
         case SDL_BUTTON_LEFT:
@@ -346,6 +365,11 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event)
 
 void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event)
 {
+    if (!SDL_GetRelativeMouseMode()) {
+        // Not capturing
+        return;
+    }
+
     if (event->xrel != 0 || event->yrel != 0) {
         LiSendMouseMoveEvent((unsigned short)event->xrel,
                              (unsigned short)event->yrel);
@@ -354,6 +378,11 @@ void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event)
 
 void SdlInputHandler::handleMouseWheelEvent(SDL_MouseWheelEvent* event)
 {
+    if (!SDL_GetRelativeMouseMode()) {
+        // Not capturing
+        return;
+    }
+
     if (event->y != 0) {
         LiSendScrollEvent((signed char)event->y);
     }

app/streaming/session.cpp

@@ -3,6 +3,7 @@
 
 #include <Limelight.h>
 #include <SDL.h>
+#include "utils.h"
 
 #include <QRandomGenerator>
 #include <QtEndian>
@@ -28,8 +29,6 @@ AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = {
     CAPABILITY_DIRECT_SUBMIT
 };
 
-DECODER_RENDERER_CALLBACKS Session::k_VideoCallbacks;
-
 Session* Session::s_ActiveSession;
 
 void Session::clStageStarting(int stage)
@@ -79,13 +78,17 @@ Session::Session(NvComputer* computer, NvApp& app)
     : m_Computer(computer),
       m_App(app)
 {
-    StreamingPreferences prefs;
+    LiInitializeVideoCallbacks(&m_VideoCallbacks);
+    m_VideoCallbacks.setup = drSetup;
+    m_VideoCallbacks.cleanup = drCleanup;
+    m_VideoCallbacks.submitDecodeUnit = drSubmitDecodeUnit;
+    m_VideoCallbacks.capabilities = getDecoderCapabilities();
 
     LiInitializeStreamConfiguration(&m_StreamConfig);
-    m_StreamConfig.width = prefs.width;
+    m_StreamConfig.width = m_Preferences.width;
-    m_StreamConfig.height = prefs.height;
+    m_StreamConfig.height = m_Preferences.height;
-    m_StreamConfig.fps = prefs.fps;
+    m_StreamConfig.fps = m_Preferences.fps;
-    m_StreamConfig.bitrate = prefs.bitrateKbps;
+    m_StreamConfig.bitrate = m_Preferences.bitrateKbps;
     m_StreamConfig.packetSize = 1024;
     m_StreamConfig.hevcBitratePercentageMultiplier = 75;
     for (unsigned int i = 0; i < sizeof(m_StreamConfig.remoteInputAesKey); i++) {
@@ -93,7 +96,7 @@ Session::Session(NvComputer* computer, NvApp& app)
                 (char)(QRandomGenerator::global()->generate() % 256);
     }
     *(int*)m_StreamConfig.remoteInputAesIv = qToBigEndian(QRandomGenerator::global()->generate());
-    switch (prefs.audioConfig)
+    switch (m_Preferences.audioConfig)
     {
     case StreamingPreferences::AC_AUTO:
         m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration();
@@ -105,7 +108,7 @@ Session::Session(NvComputer* computer, NvApp& app)
         m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_51_SURROUND;
         break;
     }
-    switch (prefs.videoCodecConfig)
+    switch (m_Preferences.videoCodecConfig)
     {
     case StreamingPreferences::VCC_AUTO:
         // TODO: Determine if HEVC is better depending on the decoder
@@ -226,7 +229,7 @@ void Session::exec()
     }
 
     int err = LiStartConnection(&hostInfo, &m_StreamConfig, &k_ConnCallbacks,
-                                &k_VideoCallbacks, &k_AudioCallbacks,
+                                &m_VideoCallbacks, &k_AudioCallbacks,
                                 NULL, 0, NULL, 0);
     if (err != 0) {
         // We already displayed an error dialog in the stage failure
@@ -238,7 +241,50 @@ void Session::exec()
     emit connectionStarted();
     QCoreApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
 
-    SDL_Window* wnd = SDL_CreateWindow("SDL Test Window", 0, 0, 1280, 720, SDL_WINDOW_INPUT_GRABBED);
+    m_Window = SDL_CreateWindow("Moonlight",
+                                SDL_WINDOWPOS_UNDEFINED,
+                                SDL_WINDOWPOS_UNDEFINED,
+                                m_StreamConfig.width,
+                                m_StreamConfig.height,
+                                (m_Preferences.fullScreen ?
+                                    SDL_WINDOW_FULLSCREEN :
+                                    0));
+    if (!m_Window) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateWindow() failed: %s",
+                     SDL_GetError());
+        LiStopConnection();
+        return;
+    }
+
+    m_Renderer = SDL_CreateRenderer(m_Window, -1,
+                                    SDL_RENDERER_ACCELERATED);
+    if (!m_Renderer) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateRenderer() failed: %s",
+                     SDL_GetError());
+        LiStopConnection();
+        SDL_DestroyWindow(m_Window);
+        return;
+    }
+
+    m_Texture = SDL_CreateTexture(m_Renderer,
+                                  SDL_PIXELFORMAT_YV12,
+                                  SDL_TEXTUREACCESS_STREAMING,
+                                  m_StreamConfig.width,
+                                  m_StreamConfig.height);
+    if (!m_Texture) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_CreateRenderer() failed: %s",
+                     SDL_GetError());
+        LiStopConnection();
+        SDL_DestroyRenderer(m_Renderer);
+        SDL_DestroyWindow(m_Window);
+        return;
+    }
+
+    // Capture the mouse
+    SDL_SetRelativeMouseMode(SDL_TRUE);
 
     // Hijack this thread to be the SDL main thread. We have to do this
     // because we want to suspend all Qt processing until the stream is over.
@@ -249,6 +295,25 @@ void Session::exec()
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                         "Quit event received");
             goto Exit;
+        case SDL_USEREVENT: {
+            SDL_Event nextEvent;
+
+            SDL_assert(event.user.code == SDL_CODE_FRAME_READY);
+
+            // Drop any earlier frames
+            while (SDL_PeepEvents(&nextEvent,
+                                  1,
+                                  SDL_GETEVENT,
+                                  SDL_USEREVENT,
+                                  SDL_USEREVENT) == 1) {
+                dropFrame(&event.user);
+                event = nextEvent;
+            }
+
+            // Render the last frame
+            renderFrame(&event.user);
+            break;
+        }
         case SDL_KEYUP:
         case SDL_KEYDOWN:
             inputHandler.handleKeyEvent(&event.key);
@@ -284,5 +349,7 @@ void Session::exec()
 Exit:
     s_ActiveSession = nullptr;
     LiStopConnection();
-    SDL_DestroyWindow(wnd);
+    SDL_DestroyTexture(m_Texture);
+    SDL_DestroyRenderer(m_Renderer);
+    SDL_DestroyWindow(m_Window);
 }

app/streaming/session.hpp

@@ -3,12 +3,21 @@
 #include <Limelight.h>
 #include <opus_multistream.h>
 #include "backend/computermanager.h"
+#include "settings/streamingpreferences.h"
 #include "input.hpp"
 
+extern "C" {
+#include <libavcodec/avcodec.h>
+}
+
+#define SDL_CODE_FRAME_READY 0
+
 class Session : public QObject
 {
     Q_OBJECT
 
+    friend class SdlInputHandler;
+
 public:
     explicit Session(NvComputer* computer, NvApp& app);
 
@@ -28,6 +37,14 @@ signals:
 private:
     bool validateLaunch();
 
+    int getDecoderCapabilities();
+
+    int sdlDetermineAudioConfiguration();
+
+    void renderFrame(SDL_UserEvent* event);
+
+    void dropFrame(SDL_UserEvent* event);
+
     static
     void clStageStarting(int stage);
 
@@ -41,7 +58,13 @@ private:
     void clLogMessage(const char* format, ...);
 
     static
-    int sdlDetermineAudioConfiguration();
+    int drSetup(int videoFormat, int width, int height, int frameRate, void*, int);
+
+    static
+    void drCleanup();
+
+    static
+    int drSubmitDecodeUnit(PDECODE_UNIT du);
 
     static
     int sdlAudioInit(int audioConfiguration,
@@ -60,9 +83,18 @@ private:
     static
     void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength);
 
+    StreamingPreferences m_Preferences;
     STREAM_CONFIGURATION m_StreamConfig;
+    DECODER_RENDERER_CALLBACKS m_VideoCallbacks;
     NvComputer* m_Computer;
     NvApp m_App;
+    SDL_Window* m_Window;
+    SDL_Renderer* m_Renderer;
+    SDL_Texture* m_Texture;
+
+    static AVPacket s_Pkt;
+    static AVCodecContext* s_VideoDecoderCtx;
+    static QByteArray s_DecodeBuffer;
 
     static SDL_AudioDeviceID s_AudioDevice;
     static OpusMSDecoder* s_OpusDecoder;
@@ -70,7 +102,6 @@ private:
     static int s_ChannelCount;
 
     static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks;
-    static DECODER_RENDERER_CALLBACKS k_VideoCallbacks;
     static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks;
     static Session* s_ActiveSession;
 };

app/streaming/video.cpp

@@ -1,2 +1,168 @@
 #include <Limelight.h>
 #include "session.hpp"
+
+AVPacket Session::s_Pkt;
+AVCodecContext* Session::s_VideoDecoderCtx;
+QByteArray Session::s_DecodeBuffer;
+
+int Session::getDecoderCapabilities()
+{
+    int caps = 0;
+
+    // Submit for decode without using a separate thread
+    caps |= CAPABILITY_DIRECT_SUBMIT;
+
+    // Slice up to 4 times for parallel decode, once slice per core
+    caps |= CAPABILITY_SLICES_PER_FRAME(std::min(4, SDL_GetCPUCount()));
+
+    return caps;
+}
+
+int Session::drSetup(int videoFormat, int width, int height, int /* frameRate */, void*, int)
+{
+    AVCodec* decoder;
+
+    av_init_packet(&s_Pkt);
+
+    switch (videoFormat) {
+    case VIDEO_FORMAT_H264:
+        decoder = avcodec_find_decoder(AV_CODEC_ID_H264);
+        break;
+    case VIDEO_FORMAT_H265:
+    case VIDEO_FORMAT_H265_MAIN10:
+        decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
+        break;
+    }
+
+    if (!decoder) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to find decoder for format: %x",
+                     videoFormat);
+        return -1;
+    }
+
+    s_VideoDecoderCtx = avcodec_alloc_context3(decoder);
+    if (!s_VideoDecoderCtx) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to allocate video decoder context");
+        return -1;
+    }
+
+    // Enable slice multi-threading for software decoding
+    s_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
+    s_VideoDecoderCtx->thread_type = FF_THREAD_SLICE;
+    s_VideoDecoderCtx->thread_count = SDL_GetCPUCount();
+
+    // Setup decoding parameters
+    s_VideoDecoderCtx->width = width;
+    s_VideoDecoderCtx->height = height;
+    s_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
+
+    int err = avcodec_open2(s_VideoDecoderCtx, decoder, nullptr);
+    if (err < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to open decoder for format: %x",
+                     videoFormat);
+        av_free(s_VideoDecoderCtx);
+        return -1;
+    }
+
+    // 1MB frame buffer to start
+    s_DecodeBuffer = QByteArray(1024 * 1024, 0);
+
+    return 0;
+}
+
+void Session::drCleanup()
+{
+    avcodec_close(s_VideoDecoderCtx);
+    av_free(s_VideoDecoderCtx);
+    s_VideoDecoderCtx = nullptr;
+}
+
+int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
+{
+    PLENTRY entry = du->bufferList;
+    int err;
+
+    if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > s_DecodeBuffer.length()) {
+        s_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0);
+    }
+
+    int offset = 0;
+    while (entry != nullptr) {
+        memcpy(&s_DecodeBuffer.data()[offset],
+               entry->data,
+               entry->length);
+        offset += entry->length;
+        entry = entry->next;
+    }
+
+    SDL_assert(offset == du->fullLength);
+
+    s_Pkt.data = reinterpret_cast<uint8_t*>(s_DecodeBuffer.data());
+    s_Pkt.size = du->fullLength;
+
+    err = avcodec_send_packet(s_VideoDecoderCtx, &s_Pkt);
+    if (err < 0) {
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Decoding failed: %d", err);
+        char errorstring[512];
+        av_strerror(err, errorstring, sizeof(errorstring));
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Decoding failed: %s", errorstring);
+        return DR_NEED_IDR;
+    }
+
+    AVFrame* frame = av_frame_alloc();
+    if (!frame) {
+        // Failed to allocate a frame but we did submit,
+        // so we can return DR_OK
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Failed to allocate frame");
+        return DR_OK;
+    }
+
+    err = avcodec_receive_frame(s_VideoDecoderCtx, frame);
+    if (err == 0) {
+        SDL_Event event;
+
+        event.type = SDL_USEREVENT;
+        event.user.code = SDL_CODE_FRAME_READY;
+        event.user.data1 = frame;
+
+        // The main thread will handle freeing this
+        SDL_PushEvent(&event);
+    }
+    else {
+        av_frame_free(&frame);
+    }
+
+    return DR_OK;
+}
+
+// Called on main thread
+void Session::renderFrame(SDL_UserEvent* event)
+{
+    AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
+
+    SDL_UpdateYUVTexture(m_Texture, nullptr,
+                         frame->data[0],
+                         frame->linesize[0],
+                         frame->data[1],
+                         frame->linesize[1],
+                         frame->data[2],
+                         frame->linesize[2]);
+    SDL_RenderClear(m_Renderer);
+    SDL_RenderCopy(m_Renderer, m_Texture, nullptr, nullptr);
+    SDL_RenderPresent(m_Renderer);
+
+    av_frame_free(&frame);
+}
+
+// Called on main thread
+void Session::dropFrame(SDL_UserEvent* event)
+{
+    AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
+    av_frame_free(&frame);
+}