From 826fe4cd03b81bb61e849af933d744b1348b9c42 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sat, 7 Jul 2018 21:52:20 -0700 Subject: [PATCH] Implemented software video decoding and special key combos --- app/gui/AppView.qml | 12 ++- app/streaming/input.cpp | 43 ++++++++-- app/streaming/session.cpp | 91 ++++++++++++++++++--- app/streaming/session.hpp | 35 +++++++- app/streaming/video.cpp | 166 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 324 insertions(+), 23 deletions(-) diff --git a/app/gui/AppView.qml b/app/gui/AppView.qml index ce7ceef7..04cc4ee2 100644 --- a/app/gui/AppView.qml +++ b/app/gui/AppView.qml @@ -66,8 +66,16 @@ GridView { anchors.fill: parent onClicked: { // TODO: Check if a different game is running - var session = appModel.createSessionForApp(index); - session.exec(); + var session = appModel.createSessionForApp(index) + + // Don't poll while the stream is running + ComputerManager.stopPollingAsync() + + // Run the streaming session to completion + session.exec() + + // Start polling again + ComputerManager.startPolling() } } } diff --git a/app/streaming/input.cpp b/app/streaming/input.cpp index c053341a..45c2a1f4 100644 --- a/app/streaming/input.cpp +++ b/app/streaming/input.cpp @@ -1,6 +1,6 @@ #include #include -#include "input.hpp" +#include "streaming/session.hpp" #define VK_0 0x30 #define VK_A 0x41 @@ -80,22 +80,29 @@ void SdlInputHandler::handleKeyEvent(SDL_KeyboardEvent* event) // Check for the unbind combo (Ctrl+Alt+Shift+Z) else if (event->keysym.sym == SDLK_z) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, - "Detected mouse unbind combo"); - - // TODO: disable mouse capture then recapture when - // we regain focus or on click. + "Detected mouse capture toggle combo"); + SDL_SetRelativeMouseMode((SDL_bool)!SDL_GetRelativeMouseMode()); return; } // Check for the full-screen combo (Ctrl+Alt+Shift+X) else if (event->keysym.sym == SDLK_x) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Detected full-screen toggle combo"); - - // TODO: toggle full-screen + if (SDL_GetWindowFlags(Session::s_ActiveSession->m_Window) & SDL_WINDOW_FULLSCREEN) { + SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, 0); + } + else { + SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, SDL_WINDOW_FULLSCREEN); + } return; } } + if (!SDL_GetRelativeMouseMode()) { + // Not capturing + return; + } + // Set modifier flags modifiers = 0; if (event->keysym.mod & KMOD_CTRL) { @@ -316,6 +323,18 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event) { int button; + // Capture the mouse again if clicked when unbound + if (event->button == SDL_BUTTON_LEFT && + event->state == SDL_PRESSED && + !SDL_GetRelativeMouseMode()) { + SDL_SetRelativeMouseMode(SDL_TRUE); + return; + } + else if (!SDL_GetRelativeMouseMode()) { + // Not capturing + return; + } + switch (event->button) { case SDL_BUTTON_LEFT: @@ -346,6 +365,11 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event) void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event) { + if (!SDL_GetRelativeMouseMode()) { + // Not capturing + return; + } + if (event->xrel != 0 || event->yrel != 0) { LiSendMouseMoveEvent((unsigned short)event->xrel, (unsigned short)event->yrel); @@ -354,6 +378,11 @@ void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event) void SdlInputHandler::handleMouseWheelEvent(SDL_MouseWheelEvent* event) { + if (!SDL_GetRelativeMouseMode()) { + // Not capturing + return; + } + if (event->y != 0) { LiSendScrollEvent((signed char)event->y); } diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 2b71f949..dc84f1b8 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -3,6 +3,7 @@ #include #include +#include "utils.h" #include #include @@ -28,8 +29,6 @@ AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = { CAPABILITY_DIRECT_SUBMIT }; -DECODER_RENDERER_CALLBACKS Session::k_VideoCallbacks; - Session* Session::s_ActiveSession; void Session::clStageStarting(int stage) @@ -79,13 +78,17 @@ Session::Session(NvComputer* computer, NvApp& app) : m_Computer(computer), m_App(app) { - StreamingPreferences prefs; + LiInitializeVideoCallbacks(&m_VideoCallbacks); + m_VideoCallbacks.setup = drSetup; + m_VideoCallbacks.cleanup = drCleanup; + m_VideoCallbacks.submitDecodeUnit = drSubmitDecodeUnit; + m_VideoCallbacks.capabilities = getDecoderCapabilities(); LiInitializeStreamConfiguration(&m_StreamConfig); - m_StreamConfig.width = prefs.width; - m_StreamConfig.height = prefs.height; - m_StreamConfig.fps = prefs.fps; - m_StreamConfig.bitrate = prefs.bitrateKbps; + m_StreamConfig.width = m_Preferences.width; + m_StreamConfig.height = m_Preferences.height; + m_StreamConfig.fps = m_Preferences.fps; + m_StreamConfig.bitrate = m_Preferences.bitrateKbps; m_StreamConfig.packetSize = 1024; m_StreamConfig.hevcBitratePercentageMultiplier = 75; for (unsigned int i = 0; i < sizeof(m_StreamConfig.remoteInputAesKey); i++) { @@ -93,7 +96,7 @@ Session::Session(NvComputer* computer, NvApp& app) (char)(QRandomGenerator::global()->generate() % 256); } *(int*)m_StreamConfig.remoteInputAesIv = qToBigEndian(QRandomGenerator::global()->generate()); - switch (prefs.audioConfig) + switch (m_Preferences.audioConfig) { case StreamingPreferences::AC_AUTO: m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration(); @@ -105,7 +108,7 @@ Session::Session(NvComputer* computer, NvApp& app) m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_51_SURROUND; break; } - switch (prefs.videoCodecConfig) + switch (m_Preferences.videoCodecConfig) { case StreamingPreferences::VCC_AUTO: // TODO: Determine if HEVC is better depending on the decoder @@ -226,7 +229,7 @@ void Session::exec() } int err = LiStartConnection(&hostInfo, &m_StreamConfig, &k_ConnCallbacks, - &k_VideoCallbacks, &k_AudioCallbacks, + &m_VideoCallbacks, &k_AudioCallbacks, NULL, 0, NULL, 0); if (err != 0) { // We already displayed an error dialog in the stage failure @@ -238,7 +241,50 @@ void Session::exec() emit connectionStarted(); QCoreApplication::processEvents(QEventLoop::ExcludeUserInputEvents); - SDL_Window* wnd = SDL_CreateWindow("SDL Test Window", 0, 0, 1280, 720, SDL_WINDOW_INPUT_GRABBED); + m_Window = SDL_CreateWindow("Moonlight", + SDL_WINDOWPOS_UNDEFINED, + SDL_WINDOWPOS_UNDEFINED, + m_StreamConfig.width, + m_StreamConfig.height, + (m_Preferences.fullScreen ? + SDL_WINDOW_FULLSCREEN : + 0)); + if (!m_Window) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "SDL_CreateWindow() failed: %s", + SDL_GetError()); + LiStopConnection(); + return; + } + + m_Renderer = SDL_CreateRenderer(m_Window, -1, + SDL_RENDERER_ACCELERATED); + if (!m_Renderer) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "SDL_CreateRenderer() failed: %s", + SDL_GetError()); + LiStopConnection(); + SDL_DestroyWindow(m_Window); + return; + } + + m_Texture = SDL_CreateTexture(m_Renderer, + SDL_PIXELFORMAT_YV12, + SDL_TEXTUREACCESS_STREAMING, + m_StreamConfig.width, + m_StreamConfig.height); + if (!m_Texture) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "SDL_CreateRenderer() failed: %s", + SDL_GetError()); + LiStopConnection(); + SDL_DestroyRenderer(m_Renderer); + SDL_DestroyWindow(m_Window); + return; + } + + // Capture the mouse + SDL_SetRelativeMouseMode(SDL_TRUE); // Hijack this thread to be the SDL main thread. We have to do this // because we want to suspend all Qt processing until the stream is over. @@ -249,6 +295,25 @@ void Session::exec() SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Quit event received"); goto Exit; + case SDL_USEREVENT: { + SDL_Event nextEvent; + + SDL_assert(event.user.code == SDL_CODE_FRAME_READY); + + // Drop any earlier frames + while (SDL_PeepEvents(&nextEvent, + 1, + SDL_GETEVENT, + SDL_USEREVENT, + SDL_USEREVENT) == 1) { + dropFrame(&event.user); + event = nextEvent; + } + + // Render the last frame + renderFrame(&event.user); + break; + } case SDL_KEYUP: case SDL_KEYDOWN: inputHandler.handleKeyEvent(&event.key); @@ -284,5 +349,7 @@ void Session::exec() Exit: s_ActiveSession = nullptr; LiStopConnection(); - SDL_DestroyWindow(wnd); + SDL_DestroyTexture(m_Texture); + SDL_DestroyRenderer(m_Renderer); + SDL_DestroyWindow(m_Window); } diff --git a/app/streaming/session.hpp b/app/streaming/session.hpp index 052a3add..254990dd 100644 --- a/app/streaming/session.hpp +++ b/app/streaming/session.hpp @@ -3,12 +3,21 @@ #include #include #include "backend/computermanager.h" +#include "settings/streamingpreferences.h" #include "input.hpp" +extern "C" { +#include +} + +#define SDL_CODE_FRAME_READY 0 + class Session : public QObject { Q_OBJECT + friend class SdlInputHandler; + public: explicit Session(NvComputer* computer, NvApp& app); @@ -28,6 +37,14 @@ signals: private: bool validateLaunch(); + int getDecoderCapabilities(); + + int sdlDetermineAudioConfiguration(); + + void renderFrame(SDL_UserEvent* event); + + void dropFrame(SDL_UserEvent* event); + static void clStageStarting(int stage); @@ -41,7 +58,13 @@ private: void clLogMessage(const char* format, ...); static - int sdlDetermineAudioConfiguration(); + int drSetup(int videoFormat, int width, int height, int frameRate, void*, int); + + static + void drCleanup(); + + static + int drSubmitDecodeUnit(PDECODE_UNIT du); static int sdlAudioInit(int audioConfiguration, @@ -60,9 +83,18 @@ private: static void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength); + StreamingPreferences m_Preferences; STREAM_CONFIGURATION m_StreamConfig; + DECODER_RENDERER_CALLBACKS m_VideoCallbacks; NvComputer* m_Computer; NvApp m_App; + SDL_Window* m_Window; + SDL_Renderer* m_Renderer; + SDL_Texture* m_Texture; + + static AVPacket s_Pkt; + static AVCodecContext* s_VideoDecoderCtx; + static QByteArray s_DecodeBuffer; static SDL_AudioDeviceID s_AudioDevice; static OpusMSDecoder* s_OpusDecoder; @@ -70,7 +102,6 @@ private: static int s_ChannelCount; static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks; - static DECODER_RENDERER_CALLBACKS k_VideoCallbacks; static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks; static Session* s_ActiveSession; }; diff --git a/app/streaming/video.cpp b/app/streaming/video.cpp index 46dd576d..be743be9 100644 --- a/app/streaming/video.cpp +++ b/app/streaming/video.cpp @@ -1,2 +1,168 @@ #include #include "session.hpp" + +AVPacket Session::s_Pkt; +AVCodecContext* Session::s_VideoDecoderCtx; +QByteArray Session::s_DecodeBuffer; + +int Session::getDecoderCapabilities() +{ + int caps = 0; + + // Submit for decode without using a separate thread + caps |= CAPABILITY_DIRECT_SUBMIT; + + // Slice up to 4 times for parallel decode, once slice per core + caps |= CAPABILITY_SLICES_PER_FRAME(std::min(4, SDL_GetCPUCount())); + + return caps; +} + +int Session::drSetup(int videoFormat, int width, int height, int /* frameRate */, void*, int) +{ + AVCodec* decoder; + + av_init_packet(&s_Pkt); + + switch (videoFormat) { + case VIDEO_FORMAT_H264: + decoder = avcodec_find_decoder(AV_CODEC_ID_H264); + break; + case VIDEO_FORMAT_H265: + case VIDEO_FORMAT_H265_MAIN10: + decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC); + break; + } + + if (!decoder) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unable to find decoder for format: %x", + videoFormat); + return -1; + } + + s_VideoDecoderCtx = avcodec_alloc_context3(decoder); + if (!s_VideoDecoderCtx) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unable to allocate video decoder context"); + return -1; + } + + // Enable slice multi-threading for software decoding + s_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY; + s_VideoDecoderCtx->thread_type = FF_THREAD_SLICE; + s_VideoDecoderCtx->thread_count = SDL_GetCPUCount(); + + // Setup decoding parameters + s_VideoDecoderCtx->width = width; + s_VideoDecoderCtx->height = height; + s_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR + + int err = avcodec_open2(s_VideoDecoderCtx, decoder, nullptr); + if (err < 0) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Unable to open decoder for format: %x", + videoFormat); + av_free(s_VideoDecoderCtx); + return -1; + } + + // 1MB frame buffer to start + s_DecodeBuffer = QByteArray(1024 * 1024, 0); + + return 0; +} + +void Session::drCleanup() +{ + avcodec_close(s_VideoDecoderCtx); + av_free(s_VideoDecoderCtx); + s_VideoDecoderCtx = nullptr; +} + +int Session::drSubmitDecodeUnit(PDECODE_UNIT du) +{ + PLENTRY entry = du->bufferList; + int err; + + if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > s_DecodeBuffer.length()) { + s_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0); + } + + int offset = 0; + while (entry != nullptr) { + memcpy(&s_DecodeBuffer.data()[offset], + entry->data, + entry->length); + offset += entry->length; + entry = entry->next; + } + + SDL_assert(offset == du->fullLength); + + s_Pkt.data = reinterpret_cast(s_DecodeBuffer.data()); + s_Pkt.size = du->fullLength; + + err = avcodec_send_packet(s_VideoDecoderCtx, &s_Pkt); + if (err < 0) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Decoding failed: %d", err); + char errorstring[512]; + av_strerror(err, errorstring, sizeof(errorstring)); + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Decoding failed: %s", errorstring); + return DR_NEED_IDR; + } + + AVFrame* frame = av_frame_alloc(); + if (!frame) { + // Failed to allocate a frame but we did submit, + // so we can return DR_OK + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Failed to allocate frame"); + return DR_OK; + } + + err = avcodec_receive_frame(s_VideoDecoderCtx, frame); + if (err == 0) { + SDL_Event event; + + event.type = SDL_USEREVENT; + event.user.code = SDL_CODE_FRAME_READY; + event.user.data1 = frame; + + // The main thread will handle freeing this + SDL_PushEvent(&event); + } + else { + av_frame_free(&frame); + } + + return DR_OK; +} + +// Called on main thread +void Session::renderFrame(SDL_UserEvent* event) +{ + AVFrame* frame = reinterpret_cast(event->data1); + + SDL_UpdateYUVTexture(m_Texture, nullptr, + frame->data[0], + frame->linesize[0], + frame->data[1], + frame->linesize[1], + frame->data[2], + frame->linesize[2]); + SDL_RenderClear(m_Renderer); + SDL_RenderCopy(m_Renderer, m_Texture, nullptr, nullptr); + SDL_RenderPresent(m_Renderer); + + av_frame_free(&frame); +} + +// Called on main thread +void Session::dropFrame(SDL_UserEvent* event) +{ + AVFrame* frame = reinterpret_cast(event->data1); + av_frame_free(&frame); +}