moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2026-04-13 11:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented software video decoding and special key combos

Browse Source
This commit is contained in:
Cameron Gutman
2018-07-07 21:52:20 -07:00
parent 02f4f193a9
commit 826fe4cd03
5 changed files with 324 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/gui/AppView.qml
View File

@@ -66,8 +66,16 @@ GridView {
anchors.fill: parent
onClicked: {
// TODO: Check if a different game is running
var session = appModel.createSessionForApp(index);
session.exec();
var session = appModel.createSessionForApp(index)
// Don't poll while the stream is running
ComputerManager.stopPollingAsync()
// Run the streaming session to completion
session.exec()
// Start polling again
ComputerManager.startPolling()
}
}
}

43
app/streaming/input.cpp
View File

@@ -1,6 +1,6 @@
#include <Limelight.h>
#include <SDL.h>
#include "input.hpp"
#include "streaming/session.hpp"
#define VK_0 0x30
#define VK_A 0x41
@@ -80,22 +80,29 @@ void SdlInputHandler::handleKeyEvent(SDL_KeyboardEvent* event)
// Check for the unbind combo (Ctrl+Alt+Shift+Z)
else if (event->keysym.sym == SDLK_z) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Detected mouse unbind combo");
// TODO: disable mouse capture then recapture when
// we regain focus or on click.
"Detected mouse capture toggle combo");
SDL_SetRelativeMouseMode((SDL_bool)!SDL_GetRelativeMouseMode());
return;
}
// Check for the full-screen combo (Ctrl+Alt+Shift+X)
else if (event->keysym.sym == SDLK_x) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Detected full-screen toggle combo");
// TODO: toggle full-screen
if (SDL_GetWindowFlags(Session::s_ActiveSession->m_Window) & SDL_WINDOW_FULLSCREEN) {
SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, 0);
}
else {
SDL_SetWindowFullscreen(Session::s_ActiveSession->m_Window, SDL_WINDOW_FULLSCREEN);
}
return;
}
}
if (!SDL_GetRelativeMouseMode()) {
// Not capturing
return;
}
// Set modifier flags
modifiers = 0;
if (event->keysym.mod & KMOD_CTRL) {
@@ -316,6 +323,18 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event)
{
int button;
// Capture the mouse again if clicked when unbound
if (event->button == SDL_BUTTON_LEFT &&
event->state == SDL_PRESSED &&
!SDL_GetRelativeMouseMode()) {
SDL_SetRelativeMouseMode(SDL_TRUE);
return;
}
else if (!SDL_GetRelativeMouseMode()) {
// Not capturing
return;
}
switch (event->button)
{
case SDL_BUTTON_LEFT:
@@ -346,6 +365,11 @@ void SdlInputHandler::handleMouseButtonEvent(SDL_MouseButtonEvent* event)
void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event)
{
if (!SDL_GetRelativeMouseMode()) {
// Not capturing
return;
}
if (event->xrel != 0 || event->yrel != 0) {
LiSendMouseMoveEvent((unsigned short)event->xrel,
(unsigned short)event->yrel);
@@ -354,6 +378,11 @@ void SdlInputHandler::handleMouseMotionEvent(SDL_MouseMotionEvent* event)
void SdlInputHandler::handleMouseWheelEvent(SDL_MouseWheelEvent* event)
{
if (!SDL_GetRelativeMouseMode()) {
// Not capturing
return;
}
if (event->y != 0) {
LiSendScrollEvent((signed char)event->y);
}

91
app/streaming/session.cpp
View File

@@ -3,6 +3,7 @@
#include <Limelight.h>
#include <SDL.h>
#include "utils.h"
#include <QRandomGenerator>
#include <QtEndian>
@@ -28,8 +29,6 @@ AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = {
CAPABILITY_DIRECT_SUBMIT
};
DECODER_RENDERER_CALLBACKS Session::k_VideoCallbacks;
Session* Session::s_ActiveSession;
void Session::clStageStarting(int stage)
@@ -79,13 +78,17 @@ Session::Session(NvComputer* computer, NvApp& app)
: m_Computer(computer),
m_App(app)
{
StreamingPreferences prefs;
LiInitializeVideoCallbacks(&m_VideoCallbacks);
m_VideoCallbacks.setup = drSetup;
m_VideoCallbacks.cleanup = drCleanup;
m_VideoCallbacks.submitDecodeUnit = drSubmitDecodeUnit;
m_VideoCallbacks.capabilities = getDecoderCapabilities();
LiInitializeStreamConfiguration(&m_StreamConfig);
m_StreamConfig.width = prefs.width;
m_StreamConfig.height = prefs.height;
m_StreamConfig.fps = prefs.fps;
m_StreamConfig.bitrate = prefs.bitrateKbps;
m_StreamConfig.width = m_Preferences.width;
m_StreamConfig.height = m_Preferences.height;
m_StreamConfig.fps = m_Preferences.fps;
m_StreamConfig.bitrate = m_Preferences.bitrateKbps;
m_StreamConfig.packetSize = 1024;
m_StreamConfig.hevcBitratePercentageMultiplier = 75;
for (unsigned int i = 0; i < sizeof(m_StreamConfig.remoteInputAesKey); i++) {
@@ -93,7 +96,7 @@ Session::Session(NvComputer* computer, NvApp& app)
(char)(QRandomGenerator::global()->generate() % 256);
}
*(int*)m_StreamConfig.remoteInputAesIv = qToBigEndian(QRandomGenerator::global()->generate());
switch (prefs.audioConfig)
switch (m_Preferences.audioConfig)
{
case StreamingPreferences::AC_AUTO:
m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration();
@@ -105,7 +108,7 @@ Session::Session(NvComputer* computer, NvApp& app)
m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_51_SURROUND;
break;
}
switch (prefs.videoCodecConfig)
switch (m_Preferences.videoCodecConfig)
{
case StreamingPreferences::VCC_AUTO:
// TODO: Determine if HEVC is better depending on the decoder
@@ -226,7 +229,7 @@ void Session::exec()
}
int err = LiStartConnection(&hostInfo, &m_StreamConfig, &k_ConnCallbacks,
&k_VideoCallbacks, &k_AudioCallbacks,
&m_VideoCallbacks, &k_AudioCallbacks,
NULL, 0, NULL, 0);
if (err != 0) {
// We already displayed an error dialog in the stage failure
@@ -238,7 +241,50 @@ void Session::exec()
emit connectionStarted();
QCoreApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
SDL_Window* wnd = SDL_CreateWindow("SDL Test Window", 0, 0, 1280, 720, SDL_WINDOW_INPUT_GRABBED);
m_Window = SDL_CreateWindow("Moonlight",
SDL_WINDOWPOS_UNDEFINED,
SDL_WINDOWPOS_UNDEFINED,
m_StreamConfig.width,
m_StreamConfig.height,
(m_Preferences.fullScreen ?
SDL_WINDOW_FULLSCREEN :
0));
if (!m_Window) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"SDL_CreateWindow() failed: %s",
SDL_GetError());
LiStopConnection();
return;
}
m_Renderer = SDL_CreateRenderer(m_Window, -1,
SDL_RENDERER_ACCELERATED);
if (!m_Renderer) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"SDL_CreateRenderer() failed: %s",
SDL_GetError());
LiStopConnection();
SDL_DestroyWindow(m_Window);
return;
}
m_Texture = SDL_CreateTexture(m_Renderer,
SDL_PIXELFORMAT_YV12,
SDL_TEXTUREACCESS_STREAMING,
m_StreamConfig.width,
m_StreamConfig.height);
if (!m_Texture) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"SDL_CreateRenderer() failed: %s",
SDL_GetError());
LiStopConnection();
SDL_DestroyRenderer(m_Renderer);
SDL_DestroyWindow(m_Window);
return;
}
// Capture the mouse
SDL_SetRelativeMouseMode(SDL_TRUE);
// Hijack this thread to be the SDL main thread. We have to do this
// because we want to suspend all Qt processing until the stream is over.
@@ -249,6 +295,25 @@ void Session::exec()
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Quit event received");
goto Exit;
case SDL_USEREVENT: {
SDL_Event nextEvent;
SDL_assert(event.user.code == SDL_CODE_FRAME_READY);
// Drop any earlier frames
while (SDL_PeepEvents(&nextEvent,
1,
SDL_GETEVENT,
SDL_USEREVENT,
SDL_USEREVENT) == 1) {
dropFrame(&event.user);
event = nextEvent;
}
// Render the last frame
renderFrame(&event.user);
break;
}
case SDL_KEYUP:
case SDL_KEYDOWN:
inputHandler.handleKeyEvent(&event.key);
@@ -284,5 +349,7 @@ void Session::exec()
Exit:
s_ActiveSession = nullptr;
LiStopConnection();
SDL_DestroyWindow(wnd);
SDL_DestroyTexture(m_Texture);
SDL_DestroyRenderer(m_Renderer);
SDL_DestroyWindow(m_Window);
}

35
app/streaming/session.hpp
View File

@@ -3,12 +3,21 @@
#include <Limelight.h>
#include <opus_multistream.h>
#include "backend/computermanager.h"
#include "settings/streamingpreferences.h"
#include "input.hpp"
extern "C" {
#include <libavcodec/avcodec.h>
}
#define SDL_CODE_FRAME_READY 0
class Session : public QObject
{
Q_OBJECT
friend class SdlInputHandler;
public:
explicit Session(NvComputer* computer, NvApp& app);
@@ -28,6 +37,14 @@ signals:
private:
bool validateLaunch();
int getDecoderCapabilities();
int sdlDetermineAudioConfiguration();
void renderFrame(SDL_UserEvent* event);
void dropFrame(SDL_UserEvent* event);
static
void clStageStarting(int stage);
@@ -41,7 +58,13 @@ private:
void clLogMessage(const char* format, ...);
static
int sdlDetermineAudioConfiguration();
int drSetup(int videoFormat, int width, int height, int frameRate, void*, int);
static
void drCleanup();
static
int drSubmitDecodeUnit(PDECODE_UNIT du);
static
int sdlAudioInit(int audioConfiguration,
@@ -60,9 +83,18 @@ private:
static
void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength);
StreamingPreferences m_Preferences;
STREAM_CONFIGURATION m_StreamConfig;
DECODER_RENDERER_CALLBACKS m_VideoCallbacks;
NvComputer* m_Computer;
NvApp m_App;
SDL_Window* m_Window;
SDL_Renderer* m_Renderer;
SDL_Texture* m_Texture;
static AVPacket s_Pkt;
static AVCodecContext* s_VideoDecoderCtx;
static QByteArray s_DecodeBuffer;
static SDL_AudioDeviceID s_AudioDevice;
static OpusMSDecoder* s_OpusDecoder;
@@ -70,7 +102,6 @@ private:
static int s_ChannelCount;
static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks;
static DECODER_RENDERER_CALLBACKS k_VideoCallbacks;
static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks;
static Session* s_ActiveSession;
};

166
app/streaming/video.cpp
View File

@@ -1,2 +1,168 @@
#include <Limelight.h>
#include "session.hpp"
AVPacket Session::s_Pkt;
AVCodecContext* Session::s_VideoDecoderCtx;
QByteArray Session::s_DecodeBuffer;
int Session::getDecoderCapabilities()
{
int caps = 0;
// Submit for decode without using a separate thread
caps |= CAPABILITY_DIRECT_SUBMIT;
// Slice up to 4 times for parallel decode, once slice per core
caps |= CAPABILITY_SLICES_PER_FRAME(std::min(4, SDL_GetCPUCount()));
return caps;
}
int Session::drSetup(int videoFormat, int width, int height, int /* frameRate */, void*, int)
{
AVCodec* decoder;
av_init_packet(&s_Pkt);
switch (videoFormat) {
case VIDEO_FORMAT_H264:
decoder = avcodec_find_decoder(AV_CODEC_ID_H264);
break;
case VIDEO_FORMAT_H265:
case VIDEO_FORMAT_H265_MAIN10:
decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
break;
}
if (!decoder) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to find decoder for format: %x",
videoFormat);
return -1;
}
s_VideoDecoderCtx = avcodec_alloc_context3(decoder);
if (!s_VideoDecoderCtx) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to allocate video decoder context");
return -1;
}
// Enable slice multi-threading for software decoding
s_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
s_VideoDecoderCtx->thread_type = FF_THREAD_SLICE;
s_VideoDecoderCtx->thread_count = SDL_GetCPUCount();
// Setup decoding parameters
s_VideoDecoderCtx->width = width;
s_VideoDecoderCtx->height = height;
s_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
int err = avcodec_open2(s_VideoDecoderCtx, decoder, nullptr);
if (err < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to open decoder for format: %x",
videoFormat);
av_free(s_VideoDecoderCtx);
return -1;
}
// 1MB frame buffer to start
s_DecodeBuffer = QByteArray(1024 * 1024, 0);
return 0;
}
void Session::drCleanup()
{
avcodec_close(s_VideoDecoderCtx);
av_free(s_VideoDecoderCtx);
s_VideoDecoderCtx = nullptr;
}
int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
{
PLENTRY entry = du->bufferList;
int err;
if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > s_DecodeBuffer.length()) {
s_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0);
}
int offset = 0;
while (entry != nullptr) {
memcpy(&s_DecodeBuffer.data()[offset],
entry->data,
entry->length);
offset += entry->length;
entry = entry->next;
}
SDL_assert(offset == du->fullLength);
s_Pkt.data = reinterpret_cast<uint8_t*>(s_DecodeBuffer.data());
s_Pkt.size = du->fullLength;
err = avcodec_send_packet(s_VideoDecoderCtx, &s_Pkt);
if (err < 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Decoding failed: %d", err);
char errorstring[512];
av_strerror(err, errorstring, sizeof(errorstring));
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Decoding failed: %s", errorstring);
return DR_NEED_IDR;
}
AVFrame* frame = av_frame_alloc();
if (!frame) {
// Failed to allocate a frame but we did submit,
// so we can return DR_OK
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Failed to allocate frame");
return DR_OK;
}
err = avcodec_receive_frame(s_VideoDecoderCtx, frame);
if (err == 0) {
SDL_Event event;
event.type = SDL_USEREVENT;
event.user.code = SDL_CODE_FRAME_READY;
event.user.data1 = frame;
// The main thread will handle freeing this
SDL_PushEvent(&event);
}
else {
av_frame_free(&frame);
}
return DR_OK;
}
// Called on main thread
void Session::renderFrame(SDL_UserEvent* event)
{
AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
SDL_UpdateYUVTexture(m_Texture, nullptr,
frame->data[0],
frame->linesize[0],
frame->data[1],
frame->linesize[1],
frame->data[2],
frame->linesize[2]);
SDL_RenderClear(m_Renderer);
SDL_RenderCopy(m_Renderer, m_Texture, nullptr, nullptr);
SDL_RenderPresent(m_Renderer);
av_frame_free(&frame);
}
// Called on main thread
void Session::dropFrame(SDL_UserEvent* event)
{
AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
av_frame_free(&frame);
}