Version 0.1.0

.travis.yml

@@ -15,6 +15,7 @@ matrix:
         apt:
           sources:
             - sourceline: 'ppa:beineri/opt-qt596-xenial'
+            - sourceline: 'ppa:jonathonf/ffmpeg-4'
           packages:
             - qt59base
             - qt59quickcontrols2
@@ -22,6 +23,10 @@ matrix:
             - libesd0-dev
             - libgl-dev
             - libopus-dev
+            - libavcodec-dev
+            - libavutil-dev
+            - libva-dev
+            - libvdpau-dev
 
 install:
     - '[ "$TRAVIS_OS_NAME" != osx ] || brew install qt5'

README.md

@@ -4,7 +4,7 @@
 
 Moonlight also has mobile versions for [Android](https://github.com/moonlight-stream/moonlight-android) and  [iOS](https://github.com/moonlight-stream/moonlight-ios).
 
-This client is currently still in development (pre-alpha), but the streaming performance is already much better than the older Moonlight Chrome client on many machines. Test releases may be available for your OS on the [Releases page](https://github.com/moonlight-stream/moonlight-qt/releases).
+See the [releases page](https://github.com/moonlight-stream/moonlight-qt/releases) for download links for Windows and Mac. For Linux, we're on [Flathub](https://flathub.org/apps/details/com.moonlight_stream.Moonlight).
 
 You can follow development on our [Discord server](https://discord.gg/6ERtzFY).
 

app/Info.plist

@@ -23,7 +23,7 @@
 	<key>NSSupportsAutomaticGraphicsSwitching</key>
 	<true/>
 	<key>CFBundleVersion</key>
-	<string>0.0.5</string>
+	<string>0.1.0</string>
 	<key>CFBundleDisplayName</key>
 	<string>Moonlight</string>
 </dict>

app/app.pro

@@ -53,6 +53,10 @@ unix:!macx {
         packagesExist(libva) {
             CONFIG += libva
         }
+
+        packagesExist(vdpau) {
+            CONFIG += libvdpau
+        }
     }
 }
 win32 {
@@ -77,7 +81,8 @@ SOURCES += \
     streaming/session.cpp \
     streaming/audio.cpp \
     gui/computermodel.cpp \
-    gui/appmodel.cpp
+    gui/appmodel.cpp \
+    streaming/streamutils.cpp
 
 HEADERS += \
     utils.h \
@@ -91,7 +96,8 @@ HEADERS += \
     streaming/session.hpp \
     gui/computermodel.h \
     gui/appmodel.h \
-    streaming/video/decoder.h
+    streaming/video/decoder.h \
+    streaming/streamutils.h
 
 # Platform-specific renderers and decoders
 ffmpeg {
@@ -112,6 +118,13 @@ libva {
     SOURCES += streaming/video/ffmpeg-renderers/vaapi.cpp
     HEADERS += streaming/video/ffmpeg-renderers/vaapi.h
 }
+libvdpau {
+    message(VDPAU renderer selected)
+
+    DEFINES += HAVE_LIBVDPAU
+    SOURCES += streaming/video/ffmpeg-renderers/vdpau.cpp
+    HEADERS += streaming/video/ffmpeg-renderers/vdpau.h
+}
 config_SLVideo {
     message(SLVideo decoder/renderer selected)
 
@@ -189,4 +202,4 @@ macx {
     QMAKE_BUNDLE_DATA += APP_BUNDLE_RESOURCES
 }
 
-VERSION = 0.0.5
+VERSION = 0.1.0

app/backend/computermanager.cpp

@@ -52,6 +52,7 @@ NvComputer::NvComputer(QSettings& settings)
     this->maxLumaPixelsHEVC = 0;
     this->serverCodecModeSupport = 0;
     this->pendingQuit = false;
+    this->gpuModel = nullptr;
 }
 
 void
@@ -134,6 +135,7 @@ NvComputer::NvComputer(QString address, QString serverInfo)
     this->currentGameId = NvHTTP::getCurrentGame(serverInfo);
     this->appVersion = NvHTTP::getXmlString(serverInfo, "appversion");
     this->gfeVersion = NvHTTP::getXmlString(serverInfo, "GfeVersion");
+    this->gpuModel = NvHTTP::getXmlString(serverInfo, "gputype");
     this->activeAddress = address;
     this->state = NvComputer::CS_ONLINE;
     this->pendingQuit = false;
@@ -266,6 +268,7 @@ bool NvComputer::update(NvComputer& that)
     ASSIGN_IF_CHANGED(gfeVersion);
     ASSIGN_IF_CHANGED(appVersion);
     ASSIGN_IF_CHANGED(maxLumaPixelsHEVC);
+    ASSIGN_IF_CHANGED(gpuModel);
     ASSIGN_IF_CHANGED_AND_NONEMPTY(appList);
     ASSIGN_IF_CHANGED_AND_NONEMPTY(displayModes);
     return changed;

app/backend/computermanager.h

@@ -65,6 +65,7 @@ public:
     QVector<NvDisplayMode> displayModes;
     int maxLumaPixelsHEVC;
     int serverCodecModeSupport;
+    QString gpuModel;
 
     // Persisted traits
     QString localAddress;

app/backend/nvhttp.cpp

@@ -67,6 +67,7 @@ NvHTTP::getServerInfo()
         serverInfo = openConnectionToString(m_BaseUrlHttps,
                                             "serverinfo",
                                             nullptr,
+                                            true,
                                             true);
         // Throws if the request failed
         verifyResponseStatus(serverInfo);
@@ -79,6 +80,7 @@ NvHTTP::getServerInfo()
             serverInfo = openConnectionToString(m_BaseUrlHttp,
                                                 "serverinfo",
                                                 nullptr,
+                                                true,
                                                 true);
             verifyResponseStatus(serverInfo);
         }
@@ -225,6 +227,7 @@ NvHTTP::getAppList()
     QString appxml = openConnectionToString(m_BaseUrlHttps,
                                             "applist",
                                             nullptr,
+                                            true,
                                             true);
     verifyResponseStatus(appxml);
 
@@ -338,9 +341,10 @@ QString
 NvHTTP::openConnectionToString(QUrl baseUrl,
                                QString command,
                                QString arguments,
-                               bool enableTimeout)
+                               bool enableTimeout,
+                               bool suppressLogging)
 {
-    QNetworkReply* reply = openConnection(baseUrl, command, arguments, enableTimeout);
+    QNetworkReply* reply = openConnection(baseUrl, command, arguments, enableTimeout, suppressLogging);
     QString ret;
 
     QTextStream stream(reply);
@@ -355,7 +359,8 @@ QNetworkReply*
 NvHTTP::openConnection(QUrl baseUrl,
                        QString command,
                        QString arguments,
-                       bool enableTimeout)
+                       bool enableTimeout,
+                       bool suppressLogging)
 {
     // Build a URL for the request
     QUrl url(baseUrl);
@@ -381,13 +386,17 @@ NvHTTP::openConnection(QUrl baseUrl,
     {
         QTimer::singleShot(REQUEST_TIMEOUT_MS, &loop, SLOT(quit()));
     }
-    qInfo() << "Executing request:" << url.toString();
+    if (!suppressLogging) {
+        qInfo() << "Executing request:" << url.toString();
+    }
     loop.exec(QEventLoop::ExcludeUserInputEvents);
 
     // Abort the request if it timed out
     if (!reply->isFinished())
     {
-        qWarning() << "Aborting timed out request for" << url.toString();
+        if (!suppressLogging) {
+            qWarning() << "Aborting timed out request for" << url.toString();
+        }
         reply->abort();
     }
 
@@ -398,7 +407,9 @@ NvHTTP::openConnection(QUrl baseUrl,
     // Handle error
     if (reply->error() != QNetworkReply::NoError)
     {
-        qWarning() << command << " request failed with error " << reply->error();
+        if (!suppressLogging) {
+            qWarning() << command << " request failed with error " << reply->error();
+        }
         GfeHttpResponseException exception(reply->error(), reply->errorString());
         delete reply;
         throw exception;

app/backend/nvhttp.h

@@ -107,7 +107,8 @@ public:
     openConnectionToString(QUrl baseUrl,
                            QString command,
                            QString arguments,
-                           bool enableTimeout);
+                           bool enableTimeout,
+                           bool suppressLogging = false);
 
     static
     QVector<int>
@@ -143,7 +144,8 @@ private:
     openConnection(QUrl baseUrl,
                    QString command,
                    QString arguments,
-                   bool enableTimeout);
+                   bool enableTimeout,
+                   bool suppressLogging = false);
 
     QString m_Address;
     QNetworkAccessManager m_Nam;

app/deploy/linux/com.moonlight_stream.Moonlight.appdata.xml

@@ -7,8 +7,16 @@
 	<summary>Stream games from your NVIDIA GameStream-enabled PC</summary>
 
 	<description>
-		<p>Moonlight can stream games or other applications from a PC with an NVIDIA GeForce GTX 600-series or higher GPU and GeForce Experience installed.</p>
+		<p>Moonlight can stream games and other applications from a PC with an NVIDIA GeForce GTX 600-series or higher GPU and GeForce Experience installed.</p>
-		<p>GameStream utilizes the NVENC hardware encoder present on NVIDIA GPUs for low-latency streaming at up 4K 60 FPS. Moonlight can utilize VAAPI hardware acceleration for high-performance H.264 and HEVC video decoding with low CPU usage.</p>
+		<p>Features include:</p>
+		<ul>
+			<li>Streaming at up to 4K resolution</li>
+			<li>Support for up to 120 FPS streaming on high refresh rate monitors</li>
+			<li>Hardware accelerated video decoding with VAAPI and VDPAU support</li>
+			<li>5.1 surround sound audio support</li>
+			<li>HEVC support for better video compression efficiency</li>
+			<li>Gamepad support with SDL mapping compatibility</li>
+		</ul>
 	</description>
 
 	<url type="homepage">https://moonlight-stream.com</url>
@@ -25,6 +33,25 @@
 	</screenshots>
 
 	<releases>
+		<release version="0.1.0" date="2018-08-05">
+			<description>
+				<p>New features:</p>
+				<ul>
+					<li>Allow streaming at the client's native resolution</li>
+					<li>VDPAU hardware decoding support for NVIDIA GPUs</li>
+					<li>Updated UI to replace status text with icons</li>
+					<li>Stream window is now resizable</li>
+					<li>Display warnings during game launch for configuration issues</li>
+				</ul>
+				<p>Bugfixes:</p>
+				<ul>
+					<li>Gamepad input now works even if the app doesn't have focus</li>
+					<li>Fixed streaming black screen if the host supported HEVC but the client didn't</li>
+					<li>Fixed broken video scaling causing extra stretching and off-center video</li>
+					<li>Fixed gamepad input issue causing some Y axis inputs to be reversed</li>
+				</ul>
+			</description>
+		</release>
 		<release version="0.0.5" date="2018-08-01">
 			<description>
 				<p>Changes from 0.0.4:</p>

app/gui/AppView.qml

@@ -61,19 +61,21 @@ GridView {
             }
         }
 
+        Image {
+            id: runningIcon
+            anchors.centerIn: appIcon
+            visible: model.running
+            source: "qrc:/res/baseline-play_circle_filled_white-48px.svg"
+            sourceSize {
+                width: 75
+                height: 75
+            }
+        }
+
         Text {
             id: appNameText
-            text: {
+            text: model.name
-                if (model.running) {
-                    return "<font color=\"green\">Running</font><font color=\"white\"> - </font>" + model.name
-                }
-                else {
-                    return model.name
-                }
-            }
-
             color: "white"
-
             width: parent.width
             height: 125
             anchors.top: appIcon.bottom

app/gui/PcView.qml

@@ -88,7 +88,7 @@ GridView {
 
         Image {
             id: pcIcon
-            anchors.horizontalCenter: parent.horizontalCenter;
+            anchors.horizontalCenter: parent.horizontalCenter
             source: {
                 model.addPc ? "qrc:/res/ic_add_to_queue_white_48px.svg" : "qrc:/res/ic_tv_white_48px.svg"
             }
@@ -98,6 +98,30 @@ GridView {
             }
         }
 
+        Image {
+            // TODO: Tooltip
+            id: stateIcon
+            anchors.horizontalCenter: pcIcon.horizontalCenter
+            anchors.verticalCenter: pcIcon.verticalCenter
+            anchors.verticalCenterOffset: -10
+            visible: !model.addPc && !model.statusUnknown && (!model.online || !model.paired)
+            source: !model.online ? "qrc:/res/baseline-warning-24px.svg" : "qrc:/res/baseline-lock-24px.svg"
+            sourceSize {
+                width: 75
+                height: 75
+            }
+        }
+
+        BusyIndicator {
+            id: statusUnknownSpinner
+            anchors.horizontalCenter: pcIcon.horizontalCenter
+            anchors.verticalCenter: pcIcon.verticalCenter
+            anchors.verticalCenterOffset: -10
+            width: 75
+            height: 75
+            visible: !model.addPc && model.statusUnknown
+        }
+
         Text {
             id: pcNameText
             text: model.name
@@ -110,44 +134,12 @@ GridView {
             wrapMode: Text.Wrap
         }
 
-        Text {
-            function getStatusText(model)
-            {
-                if (model.online) {
-                    var text = "<font color=\"green\">Online</font>"
-                    text += "<font color=\"white\"> - </font>"
-                    if (model.paired) {
-                        text += "<font color=\"skyblue\">Paired</font>"
-                    }
-                    else if (model.busy) {
-                        text += "<font color=\"red\">Busy</font>"
-                    }
-                    else {
-                        text += "<font color=\"red\">Not Paired</font>"
-                    }
-                    return text
-                }
-                else {
-                    return "<font color=\"red\">Offline</font>";
-                }
-            }
-
-            id: pcPairedText
-            text: getStatusText(model)
-            visible: !model.addPc
-
-            width: parent.width
-            anchors.top: pcNameText.bottom
-            font.pointSize: 24
-            horizontalAlignment: Text.AlignHCenter
-        }
-
         Menu {
             id: pcContextMenu
             MenuItem {
                 text: "Wake PC"
                 onTriggered: computerModel.wakeComputer(index)
-                visible: !model.online && model.wakeable
+                visible: !model.addPc && !model.online && model.wakeable
                 height: visible ? implicitHeight : 0
             }
             MenuItem {

app/gui/SettingsView.qml

@@ -43,7 +43,7 @@ ScrollView {
                 Label {
                     width: parent.width
                     id: resFPSdesc
-                    text: qsTr("Setting values too high for your PC may cause lag, stuttering, or errors")
+                    text: qsTr("Setting values too high for your PC may cause lag, stuttering, or errors.")
                     font.pointSize: 9
                     wrapMode: Text.Wrap
                     color: "white"
@@ -55,6 +55,41 @@ ScrollView {
                     ComboBox {
                         // ignore setting the index at first, and actually set it when the component is loaded
                         Component.onCompleted: {
+                            // Add native resolutions for all attached displays
+                            for (var displayIndex = 0;; displayIndex++) {
+                                var screenRect = prefs.getDisplayResolution(displayIndex)
+                                if (screenRect.width === 0) {
+                                    // Exceeded max count of displays
+                                    break
+                                }
+
+                                var indexToAdd = 0
+                                for (var j = 0; j < resolutionComboBox.count; j++) {
+                                    var existing_width = parseInt(resolutionListModel.get(j).video_width);
+                                    var existing_height = parseInt(resolutionListModel.get(j).video_height);
+
+                                    if (screenRect.width === existing_width && screenRect.height === existing_height) {
+                                        // Duplicate entry, skip
+                                        indexToAdd = -1
+                                        break
+                                    }
+                                    else if (screenRect.width * screenRect.height > existing_width * existing_height) {
+                                        // Candidate entrypoint after this entry
+                                        indexToAdd = j + 1
+                                    }
+                                }
+
+                                // Insert this display's resolution if it's not a duplicate
+                                if (indexToAdd >= 0) {
+                                    resolutionListModel.insert(indexToAdd,
+                                                               {
+                                                                   "text": "Native ("+screenRect.width+"x"+screenRect.height+")",
+                                                                   "video_width": ""+screenRect.width,
+                                                                   "video_height": ""+screenRect.height
+                                                               })
+                                }
+                            }
+
                             // load the saved width/height, and iterate through the ComboBox until a match is found
                             // and set it to that index.
                             var saved_width = prefs.width
@@ -63,17 +98,21 @@ ScrollView {
                             for (var i = 0; i < resolutionComboBox.count; i++) {
                                 var el_width = parseInt(resolutionListModel.get(i).video_width);
                                 var el_height = parseInt(resolutionListModel.get(i).video_height);
-                                if (saved_width === el_width && saved_height === el_height) {
+
+                                // Pick the highest value lesser or equal to the saved resolution
+                                if (saved_width * saved_height >= el_width * el_height) {
                                     currentIndex = i
                                 }
                             }
                         }
 
                         id: resolutionComboBox
-                        font.pointSize: 9
+                        width: 200
                         textRole: "text"
                         model: ListModel {
                             id: resolutionListModel
+                            // Other elements may be added at runtime
+                            // based on attached display resolution
                             ListElement {
                                 text: "720p"
                                 video_width: "1280"
@@ -121,14 +160,15 @@ ScrollView {
                             currentIndex = 0
                             for (var i = 0; i < fpsComboBox.count; i++) {
                                 var el_fps = parseInt(fpsListModel.get(i).video_fps);
-                                if (el_fps === saved_fps) {
+
+                                // Pick the highest value lesser or equal to the saved FPS
+                                if (saved_fps >= el_fps) {
                                     currentIndex = i
                                 }
                             }
                         }
 
                         id: fpsComboBox
-                        font.pointSize: 9
                         textRole: "text"
                         model: ListModel {
                             id: fpsListModel
@@ -238,7 +278,6 @@ ScrollView {
 
                     id: audioComboBox
                     width: Math.min(bitrateDesc.implicitWidth, parent.width)
-                    font.pointSize: 9
                     textRole: "text"
                     model: ListModel {
                         id: audioListModel
@@ -362,7 +401,6 @@ ScrollView {
 
                     id: decoderComboBox
                     width: Math.min(bitrateDesc.implicitWidth, parent.width)
-                    font.pointSize: 9
                     textRole: "text"
                     model: ListModel {
                         id: decoderListModel
@@ -409,7 +447,6 @@ ScrollView {
 
                     id: codecComboBox
                     width: Math.min(bitrateDesc.implicitWidth, parent.width)
-                    font.pointSize: 9
                     textRole: "text"
                     model: ListModel {
                         id: codecListModel

app/gui/StreamSegue.qml

@@ -47,7 +47,13 @@ Item {
 
     function displayLaunchWarning(text)
     {
-        // TODO: toast
+        // This toast appears for 3 seconds, just shorter than how long
+        // Session will wait for it to be displayed. This gives it time
+        // to transition to invisible before continuing.
+        var toast = Qt.createQmlObject('import QtQuick.Controls 2.3; ToolTip {}', parent, '')
+        toast.text = text
+        toast.timeout = 3000
+        toast.visible = true
     }
 
     onVisibleChanged: {

app/gui/computermodel.cpp

@@ -50,6 +50,8 @@ QVariant ComputerModel::data(const QModelIndex& index, int role) const
         return !computer->macAddress.isEmpty();
     case AddPcRole:
         return false;
+    case StatusUnknownRole:
+        return computer->state == NvComputer::CS_UNKNOWN;
     default:
         return QVariant();
     }
@@ -77,6 +79,7 @@ QHash<int, QByteArray> ComputerModel::roleNames() const
     names[BusyRole] = "busy";
     names[AddPcRole] = "addPc";
     names[WakeableRole] = "wakeable";
+    names[StatusUnknownRole] = "statusUnknown";
 
     return names;
 }

app/gui/computermodel.h

@@ -14,6 +14,7 @@ class ComputerModel : public QAbstractListModel
         PairedRole,
         BusyRole,
         WakeableRole,
+        StatusUnknownRole,
         AddPcRole
     };
 

app/res/baseline-lock-24px.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="#FFFFFF" width="24" height="24" viewBox="0 0 24 24">
+    <path d="M0 0h24v24H0z" fill="none"/>
+    <path d="M18 8h-1V6c0-2.76-2.24-5-5-5S7 3.24 7 6v2H6c-1.1 0-2 .9-2 2v10c0 1.1.9 2 2 2h12c1.1 0 2-.9 2-2V10c0-1.1-.9-2-2-2zm-6 9c-1.1 0-2-.9-2-2s.9-2 2-2 2 .9 2 2-.9 2-2 2zm3.1-9H8.9V6c0-1.71 1.39-3.1 3.1-3.1 1.71 0 3.1 1.39 3.1 3.1v2z"/>
+</svg>

app/res/baseline-play_circle_filled_white-48px.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" viewBox="0 0 48 48" fill="#fff">
+    <path d="M0 0h48v48H0z" fill="none"/>
+    <path d="M24 4C12.95 4 4 12.95 4 24s8.95 20 20 20 20-8.95 20-20S35.05 4 24 4zm-4 29V15l12 9-12 9z"/>
+</svg>

app/res/baseline-warning-24px.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="#FFFFFF" width="24" height="24" viewBox="0 0 24 24">
+    <path d="M0 0h24v24H0z" fill="none"/>
+    <path d="M1 21h22L12 2 1 21zm12-3h-2v-2h2v2zm0-4h-2v-4h2v4z"/>
+</svg>

app/resources.qrc

@@ -5,6 +5,9 @@
         <file>res/ic_videogame_asset_white_48px.svg</file>
         <file>res/ic_tv_white_48px.svg</file>
         <file>res/ic_add_to_queue_white_48px.svg</file>
+        <file>res/baseline-lock-24px.svg</file>
+        <file>res/baseline-play_circle_filled_white-48px.svg</file>
+        <file>res/baseline-warning-24px.svg</file>
         <file>res/no_app_image.png</file>
         <file>res/settings.svg</file>
         <file>res/arrow_left.svg</file>

app/settings/streamingpreferences.cpp

@@ -93,15 +93,50 @@ int StreamingPreferences::getMaximumStreamingFrameRate()
     return maxFrameRate;
 }
 
+QRect StreamingPreferences::getDisplayResolution(int displayIndex)
+{
+    if (SDL_InitSubSystem(SDL_INIT_VIDEO) != 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_InitSubSystem(SDL_INIT_VIDEO) failed: %s",
+                     SDL_GetError());
+        return QRect();
+    }
+
+    if (displayIndex >= SDL_GetNumVideoDisplays()) {
+        SDL_QuitSubSystem(SDL_INIT_VIDEO);
+        return QRect();
+    }
+
+    SDL_DisplayMode mode;
+    int err = SDL_GetCurrentDisplayMode(displayIndex, &mode);
+    SDL_QuitSubSystem(SDL_INIT_VIDEO);
+
+    if (err == 0) {
+        return QRect(0, 0, mode.w, mode.h);
+    }
+    else {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "SDL_GetCurrentDisplayMode() failed: %s",
+                     SDL_GetError());
+        return QRect();
+    }
+}
+
 int StreamingPreferences::getDefaultBitrate(int width, int height, int fps)
 {
-    if (width * height <= 1280 * 720) {
+    // This table prefers 16:10 resolutions because they are
+    // only slightly more pixels than the 16:9 equivalents, so
+    // we don't want to bump those 16:10 resolutions up to the
+    // next 16:9 slot.
+
+    // This covers 1280x720 and 1280x800 too
+    if (width * height <= 1366 * 768) {
         return static_cast<int>(5000 * (fps / 30.0));
     }
-    else if (width * height <= 1920 * 1080) {
+    else if (width * height <= 1920 * 1200) {
         return static_cast<int>(10000 * (fps / 30.0));
     }
-    else if (width * height <= 2560 * 1440) {
+    else if (width * height <= 2560 * 1600) {
         return static_cast<int>(20000 * (fps / 30.0));
     }
     else /* if (width * height <= 3840 * 2160) */ {

app/settings/streamingpreferences.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <QObject>
+#include <QRect>
 
 class StreamingPreferences : public QObject
 {
@@ -18,6 +19,8 @@ public:
 
     Q_INVOKABLE static int getMaximumStreamingFrameRate();
 
+    Q_INVOKABLE QRect getDisplayResolution(int displayIndex);
+
     void reload();
 
     enum AudioConfig

app/streaming/input.cpp

@@ -2,6 +2,8 @@
 #include <SDL.h>
 #include "streaming/session.hpp"
 
+#include <QtGlobal>
+
 #define VK_0 0x30
 #define VK_A 0x41
 
@@ -23,6 +25,9 @@ const int SdlInputHandler::k_ButtonMap[] = {
 SdlInputHandler::SdlInputHandler(bool multiController)
     : m_MultiController(multiController)
 {
+    // Allow gamepad input when the app doesn't have focus
+    SDL_SetHint(SDL_HINT_JOYSTICK_ALLOW_BACKGROUND_EVENTS, "1");
+
     // We need to reinit this each time, since you only get
     // an initial set of gamepad arrival events once per init.
     SDL_assert(!SDL_WasInit(SDL_INIT_GAMECONTROLLER));
@@ -438,13 +443,18 @@ void SdlInputHandler::handleControllerAxisEvent(SDL_ControllerAxisEvent* event)
             state->lsX = event->value;
             break;
         case SDL_CONTROLLER_AXIS_LEFTY:
-            state->lsY = -event->value;
+            // Signed values have one more negative value than
+            // positive value, so inverting the sign on -32768
+            // could actually cause the value to overflow and
+            // wrap around to be negative again. Avoid that by
+            // capping the value at 32767.
+            state->lsY = -qMax(event->value, (short)-32767);
             break;
         case SDL_CONTROLLER_AXIS_RIGHTX:
             state->rsX = event->value;
             break;
         case SDL_CONTROLLER_AXIS_RIGHTY:
-            state->rsY = -event->value;
+            state->rsY = -qMax(event->value, (short)-32767);
             break;
         case SDL_CONTROLLER_AXIS_TRIGGERLEFT:
             state->lt = (unsigned char)(event->value * 255UL / 32767);

app/streaming/session.cpp

@@ -138,6 +138,10 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
     }
 #endif
 
+#if !defined(HAVE_FFMPEG) && !defined(HAVE_SLVIDEO)
+#error No video decoding libraries available!
+#endif
+
     // If we reach this, we didn't initialize any decoders successfully
     return false;
 }
@@ -232,6 +236,8 @@ Session::Session(NvComputer* computer, NvApp& app)
       m_DecoderLock(0),
       m_NeedsIdr(false)
 {
+    qDebug() << "Server GPU:" << m_Computer->gpuModel;
+
     LiInitializeVideoCallbacks(&m_VideoCallbacks);
     m_VideoCallbacks.setup = drSetup;
     m_VideoCallbacks.submitDecodeUnit = drSubmitDecodeUnit;
@@ -240,7 +246,11 @@ Session::Session(NvComputer* computer, NvApp& app)
     m_VideoCallbacks.capabilities |= CAPABILITY_DIRECT_SUBMIT;
 
     // Slice up to 4 times for parallel decode, once slice per core
-    m_VideoCallbacks.capabilities |= CAPABILITY_SLICES_PER_FRAME(qMin(MAX_SLICES, SDL_GetCPUCount()));
+    int slices = qMin(MAX_SLICES, SDL_GetCPUCount());
+    m_VideoCallbacks.capabilities |= CAPABILITY_SLICES_PER_FRAME(slices);
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Encoder configured for %d slices per frame",
+                slices);
 
     LiInitializeStreamConfiguration(&m_StreamConfig);
     m_StreamConfig.width = m_Preferences.width;
@@ -248,6 +258,11 @@ Session::Session(NvComputer* computer, NvApp& app)
     m_StreamConfig.fps = m_Preferences.fps;
     m_StreamConfig.bitrate = m_Preferences.bitrateKbps;
     m_StreamConfig.hevcBitratePercentageMultiplier = 75;
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Video bitrate: %d kbps",
+                m_StreamConfig.bitrate);
+
     RAND_bytes(reinterpret_cast<unsigned char*>(m_StreamConfig.remoteInputAesKey),
                sizeof(m_StreamConfig.remoteInputAesKey));
 
@@ -313,10 +328,30 @@ Session::Session(NvComputer* computer, NvApp& app)
     }
 }
 
+void Session::emitLaunchWarning(QString text)
+{
+    // Emit the warning to the UI
+    emit displayLaunchWarning(text);
+
+    // Wait a little bit so the user can actually read what we just said.
+    // This wait is a little longer than the actual toast timeout (3 seconds)
+    // to allow it to transition off the screen before continuing.
+    uint32_t start = SDL_GetTicks();
+    while (!SDL_TICKS_PASSED(SDL_GetTicks(), start + 3500)) {
+        // Pump the UI loop while we wait
+        SDL_Delay(5);
+        QCoreApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
+    }
+}
+
 bool Session::validateLaunch()
 {
     QStringList warningList;
 
+    if (m_Preferences.videoDecoderSelection == StreamingPreferences::VDS_FORCE_SOFTWARE) {
+        emitLaunchWarning("Your settings selection to force software decoding may cause poor streaming performance.");
+    }
+
     if (m_StreamConfig.supportsHevc) {
         bool hevcForced = m_Preferences.videoCodecConfig == StreamingPreferences::VCC_FORCE_HEVC ||
                 m_Preferences.videoCodecConfig == StreamingPreferences::VCC_FORCE_HEVC_HDR;
@@ -325,22 +360,21 @@ bool Session::validateLaunch()
                                        VIDEO_FORMAT_H265,
                                        m_StreamConfig.width,
                                        m_StreamConfig.height,
-                                       m_StreamConfig.fps)) {
+                                       m_StreamConfig.fps) &&
-            // NOTE: HEVC currently uses only 1 slice regardless of what
+                m_Preferences.videoDecoderSelection != StreamingPreferences::VDS_FORCE_SOFTWARE) {
-            // we provide in CAPABILITY_SLICES_PER_FRAME(), so we should
-            // never use it for software decoding (unless common-c starts
-            // respecting it for HEVC).
-            m_StreamConfig.supportsHevc = false;
-
             if (hevcForced) {
-                emit displayLaunchWarning("This PC's GPU doesn't support HEVC decoding.");
+                emitLaunchWarning("Using software decoding due to your selection to force HEVC. This may cause poor streaming performance.");
+            }
+            else {
+                emitLaunchWarning("This PC's GPU doesn't support HEVC decoding.");
+                m_StreamConfig.supportsHevc = false;
             }
         }
 
         if (hevcForced) {
             if (m_Computer->maxLumaPixelsHEVC == 0) {
-                emit displayLaunchWarning("Your host PC GPU doesn't support HEVC. "
+                emitLaunchWarning("Your host PC GPU doesn't support HEVC. "
-                                          "A GeForce GTX 900-series (Maxwell) or later GPU is required for HEVC streaming.");
+                                  "A GeForce GTX 900-series (Maxwell) or later GPU is required for HEVC streaming.");
             }
         }
     }
@@ -351,19 +385,19 @@ bool Session::validateLaunch()
 
         // Check that the app supports HDR
         if (!m_App.hdrSupported) {
-            emit displayLaunchWarning(m_App.name + " doesn't support HDR10.");
+            emitLaunchWarning(m_App.name + " doesn't support HDR10.");
         }
         // Check that the server GPU supports HDR
         else if (!(m_Computer->serverCodecModeSupport & 0x200)) {
-            emit displayLaunchWarning("Your host PC GPU doesn't support HDR streaming. "
+            emitLaunchWarning("Your host PC GPU doesn't support HDR streaming. "
-                                      "A GeForce GTX 1000-series (Pascal) or later GPU is required for HDR streaming.");
+                              "A GeForce GTX 1000-series (Pascal) or later GPU is required for HDR streaming.");
         }
         else if (!isHardwareDecodeAvailable(m_Preferences.videoDecoderSelection,
                                             VIDEO_FORMAT_H265_MAIN10,
                                             m_StreamConfig.width,
                                             m_StreamConfig.height,
                                             m_StreamConfig.fps)) {
-            emit displayLaunchWarning("This PC's GPU doesn't support HEVC Main10 decoding for HDR streaming.");
+            emitLaunchWarning("This PC's GPU doesn't support HEVC Main10 decoding for HDR streaming.");
         }
         else {
             // TODO: Also validate display capabilites
@@ -376,7 +410,7 @@ bool Session::validateLaunch()
     if (m_StreamConfig.width >= 3840) {
         // Only allow 4K on GFE 3.x+
         if (m_Computer->gfeVersion.isNull() || m_Computer->gfeVersion.startsWith("2.")) {
-            emit displayLaunchWarning("GeForce Experience 3.0 or higher is required for 4K streaming.");
+            emitLaunchWarning("GeForce Experience 3.0 or higher is required for 4K streaming.");
 
             m_StreamConfig.width = 1920;
             m_StreamConfig.height = 1080;
@@ -384,8 +418,8 @@ bool Session::validateLaunch()
         // This list is sorted from least to greatest
         else if (m_Computer->displayModes.last().width < 3840 ||
                  (m_Computer->displayModes.last().refreshRate < 60 && m_StreamConfig.fps >= 60)) {
-            emit displayLaunchWarning("Your host PC GPU doesn't support 4K streaming. "
+            emitLaunchWarning("Your host PC GPU doesn't support 4K streaming. "
-                                      "A GeForce GTX 900-series (Maxwell) or later GPU is required for 4K streaming.");
+                              "A GeForce GTX 900-series (Maxwell) or later GPU is required for 4K streaming.");
 
             m_StreamConfig.width = 1920;
             m_StreamConfig.height = 1080;
@@ -416,6 +450,24 @@ void Session::getWindowDimensions(bool fullScreen,
 {
     int displayIndex = 0;
 
+    // If there's a display matching this exact resolution, pick that
+    // one (for native full-screen streaming). Otherwise, assume
+    // display 0 for now. TODO: Default to the screen that the Qt window is on
+    if (fullScreen) {
+        for (int i = 0; i < SDL_GetNumVideoDisplays(); i++) {
+            SDL_DisplayMode mode;
+            if (SDL_GetCurrentDisplayMode(i, &mode) == 0 &&
+                    m_ActiveVideoWidth == mode.w &&
+                    m_ActiveVideoHeight == mode.h) {
+                SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                            "Found exact resolution match on display: %d",
+                            i);
+                displayIndex = i;
+                break;
+            }
+        }
+    }
+
     if (m_Window != nullptr) {
         displayIndex = SDL_GetWindowDisplayIndex(m_Window);
         if (displayIndex < 0) {
@@ -471,6 +523,13 @@ void Session::getWindowDimensions(bool fullScreen,
 
                 width -= left + right;
                 height -= top + bottom;
+
+                // If the stream window can fit within the usable drawing area with 1:1
+                // scaling, do that rather than filling the screen.
+                if (m_StreamConfig.width < width && m_StreamConfig.height < height) {
+                    width = m_StreamConfig.width;
+                    height = m_StreamConfig.height;
+                }
             }
         }
         else {
@@ -495,6 +554,7 @@ void Session::toggleFullscreen()
     // that way we have accurate display size metrics (not the size our stream changed it to).
     if (!fullScreen) {
         SDL_SetWindowFullscreen(m_Window, 0);
+        SDL_SetWindowResizable(m_Window, SDL_TRUE);
     }
 
     getWindowDimensions(fullScreen, x, y, width, height);
@@ -503,6 +563,7 @@ void Session::toggleFullscreen()
     SDL_SetWindowSize(m_Window, width, height);
 
     if (fullScreen) {
+        SDL_SetWindowResizable(m_Window, SDL_FALSE);
         SDL_SetWindowFullscreen(m_Window, SDL_OS_FULLSCREEN_FLAG);
     }
 }
@@ -533,6 +594,18 @@ void Session::exec()
     Q_ASSERT(m_Computer->currentGameId == 0 ||
              m_Computer->currentGameId == m_App.id);
 
+    bool enableGameOptimizations = false;
+    for (const NvDisplayMode &mode : m_Computer->displayModes) {
+        if (mode.width == m_StreamConfig.width &&
+                mode.height == m_StreamConfig.height) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Found host supported resolution: %dx%d",
+                        mode.width, mode.height);
+            enableGameOptimizations = prefs.gameOptimizations;
+            break;
+        }
+    }
+
     try {
         NvHTTP http(m_Computer->activeAddress);
         if (m_Computer->currentGameId != 0) {
@@ -540,7 +613,7 @@ void Session::exec()
         }
         else {
             http.launchApp(m_App.id, &m_StreamConfig,
-                           prefs.gameOptimizations,
+                           enableGameOptimizations,
                            prefs.playAudioOnHost,
                            inputHandler.getAttachedGamepadMask());
         }
@@ -619,6 +692,10 @@ void Session::exec()
 
         SDL_SetWindowPosition(m_Window, x, y);
         SDL_SetWindowSize(m_Window, width, height);
+
+        // Passing SDL_WINDOW_RESIZABLE to set this during window
+        // creation causes our window to be full screen for some reason
+        SDL_SetWindowResizable(m_Window, SDL_TRUE);
     }
 
     QSvgRenderer svgIconRenderer(QString(":/res/moonlight.svg"));

app/streaming/session.hpp

@@ -39,6 +39,8 @@ signals:
 private:
     bool validateLaunch();
 
+    void emitLaunchWarning(QString text);
+
     int getDecoderCapabilities();
 
     int sdlDetermineAudioConfiguration();

app/streaming/streamutils.cpp

@@ -0,0 +1,20 @@
+#include "streamutils.h"
+
+void StreamUtils::scaleSourceToDestinationSurface(SDL_Rect* src, SDL_Rect* dst)
+{
+    int dstH = dst->w * src->h / src->w;
+    int dstW = dst->h * src->w / src->h;
+
+    if (dstH > dst->h) {
+        dst->y = 0;
+        dst->x = (dst->w - dstW) / 2;
+        dst->w = dstW;
+        SDL_assert(dst->w * src->h / src->w <= dst->h);
+    }
+    else {
+        dst->x = 0;
+        dst->y = (dst->h - dstH) / 2;
+        dst->h = dstH;
+        SDL_assert(dst->h * src->w / src->h <= dst->w);
+    }
+}

app/streaming/streamutils.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <SDL.h>
+
+class StreamUtils
+{
+public:
+    static
+    void scaleSourceToDestinationSurface(SDL_Rect* src, SDL_Rect* dst);
+};

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -1,5 +1,7 @@
 #include <initguid.h>
 #include "dxva2.h"
+#include "../ffmpeg.h"
+#include <streaming/streamutils.h>
 
 #include <Limelight.h>
 
@@ -75,11 +77,8 @@ bool DXVA2Renderer::prepareDecoderContext(AVCodecContext* context)
 
     context->hwaccel_context = &m_DXVAContext;
 
-    context->get_format = ffGetFormat;
     context->get_buffer2 = ffGetBuffer2;
 
-    context->opaque = this;
-
     m_Pool = av_buffer_pool_init2(ARRAYSIZE(m_DecSurfaces), this, ffPoolAlloc, nullptr);
     if (!m_Pool) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -95,7 +94,7 @@ bool DXVA2Renderer::prepareDecoderContext(AVCodecContext* context)
 
 int DXVA2Renderer::ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int)
 {
-    DXVA2Renderer* me = reinterpret_cast<DXVA2Renderer*>(context->opaque);
+    DXVA2Renderer* me = (DXVA2Renderer*)((FFmpegVideoDecoder*)context->opaque)->getRenderer();
 
     frame->buf[0] = av_buffer_pool_get(me->m_Pool);
     if (!frame->buf[0]) {
@@ -110,21 +109,6 @@ int DXVA2Renderer::ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int)
     return 0;
 }
 
-enum AVPixelFormat DXVA2Renderer::ffGetFormat(AVCodecContext*,
-                                              const enum AVPixelFormat* pixFmts)
-{
-    const enum AVPixelFormat *p;
-
-    for (p = pixFmts; *p != -1; p++) {
-        if (*p == AV_PIX_FMT_DXVA2_VLD)
-            return *p;
-    }
-
-    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                 "Failed to find DXVA2 HW surface format");
-    return AV_PIX_FMT_NONE;
-}
-
 bool DXVA2Renderer::initializeDecoder()
 {
     HRESULT hr;
@@ -215,18 +199,11 @@ bool DXVA2Renderer::initializeDecoder()
         return false;
     }
 
-    int alignment;
+    // Alignment was already taken care of
-
+    SDL_assert(m_Desc.SampleWidth % 16 == 0);
-    // HEVC using DXVA requires 128B alignment
+    SDL_assert(m_Desc.SampleHeight % 16 == 0);
-    if (m_VideoFormat & VIDEO_FORMAT_MASK_H265) {
+    hr = m_DecService->CreateSurface(m_Desc.SampleWidth,
-        alignment = 128;
+                                     m_Desc.SampleHeight,
-    }
-    else {
-        alignment = 16;
-    }
-
-    hr = m_DecService->CreateSurface(FFALIGN(m_VideoWidth, alignment),
-                                     FFALIGN(m_VideoHeight, alignment),
                                      ARRAYSIZE(m_DecSurfaces) - 1,
                                      m_Desc.Format,
                                      D3DPOOL_DEFAULT,
@@ -482,8 +459,19 @@ bool DXVA2Renderer::initialize(SDL_Window* window, int videoFormat, int width, i
     SDL_RenderPresent(m_SdlRenderer);
 
     RtlZeroMemory(&m_Desc, sizeof(m_Desc));
-    m_Desc.SampleWidth = m_VideoWidth;
+
-    m_Desc.SampleHeight = m_VideoHeight;
+    int alignment;
+
+    // HEVC using DXVA requires 128B alignment
+    if (m_VideoFormat & VIDEO_FORMAT_MASK_H265) {
+        alignment = 128;
+    }
+    else {
+        alignment = 16;
+    }
+
+    m_Desc.SampleWidth = FFALIGN(m_VideoWidth, alignment);
+    m_Desc.SampleHeight = FFALIGN(m_VideoHeight, alignment);
     m_Desc.SampleFormat.VideoChromaSubsampling = DXVA2_VideoChromaSubsampling_Unknown;
     m_Desc.SampleFormat.NominalRange = DXVA2_NominalRange_Unknown;
     m_Desc.SampleFormat.VideoTransferMatrix = DXVA2_VideoTransferMatrix_Unknown;
@@ -604,28 +592,26 @@ void DXVA2Renderer::renderFrame(AVFrame* frame)
     sample.Start = m_FrameIndex;
     sample.End = m_FrameIndex + 1;
     sample.SrcSurface = surface;
-    sample.SrcRect.right = m_Desc.SampleWidth;
+    sample.SrcRect.right = m_VideoWidth;
-    sample.SrcRect.bottom = m_Desc.SampleHeight;
+    sample.SrcRect.bottom = m_VideoHeight;
     sample.SampleFormat = m_Desc.SampleFormat;
     sample.PlanarAlpha = DXVA2_Fixed32OpaqueAlpha();
 
     // Center in frame and preserve aspect ratio
-    double srcAspectRatio = (double)m_Desc.SampleWidth / (double)m_Desc.SampleHeight;
+    SDL_Rect src, dst;
-    double dstAspectRatio = (double)m_DisplayWidth / (double)m_DisplayHeight;
+    src.x = src.y = 0;
-    if (dstAspectRatio < srcAspectRatio) {
+    src.w = m_VideoWidth;
-        // Greater height per width
+    src.h = m_VideoHeight;
-        int drawHeight = (int)(m_DisplayWidth / srcAspectRatio);
+    dst.x = dst.y = 0;
-        sample.DstRect.top = (m_DisplayHeight - drawHeight) / 2;
+    dst.w = m_DisplayWidth;
-        sample.DstRect.bottom = sample.DstRect.bottom + drawHeight;
+    dst.h = m_DisplayHeight;
-        sample.DstRect.right = m_DisplayWidth;
+
-    }
+    StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
-    else {
+
-        // Greater width per height
+    sample.DstRect.left = dst.x;
-        int drawWidth = (int)(m_DisplayHeight * srcAspectRatio);
+    sample.DstRect.right = dst.x + dst.w;
-        sample.DstRect.bottom = m_DisplayHeight;
+    sample.DstRect.top = dst.y;
-        sample.DstRect.left = (m_DisplayWidth - drawWidth) / 2;
+    sample.DstRect.bottom = dst.y + dst.h;
-        sample.DstRect.right = sample.DstRect.left + drawWidth;
-    }
 
     DXVA2_VideoProcessBltParams bltParams = {};
 

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -35,10 +35,6 @@ private:
     static
     int ffGetBuffer2(AVCodecContext* context, AVFrame* frame, int flags);
 
-    static
-    enum AVPixelFormat ffGetFormat(AVCodecContext*,
-                                   const enum AVPixelFormat* pixFmts);
-
     int m_VideoFormat;
     int m_VideoWidth;
     int m_VideoHeight;

app/streaming/video/ffmpeg-renderers/sdl.cpp

@@ -31,18 +31,10 @@ bool SdlRenderer::prepareDecoderContext(AVCodecContext*)
 }
 
 bool SdlRenderer::initialize(SDL_Window* window,
-                             int videoFormat,
+                             int,
                              int width,
                              int height)
 {
-    // NOTE: HEVC currently uses only 1 slice regardless of what
-    // we provide in CAPABILITY_SLICES_PER_FRAME(), so we should
-    // never use it for software decoding (unless common-c starts
-    // respecting it for HEVC).
-    if (videoFormat != VIDEO_FORMAT_H264) {
-        return false;
-    }
-
     m_Renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
     if (!m_Renderer) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -1,4 +1,5 @@
 #include "vaapi.h"
+#include <streaming/streamutils.h>
 
 #include <dlfcn.h>
 
@@ -70,7 +71,6 @@ VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
         return false;
     }
 
-
     err = av_hwdevice_ctx_create(&m_HwContext,
                                  AV_HWDEVICE_TYPE_VAAPI,
                                  nullptr, nullptr, 0);
@@ -81,6 +81,14 @@ VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
         return false;
     }
 
+    // This quirk is set for the VDPAU wrapper which doesn't work with our VAAPI renderer
+    if (((AVVAAPIDeviceContext*)((AVHWDeviceContext*)(m_HwContext->data))->hwctx)->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES) {
+        // Fail and let our VDPAU renderer pick this up
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Avoiding VDPAU wrapper for VAAPI decoding");
+        return false;
+    }
+
     return true;
 }
 
@@ -102,34 +110,23 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
     AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
     AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
 
-    // Center in frame and preserve aspect ratio
+    SDL_Rect src, dst;
-    int x, y, width, height;
+    src.x = src.y = 0;
-    double srcAspectRatio = (double)m_VideoWidth / (double)m_VideoHeight;
+    src.w = m_VideoWidth;
-    double dstAspectRatio = (double)m_DisplayWidth / (double)m_DisplayHeight;
+    src.h = m_VideoHeight;
-    if (dstAspectRatio < srcAspectRatio) {
+    dst.x = dst.y = 0;
-        // Greater height per width
+    dst.w = m_DisplayWidth;
-        int drawHeight = (int)(m_DisplayWidth / srcAspectRatio);
+    dst.h = m_DisplayHeight;
-        y = (m_DisplayHeight - drawHeight) / 2;
+
-        height = drawHeight;
+    StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
-        x = 0;
-        width = m_DisplayWidth;
-    }
-    else {
-        // Greater width per height
-        int drawWidth = (int)(m_DisplayHeight * srcAspectRatio);
-        y = 0;
-        height = m_DisplayHeight;
-        x = (m_DisplayWidth - drawWidth) / 2;
-        width = drawWidth;
-    }
 
     m_vaPutSurface(vaDeviceContext->display,
                    surface,
                    m_XWindow,
                    0, 0,
                    m_VideoWidth, m_VideoHeight,
-                   x, y,
+                   dst.x, dst.y,
-                   width, height,
+                   dst.w, dst.h,
                    NULL, 0, 0);
 
     av_frame_free(&frame);

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -47,4 +47,3 @@ private:
     int m_DisplayWidth;
     int m_DisplayHeight;
 };
-

app/streaming/video/ffmpeg-renderers/vdpau.cpp

@@ -0,0 +1,332 @@
+#include "vdpau.h"
+#include <streaming/streamutils.h>
+
+#include <SDL_syswm.h>
+
+#define BAIL_ON_FAIL(status, something) if ((status) != VDP_STATUS_OK) { \
+                                            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, \
+                                                        #something " failed: %d", (status)); \
+                                            return false; \
+                                        }
+
+#define GET_PROC_ADDRESS(id, func) status = vdpauCtx->get_proc_address(m_Device, id, (void**)func); \
+                                   BAIL_ON_FAIL(status, id)
+
+const VdpRGBAFormat VDPAURenderer::k_OutputFormats[] = {
+    VDP_RGBA_FORMAT_B8G8R8A8,
+    VDP_RGBA_FORMAT_R8G8B8A8
+};
+
+VDPAURenderer::VDPAURenderer()
+    : m_HwContext(nullptr),
+      m_PresentationQueueTarget(0),
+      m_PresentationQueue(0),
+      m_VideoMixer(0),
+      m_NextSurfaceIndex(0)
+{
+    SDL_zero(m_OutputSurface);
+}
+
+VDPAURenderer::~VDPAURenderer()
+{
+    if (m_PresentationQueue != 0) {
+        m_VdpPresentationQueueDestroy(m_PresentationQueue);
+    }
+
+    if (m_VideoMixer != 0) {
+        m_VdpVideoMixerDestroy(m_VideoMixer);
+    }
+
+    if (m_PresentationQueueTarget != 0) {
+        m_VdpPresentationQueueTargetDestroy(m_PresentationQueueTarget);
+    }
+
+    for (int i = 0; i < OUTPUT_SURFACE_COUNT; i++) {
+        if (m_OutputSurface[i] != 0) {
+            m_VdpOutputSurfaceDestroy(m_OutputSurface[i]);
+        }
+    }
+
+    // This must be done last as it frees VDPAU context required to call
+    // the functions above.
+    if (m_HwContext != nullptr) {
+        av_buffer_unref(&m_HwContext);
+    }
+}
+
+bool VDPAURenderer::initialize(SDL_Window* window, int, int width, int height)
+{
+    int err;
+    VdpStatus status;
+    SDL_SysWMinfo info;
+
+    m_VideoWidth = width;
+    m_VideoHeight = height;
+
+    err = av_hwdevice_ctx_create(&m_HwContext,
+                                 AV_HWDEVICE_TYPE_VDPAU,
+                                 nullptr, nullptr, 0);
+    if (err < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to create VDPAU context: %d",
+                     err);
+        return false;
+    }
+
+    AVHWDeviceContext* devCtx = (AVHWDeviceContext*)m_HwContext->data;
+    AVVDPAUDeviceContext* vdpauCtx = (AVVDPAUDeviceContext*)devCtx->hwctx;
+    m_Device = vdpauCtx->device;
+
+    GET_PROC_ADDRESS(VDP_FUNC_ID_GET_ERROR_STRING, &m_VdpGetErrorString);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_TARGET_DESTROY, &m_VdpPresentationQueueTargetDestroy);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_VIDEO_MIXER_CREATE, &m_VdpVideoMixerCreate);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_VIDEO_MIXER_DESTROY, &m_VdpVideoMixerDestroy);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_VIDEO_MIXER_RENDER, &m_VdpVideoMixerRender);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_CREATE, &m_VdpPresentationQueueCreate);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_DESTROY, &m_VdpPresentationQueueDestroy);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_DISPLAY, &m_VdpPresentationQueueDisplay);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_SET_BACKGROUND_COLOR, &m_VdpPresentationQueueSetBackgroundColor);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_BLOCK_UNTIL_SURFACE_IDLE, &m_VdpPresentationQueueBlockUntilSurfaceIdle);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_OUTPUT_SURFACE_CREATE, &m_VdpOutputSurfaceCreate);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_OUTPUT_SURFACE_DESTROY, &m_VdpOutputSurfaceDestroy);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_OUTPUT_SURFACE_QUERY_CAPABILITIES, &m_VdpOutputSurfaceQueryCapabilities);
+    GET_PROC_ADDRESS(VDP_FUNC_ID_VIDEO_SURFACE_GET_PARAMETERS, &m_VdpVideoSurfaceGetParameters);
+
+    SDL_GetWindowSize(window, (int*)&m_DisplayWidth, (int*)&m_DisplayHeight);
+
+    SDL_VERSION(&info.version);
+
+    if (!SDL_GetWindowWMInfo(window, &info)) {
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "SDL_GetWindowWMInfo() failed: %s",
+                    SDL_GetError());
+        return false;
+    }
+
+    SDL_assert(info.subsystem == SDL_SYSWM_X11);
+
+    if (info.subsystem == SDL_SYSWM_X11) {
+        GET_PROC_ADDRESS(VDP_FUNC_ID_PRESENTATION_QUEUE_TARGET_CREATE_X11,
+                         &m_VdpPresentationQueueTargetCreateX11);
+        status = m_VdpPresentationQueueTargetCreateX11(m_Device,
+                                                       info.info.x11.window,
+                                                       &m_PresentationQueueTarget);
+        if (status != VDP_STATUS_OK) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "VdpPresentationQueueTargetCreateX11() failed: %s",
+                         m_VdpGetErrorString(status));
+            return false;
+        }
+    }
+    else if (info.subsystem == SDL_SYSWM_WAYLAND) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VDPAU backend does not currently support Wayland");
+        return false;
+    }
+    else {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unsupported VDPAU rendering subsystem: %d",
+                     info.subsystem);
+        return false;
+    }
+
+    // Try our available output formats to find something the GPU supports
+    bool foundFormat = false;
+    for (int i = 0; i < OUTPUT_SURFACE_FORMAT_COUNT; i++) {
+        VdpBool supported;
+        uint32_t maxWidth, maxHeight;
+        status = m_VdpOutputSurfaceQueryCapabilities(m_Device, k_OutputFormats[i],
+                                                     &supported, &maxWidth, &maxHeight);
+        if (status != VDP_STATUS_OK) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "VdpOutputSurfaceQueryCapabilities() failed: %s",
+                         m_VdpGetErrorString(status));
+            return false;
+        }
+
+        if (supported) {
+            if (m_DisplayWidth <= maxWidth && m_DisplayHeight <= maxHeight) {
+                m_OutputSurfaceFormat = k_OutputFormats[i];
+                foundFormat = true;
+                break;
+            }
+            else  {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "Display size not within capabilities %dx%d vs %dx%d",
+                            m_DisplayWidth, m_DisplayWidth,
+                            maxWidth, maxHeight);
+            }
+        }
+    }
+
+    if (!foundFormat) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "No compatible output surface format found!");
+        return false;
+    }
+
+    // Create the output surfaces
+    for (int i = 0; i < OUTPUT_SURFACE_COUNT; i++) {
+        // It seems there's some lazy freeing going on or something in VDPAU
+        // because we can get VDP_STATUS_RESOURCES, then wait a bit and it'll
+        // complete without a problem.
+        int tries = 1;
+        do {
+            status = m_VdpOutputSurfaceCreate(m_Device, m_OutputSurfaceFormat,
+                                              m_DisplayWidth, m_DisplayHeight,
+                                              &m_OutputSurface[i]);
+            if (status != VDP_STATUS_OK) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "VdpOutputSurfaceCreate() try #%d: %s",
+                            tries,
+                            m_VdpGetErrorString(status));
+                SDL_Delay(250);
+            }
+        } while (status == VDP_STATUS_RESOURCES && ++tries <= 10);
+
+        if (status != VDP_STATUS_OK) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "VdpOutputSurfaceCreate() failed: %s",
+                         m_VdpGetErrorString(status));
+            return false;
+        }
+    }
+
+    status = m_VdpPresentationQueueCreate(m_Device, m_PresentationQueueTarget,
+                                          &m_PresentationQueue);
+    if (status != VDP_STATUS_OK) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VdpPresentationQueueCreate() failed: %s",
+                     m_VdpGetErrorString(status));
+        return false;
+    }
+
+    // Set the background to opaque black
+    VdpColor color = {0.0, 0.0, 0.0, 1.0};
+    m_VdpPresentationQueueSetBackgroundColor(m_PresentationQueue, &color);
+
+    return true;
+}
+
+bool VDPAURenderer::prepareDecoderContext(AVCodecContext* context)
+{
+    context->hw_device_ctx = av_buffer_ref(m_HwContext);
+
+    // Allow HEVC usage on VDPAU. This was disabled by FFmpeg due to
+    // GL interop issues, but we use VDPAU for rendering so it's no issue.
+    // https://github.com/FFmpeg/FFmpeg/commit/64ecb78b7179cab2dbdf835463104679dbb7c895
+    context->hwaccel_flags |= AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH;
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Using VDPAU accelerated renderer");
+
+    return true;
+}
+
+void VDPAURenderer::renderFrame(AVFrame* frame)
+{
+    VdpStatus status;
+    VdpVideoSurface videoSurface = (VdpVideoSurface)(uintptr_t)frame->data[3];
+
+    // This is safe without locking because this is always called on the main thread
+    VdpOutputSurface chosenSurface = m_OutputSurface[m_NextSurfaceIndex];
+    m_NextSurfaceIndex = (m_NextSurfaceIndex + 1) % OUTPUT_SURFACE_COUNT;
+
+    // We need to create the mixer on the fly, because we don't know the dimensions
+    // of our video surfaces in advance of decoding
+    if (m_VideoMixer == 0) {
+        VdpChromaType chroma;
+        uint32_t videoSurfaceWidth, videoSurfaceHeight;
+        status = m_VdpVideoSurfaceGetParameters(videoSurface, &chroma, &videoSurfaceWidth, &videoSurfaceHeight);
+        if (status != VDP_STATUS_OK) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "VdpVideoSurfaceGetParameters() failed: %s",
+                         m_VdpGetErrorString(status));
+            av_frame_free(&frame);
+            return;
+        }
+
+        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                    "VDPAU surface size: %dx%d",
+                    videoSurfaceWidth, videoSurfaceHeight);
+
+    #define PARAM_COUNT 2
+        const VdpVideoMixerParameter params[PARAM_COUNT] = {
+            VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_WIDTH,
+            VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_HEIGHT,
+        };
+        const void* const paramValues[PARAM_COUNT] = {
+            &videoSurfaceWidth,
+            &videoSurfaceHeight,
+        };
+
+        status = m_VdpVideoMixerCreate(m_Device, 0, nullptr,
+                                       PARAM_COUNT, params, paramValues,
+                                       &m_VideoMixer);
+        if (status != VDP_STATUS_OK) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "VdpVideoMixerCreate() failed: %s",
+                         m_VdpGetErrorString(status));
+            av_frame_free(&frame);
+            return;
+        }
+    }
+
+    // Wait for this frame to be off the screen
+    VdpTime pts;
+    m_VdpPresentationQueueBlockUntilSurfaceIdle(m_PresentationQueue, chosenSurface, &pts);
+
+    VdpRect sourceRect, outputRect;
+
+    SDL_Rect src, dst;
+    src.x = src.y = 0;
+    src.w = m_VideoWidth;
+    src.h = m_VideoHeight;
+    dst.x = dst.y = 0;
+    dst.w = m_DisplayWidth;
+    dst.h = m_DisplayHeight;
+
+    StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
+
+    outputRect.x0 = dst.x;
+    outputRect.x1 = dst.x + dst.w;
+    outputRect.y0 = dst.y;
+    outputRect.y1 = dst.y + dst.h;
+
+    sourceRect.x0 = sourceRect.y0 = 0;
+    sourceRect.x1 = m_VideoWidth;
+    sourceRect.y1 = m_VideoHeight;
+
+    // Render the next frame into the output surface
+    status = m_VdpVideoMixerRender(m_VideoMixer,
+                                   VDP_INVALID_HANDLE, nullptr,
+                                   VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME,
+                                   0, nullptr,
+                                   videoSurface,
+                                   0, nullptr,
+                                   &sourceRect,
+                                   chosenSurface,
+                                   &outputRect,
+                                   nullptr,
+                                   0,
+                                   nullptr);
+
+    // The decoder can have this surface back now
+    av_frame_free(&frame);
+
+    if (status != VDP_STATUS_OK) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VdpVideoMixerRender() failed: %s",
+                     m_VdpGetErrorString(status));
+        return;
+    }
+
+    // Queue the frame for display immediately
+    status = m_VdpPresentationQueueDisplay(m_PresentationQueue, chosenSurface, 0, 0, 0);
+    if (status != VDP_STATUS_OK) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VdpPresentationQueueDisplay() failed: %s",
+                     m_VdpGetErrorString(status));
+        return;
+    }
+}

app/streaming/video/ffmpeg-renderers/vdpau.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "renderer.h"
+
+extern "C" {
+#include <vdpau/vdpau.h>
+#include <vdpau/vdpau_x11.h>
+#include <libavutil/hwcontext_vdpau.h>
+}
+
+class VDPAURenderer : public IFFmpegRenderer
+{
+public:
+    VDPAURenderer();
+    virtual ~VDPAURenderer();
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int width,
+                            int height);
+    virtual bool prepareDecoderContext(AVCodecContext* context);
+    virtual void renderFrame(AVFrame* frame);
+
+private:
+    uint32_t m_VideoWidth, m_VideoHeight;
+    uint32_t m_DisplayWidth, m_DisplayHeight;
+    AVBufferRef* m_HwContext;
+    VdpPresentationQueueTarget m_PresentationQueueTarget;
+    VdpPresentationQueue m_PresentationQueue;
+    VdpVideoMixer m_VideoMixer;
+    VdpRGBAFormat m_OutputSurfaceFormat;
+    VdpDevice m_Device;
+
+#define OUTPUT_SURFACE_COUNT 3
+    VdpOutputSurface m_OutputSurface[OUTPUT_SURFACE_COUNT];
+    int m_NextSurfaceIndex;
+
+#define OUTPUT_SURFACE_FORMAT_COUNT 2
+    static const VdpRGBAFormat k_OutputFormats[OUTPUT_SURFACE_FORMAT_COUNT];
+
+    VdpGetErrorString* m_VdpGetErrorString;
+    VdpPresentationQueueTargetDestroy* m_VdpPresentationQueueTargetDestroy;
+    VdpVideoMixerCreate* m_VdpVideoMixerCreate;
+    VdpVideoMixerDestroy* m_VdpVideoMixerDestroy;
+    VdpVideoMixerRender* m_VdpVideoMixerRender;
+    VdpPresentationQueueCreate* m_VdpPresentationQueueCreate;
+    VdpPresentationQueueDestroy* m_VdpPresentationQueueDestroy;
+    VdpPresentationQueueDisplay* m_VdpPresentationQueueDisplay;
+    VdpPresentationQueueSetBackgroundColor* m_VdpPresentationQueueSetBackgroundColor;
+    VdpPresentationQueueBlockUntilSurfaceIdle* m_VdpPresentationQueueBlockUntilSurfaceIdle;
+    VdpOutputSurfaceCreate* m_VdpOutputSurfaceCreate;
+    VdpOutputSurfaceDestroy* m_VdpOutputSurfaceDestroy;
+    VdpOutputSurfaceQueryCapabilities* m_VdpOutputSurfaceQueryCapabilities;
+    VdpVideoSurfaceGetParameters* m_VdpVideoSurfaceGetParameters;
+
+    // X11 stuff
+    VdpPresentationQueueTargetCreateX11* m_VdpPresentationQueueTargetCreateX11;
+};
+
+

app/streaming/video/ffmpeg.cpp

@@ -13,93 +13,39 @@
 #include "ffmpeg-renderers/vaapi.h"
 #endif
 
-bool FFmpegVideoDecoder::chooseDecoder(
+#ifdef HAVE_LIBVDPAU
-        StreamingPreferences::VideoDecoderSelection vds,
+#include "ffmpeg-renderers/vdpau.h"
-        SDL_Window* window,
-        int videoFormat,
-        int width, int height,
-        AVCodec*& chosenDecoder,
-        const AVCodecHWConfig*& chosenHwConfig,
-        IFFmpegRenderer*& newRenderer)
-{
-    if (videoFormat & VIDEO_FORMAT_MASK_H264) {
-        chosenDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
-    }
-    else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
-        chosenDecoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
-    }
-    else {
-        Q_ASSERT(false);
-        chosenDecoder = nullptr;
-    }
-
-    if (!chosenDecoder) {
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "Unable to find decoder for format: %x",
-                     videoFormat);
-        return false;
-    }
-
-    for (int i = 0;; i++) {
-        const AVCodecHWConfig *config = avcodec_get_hw_config(chosenDecoder, i);
-        if (!config || vds == StreamingPreferences::VDS_FORCE_SOFTWARE) {
-            // No matching hardware acceleration support.
-            // This is not an error.
-            chosenHwConfig = nullptr;
-            newRenderer = new SdlRenderer();
-            if (vds != StreamingPreferences::VDS_FORCE_HARDWARE &&
-                    newRenderer->initialize(window, videoFormat, width, height)) {
-                return true;
-            }
-            else {
-                delete newRenderer;
-                newRenderer = nullptr;
-                return false;
-            }
-        }
-
-        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
-            continue;
-        }
-
-        // Look for acceleration types we support
-        switch (config->device_type) {
-#ifdef Q_OS_WIN32
-        case AV_HWDEVICE_TYPE_DXVA2:
-            newRenderer = new DXVA2Renderer();
-            break;
 #endif
-#ifdef Q_OS_DARWIN
-        case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
-            newRenderer = VTRendererFactory::createRenderer();
-            break;
-#endif
-#ifdef HAVE_LIBVA
-        case AV_HWDEVICE_TYPE_VAAPI:
-            newRenderer = new VAAPIRenderer();
-            break;
-#endif
-        default:
-            continue;
-        }
 
-        if (newRenderer->initialize(window, videoFormat, width, height)) {
+// This is gross but it allows us to use sizeof()
-            chosenHwConfig = config;
+#include "ffmpeg_videosamples.cpp"
-            return true;
-        }
-        else {
-            // Failed to initialize
-            delete newRenderer;
-            newRenderer = nullptr;
-        }
-    }
-}
 
 bool FFmpegVideoDecoder::isHardwareAccelerated()
 {
     return m_HwDecodeCfg != nullptr;
 }
 
+enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
+                                                   const enum AVPixelFormat* pixFmts)
+{
+    FFmpegVideoDecoder* decoder = (FFmpegVideoDecoder*)context->opaque;
+    const enum AVPixelFormat *p;
+
+    for (p = pixFmts; *p != -1; p++) {
+        // Only match our hardware decoding codec or SW pixel
+        // format (if not using hardware decoding). It's crucial
+        // to override the default get_format() which will try
+        // to gracefully fall back to software decode and break us.
+        if (*p == (decoder->m_HwDecodeCfg ?
+                   decoder->m_HwDecodeCfg->pix_fmt :
+                   context->pix_fmt)) {
+            return *p;
+        }
+    }
+
+    return AV_PIX_FMT_NONE;
+}
+
 FFmpegVideoDecoder::FFmpegVideoDecoder()
     : m_VideoDecoderCtx(nullptr),
       m_DecodeBuffer(1024 * 1024, 0),
@@ -114,6 +60,16 @@ FFmpegVideoDecoder::FFmpegVideoDecoder()
 }
 
 FFmpegVideoDecoder::~FFmpegVideoDecoder()
+{
+    reset();
+}
+
+IFFmpegRenderer* FFmpegVideoDecoder::getRenderer()
+{
+    return m_Renderer;
+}
+
+void FFmpegVideoDecoder::reset()
 {
     // Drop any frames still queued to ensure
     // they are properly freed.
@@ -140,24 +96,11 @@ FFmpegVideoDecoder::~FFmpegVideoDecoder()
     avcodec_free_context(&m_VideoDecoderCtx);
 
     delete m_Renderer;
+    m_Renderer = nullptr;
 }
 
-bool FFmpegVideoDecoder::initialize(
+bool FFmpegVideoDecoder::completeInitialization(AVCodec* decoder, int videoFormat, int width, int height, bool testOnly)
-        StreamingPreferences::VideoDecoderSelection vds,
-        SDL_Window* window,
-        int videoFormat,
-        int width,
-        int height,
-        int)
 {
-    AVCodec* decoder;
-
-    if (!chooseDecoder(vds, window, videoFormat, width, height,
-                       decoder, m_HwDecodeCfg, m_Renderer)) {
-        // Error logged in chooseDecoder()
-        return false;
-    }
-
     m_VideoDecoderCtx = avcodec_alloc_context3(decoder);
     if (!m_VideoDecoderCtx) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -182,12 +125,20 @@ bool FFmpegVideoDecoder::initialize(
     m_VideoDecoderCtx->width = width;
     m_VideoDecoderCtx->height = height;
     m_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
+    m_VideoDecoderCtx->get_format = ffGetFormat;
 
     // Allow the renderer to attach data to this decoder
     if (!m_Renderer->prepareDecoderContext(m_VideoDecoderCtx)) {
         return false;
     }
 
+    // Nobody must override our ffGetFormat
+    SDL_assert(m_VideoDecoderCtx->get_format == ffGetFormat);
+
+    // Stash a pointer to this object in the context
+    SDL_assert(m_VideoDecoderCtx->opaque == nullptr);
+    m_VideoDecoderCtx->opaque = this;
+
     int err = avcodec_open2(m_VideoDecoderCtx, decoder, nullptr);
     if (err < 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -196,9 +147,148 @@ bool FFmpegVideoDecoder::initialize(
         return false;
     }
 
+    // FFMpeg doesn't completely initialize the codec until the codec
+    // config data comes in. This would be too late for us to change
+    // our minds on the selected video codec, so we'll do a trial run
+    // now to see if things will actually work when the video stream
+    // comes in.
+    if (testOnly) {
+        if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+            m_Pkt.data = (uint8_t*)k_H264TestFrame;
+            m_Pkt.size = sizeof(k_H264TestFrame);
+        }
+        else {
+            m_Pkt.data = (uint8_t*)k_HEVCTestFrame;
+            m_Pkt.size = sizeof(k_HEVCTestFrame);
+        }
+
+        err = avcodec_send_packet(m_VideoDecoderCtx, &m_Pkt);
+        if (err < 0) {
+            char errorstring[512];
+            av_strerror(err, errorstring, sizeof(errorstring));
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "Test decode failed: %s", errorstring);
+            return false;
+        }
+    }
+
+#ifdef QT_DEBUG
+    // Restore default log level before streaming
+    av_log_set_level(AV_LOG_INFO);
+#endif
+
     return true;
 }
 
+IFFmpegRenderer* FFmpegVideoDecoder::createAcceleratedRenderer(const AVCodecHWConfig* hwDecodeCfg)
+{
+    if (!(hwDecodeCfg->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
+        return nullptr;
+    }
+
+    // Look for acceleration types we support
+    switch (hwDecodeCfg->device_type) {
+#ifdef Q_OS_WIN32
+    case AV_HWDEVICE_TYPE_DXVA2:
+        return new DXVA2Renderer();
+#endif
+#ifdef Q_OS_DARWIN
+    case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
+        return VTRendererFactory::createRenderer();
+#endif
+#ifdef HAVE_LIBVA
+    case AV_HWDEVICE_TYPE_VAAPI:
+        return new VAAPIRenderer();
+#endif
+#ifdef HAVE_LIBVDPAU
+    case AV_HWDEVICE_TYPE_VDPAU:
+        return new VDPAURenderer();
+#endif
+    default:
+        return nullptr;
+    }
+}
+
+bool FFmpegVideoDecoder::initialize(
+        StreamingPreferences::VideoDecoderSelection vds,
+        SDL_Window* window,
+        int videoFormat,
+        int width,
+        int height,
+        int)
+{
+    AVCodec* decoder;
+
+#ifdef QT_DEBUG
+    // Increase log level during initialization
+    av_log_set_level(AV_LOG_DEBUG);
+#endif
+
+    if (videoFormat & VIDEO_FORMAT_MASK_H264) {
+        decoder = avcodec_find_decoder(AV_CODEC_ID_H264);
+    }
+    else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
+        decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
+    }
+    else {
+        Q_ASSERT(false);
+        decoder = nullptr;
+    }
+
+    if (!decoder) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unable to find decoder for format: %x",
+                     videoFormat);
+        return false;
+    }
+
+    for (int i = 0;; i++) {
+        const AVCodecHWConfig *config = avcodec_get_hw_config(decoder, i);
+        if (!config || vds == StreamingPreferences::VDS_FORCE_SOFTWARE) {
+            // No matching hardware acceleration support.
+            // This is not an error.
+            m_HwDecodeCfg = nullptr;
+            m_Renderer = new SdlRenderer();
+            if (vds != StreamingPreferences::VDS_FORCE_HARDWARE &&
+                    m_Renderer->initialize(window, videoFormat, width, height) &&
+                    completeInitialization(decoder, videoFormat, width, height, false)) {
+                return true;
+            }
+            else {
+                reset();
+                return false;
+            }
+        }
+
+        m_Renderer = createAcceleratedRenderer(config);
+        if (!m_Renderer) {
+            continue;
+        }
+
+        m_HwDecodeCfg = config;
+        // Submit test frame to ensure this codec really works
+        if (m_Renderer->initialize(window, videoFormat, width, height) &&
+                completeInitialization(decoder, videoFormat, width, height, true)) {
+            // OK, it worked, so now let's initialize it for real
+            reset();
+            if ((m_Renderer = createAcceleratedRenderer(config)) != nullptr &&
+                    m_Renderer->initialize(window, videoFormat, width, height) &&
+                    completeInitialization(decoder, videoFormat, width, height, false)) {
+                return true;
+            }
+            else {
+                SDL_LogCritical(SDL_LOG_CATEGORY_APPLICATION,
+                                "Decoder failed to initialize after successful test");
+                reset();
+            }
+        }
+        else {
+            // Failed to initialize or test frame failed, so keep looking
+            reset();
+        }
+    }
+}
+
 int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
 {
     PLENTRY entry = du->bufferList;
@@ -224,8 +314,6 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
 
     err = avcodec_send_packet(m_VideoDecoderCtx, &m_Pkt);
     if (err < 0) {
-        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
-                    "Decoding failed: %d", err);
         char errorstring[512];
         av_strerror(err, errorstring, sizeof(errorstring));
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,

app/streaming/video/ffmpeg.h

@@ -22,16 +22,19 @@ public:
     virtual void renderFrame(SDL_UserEvent* event) override;
     virtual void dropFrame(SDL_UserEvent* event) override;
 
+    virtual IFFmpegRenderer* getRenderer();
+
 private:
-    bool
+    bool completeInitialization(AVCodec* decoder, int videoFormat,
-    chooseDecoder(
+                                int width, int height, bool testOnly);
-        StreamingPreferences::VideoDecoderSelection vds,
+
-        SDL_Window* window,
+    IFFmpegRenderer* createAcceleratedRenderer(const AVCodecHWConfig* hwDecodeCfg);
-        int videoFormat,
+
-        int width, int height,
+    void reset();
-        AVCodec*& chosenDecoder,
+
-        const AVCodecHWConfig*& chosenHwConfig,
+    static
-        IFFmpegRenderer*& newRenderer);
+    enum AVPixelFormat ffGetFormat(AVCodecContext* context,
+                                   const enum AVPixelFormat* pixFmts);
 
     AVPacket m_Pkt;
     AVCodecContext* m_VideoDecoderCtx;
@@ -39,4 +42,7 @@ private:
     const AVCodecHWConfig* m_HwDecodeCfg;
     IFFmpegRenderer* m_Renderer;
     SDL_atomic_t m_QueuedFrames;
+
+    static const uint8_t k_H264TestFrame[];
+    static const uint8_t k_HEVCTestFrame[];
 };

app/streaming/video/ffmpeg_videosamples.cpp

@@ -0,0 +1,19 @@
+#include "ffmpeg.h"
+
+// 720p 60 FPS H.264 with 1 reference frame
+const uint8_t FFmpegVideoDecoder::k_H264TestFrame[] = {
+    0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20, 0xac, 0x2b, 0x40, 0x28, 0x02, 0xdd, 0x80, 0xb5, 0x06, 0x06, 0x06, 0xa5, 0x00, 0x00, 0x03, 0x03, 0xe8, 0x00, 0x01, 0xd4, 0xc0, 0x8f, 0x4a, 0xa0,
+    0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0xb0,
+    0x00, 0x00, 0x00, 0x01, 0x65, 0xb8, 0x02, 0x01, 0x67, 0x25, 0x1b, 0xf4, 0xfa, 0x7d, 0x40, 0x1a, 0x78, 0xe5, 0x10, 0x52, 0xc2, 0xee, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0xc6, 0xef, 0xbb, 0x81, 0x85, 0x2d, 0x47, 0xda, 0xca, 0x4c, 0x00, 0x00, 0x03, 0x00, 0x02, 0x7b, 0xcf, 0x80, 0x00, 0x45, 0x40, 0x01, 0x8d, 0xa6, 0x00, 0x01, 0x64, 0x00, 0x0e, 0x03, 0xc8, 0x00, 0x0e, 0x10, 0x00, 0xbd, 0xc5, 0x00, 0x01, 0x11, 0x00, 0x0e, 0xa3, 0x80, 0x00, 0x38, 0xa0,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x6e, 0x2e, 0x00, 0x83, 0x7f, 0xb4, 0x8e, 0x79, 0xa5, 0xff, 0x84, 0x3f, 0x7f, 0x34, 0x3f, 0x97, 0x1b, 0xaf, 0x31, 0x5f, 0x6e, 0xaa, 0xb6, 0xac, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x78, 0x36, 0x9d, 0xa4, 0x65, 0xf9, 0x1e, 0x5b, 0x3a, 0xb0, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x41, 0x80, 0x00, 0xc5, 0xc8, 0x00, 0x00, 0xfa, 0x00, 0x03, 0x24, 0x00, 0x0e, 0x20, 0x00, 0x3f, 0x80, 0x01, 0x32, 0x00, 0x08, 0x68, 0x00, 0x3e, 0xc0, 0x03, 0x8a,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x37, 0x0b, 0x80, 0x21, 0x7f, 0xe3, 0x85, 0x1c, 0xd9, 0xff, 0xe1, 0x1b, 0x01, 0xfa, 0xc0, 0x3e, 0x11, 0x7e, 0xfe, 0x45, 0x5c, 0x43, 0x69, 0x31, 0x4b, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x02, 0x24, 0xae, 0x1a, 0xbb, 0xae, 0x75, 0x9e, 0x35, 0xae, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x03, 0x64, 0x00, 0x09, 0x98, 0x00, 0x1e, 0xc0, 0x00, 0x64, 0x80, 0x01, 0xc4, 0x00, 0x07, 0xf0, 0x00, 0x42, 0xf0, 0x00, 0x00, 0xe1, 0x98, 0x00, 0x05, 0x4b, 0x28, 0x00, 0x06, 0x04,
+    0x00, 0x00, 0x01, 0x65, 0x00, 0x14, 0xa2, 0xe0, 0x08, 0x5f, 0xe3, 0x85, 0x1c, 0xd9, 0xff, 0xe1, 0x1b, 0x01, 0xfa, 0xc0, 0x3e, 0x11, 0x7e, 0xfe, 0x45, 0x5c, 0x43, 0x69, 0x31, 0x4b, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x02, 0x24, 0xb9, 0x7d, 0xb4, 0x70, 0x4d, 0x15, 0xc5, 0x0a, 0x4e, 0x60, 0x00, 0x00, 0x03, 0x00, 0x00, 0x26, 0xa8, 0xb0, 0x00, 0x13, 0xcd, 0xcc, 0x00, 0x07, 0x48, 0x88, 0x00, 0x06, 0x29, 0x69, 0x00, 0x01, 0x16, 0xac, 0x80, 0x00, 0x9e, 0x4e, 0x80, 0x00, 0x3f, 0x84, 0x20, 0x00, 0x6f, 0x41, 0xa0, 0x00, 0x20, 0x00, 0x02, 0x16, 0xb8, 0x00, 0x08, 0x08
+};
+
+// 720p 60 FPS HEVC with 1 reference frame
+const uint8_t FFmpegVideoDecoder::k_HEVCTestFrame[] = {
+    0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xac, 0x09,
+    0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x02, 0x80, 0x80, 0x2e, 0x1f, 0x13, 0x96, 0xb4, 0xa4, 0x25, 0x92, 0xe3, 0x01, 0x6a, 0x0c, 0x0c, 0x0d, 0x48, 0x20, 0x00, 0x00, 0x03, 0x00, 0x20, 0x00, 0x00, 0x07, 0x85, 0xf1, 0xa2, 0xd0,
+    0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0xf7, 0xc0, 0xcc, 0x90,
+    0x00, 0x00, 0x00, 0x01, 0x2a, 0x01, 0xac, 0x0c, 0xf0, 0xe6, 0xf6, 0x5e, 0xff, 0xd9, 0x97, 0xeb, 0x44, 0x7d, 0x1f, 0xa4, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x03, 0x52, 0x7c, 0x4e, 0x5b, 0xfe, 0xbe, 0xc2, 0x87, 0x20, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x2b, 0x60,
+};

scripts/build-ffmpeg-mac.sh

@@ -0,0 +1,3 @@
+./configure --extra-cflags="-mmacosx-version-min=10.10" --enable-pic --enable-shared --disable-static --disable-all --enable-avcodec --enable-decoder=h264 --enable-decoder=hevc --enable-hwaccel=h264_videotoolbox --enable-hwaccel=hevc_videotoolbox
+make clean
+make -j4

scripts/build-ffmpeg-win32.sh

@@ -0,0 +1,3 @@
+./configure --toolchain=msvc --enable-cross-compile --enable-pic --enable-shared --disable-static --disable-all --enable-avcodec --enable-decoder=h264 --enable-decoder=hevc --enable-hwaccel=h264_dxva2 --enable-hwaccel=hevc_dxva2
+make clean
+make -j4

scripts/build-ffmpeg-win64.sh

@@ -0,0 +1,3 @@
+./configure --toolchain=msvc --enable-pic --enable-shared --disable-static --disable-all --enable-avcodec --enable-decoder=h264 --enable-decoder=hevc --enable-hwaccel=h264_dxva2 --enable-hwaccel=hevc_dxva2
+make clean
+make -j4