Add performance overlay for DXVA2 renderer

app/app.pro

@@ -37,13 +37,17 @@ DEFINES += QT_DEPRECATED_WARNINGS
 DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0
 
 win32 {
-    INCLUDEPATH += $$PWD/../libs/windows/include
+    INCLUDEPATH += \
+        $$PWD/../libs/windows/include \
+        $$(DXSDK_DIR)/Include
 
     contains(QT_ARCH, i386) {
         LIBS += -L$$PWD/../libs/windows/lib/x86
+        LIBS += -L$$(DXSDK_DIR)/Lib/x86
     }
     contains(QT_ARCH, x86_64) {
         LIBS += -L$$PWD/../libs/windows/lib/x64
+        LIBS += -L$$(DXSDK_DIR)/Lib/x64
     }
 
     LIBS += ws2_32.lib winmm.lib dxva2.lib ole32.lib gdi32.lib user32.lib d3d9.lib dwmapi.lib dbghelp.lib
@@ -85,7 +89,7 @@ unix:!macx {
     }
 }
 win32 {
-    LIBS += -llibssl -llibcrypto -lSDL2 -lavcodec -lavutil -lopus
+    LIBS += -llibssl -llibcrypto -lSDL2 -lavcodec -lavutil -lopus -ld3dx9
     CONFIG += ffmpeg soundio
 }
 macx {
@@ -121,7 +125,8 @@ SOURCES += \
     backend/autoupdatechecker.cpp \
     path.cpp \
     settings/mappingmanager.cpp \
-    gui/sdlgamepadkeynavigation.cpp
+    gui/sdlgamepadkeynavigation.cpp \
+    streaming/video/overlaymanager.cpp
 
 HEADERS += \
     utils.h \
@@ -147,7 +152,8 @@ HEADERS += \
     backend/autoupdatechecker.h \
     path.h \
     settings/mappingmanager.h \
-    gui/sdlgamepadkeynavigation.h
+    gui/sdlgamepadkeynavigation.h \
+    streaming/video/overlaymanager.h
 
 # Platform-specific renderers and decoders
 ffmpeg {

app/streaming/input.cpp

@@ -142,6 +142,19 @@ void SdlInputHandler::handleKeyEvent(SDL_KeyboardEvent* event)
                         "Detected full-screen toggle combo");
             Session::s_ActiveSession->toggleFullscreen();
 
+            // Force raise all keys just be safe across this full-screen/windowed
+            // transition just in case key events get lost.
+            raiseAllKeys();
+            return;
+        }
+        else if (event->keysym.sym == SDLK_s) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Detected stats toggle combo");
+
+            // Toggle the stats overlay
+            Session::get()->getOverlayManager().setOverlayState(OverlayManager::OverlayDebug,
+                                                                !Session::get()->getOverlayManager().isOverlayEnabled(OverlayManager::OverlayDebug));
+
             // Force raise all keys just be safe across this full-screen/windowed
             // transition just in case key events get lost.
             raiseAllKeys();

app/streaming/session.h

@@ -4,11 +4,11 @@
 
 #include <Limelight.h>
 #include <opus_multistream.h>
-#include "backend/computermanager.h"
 #include "settings/streamingpreferences.h"
 #include "input.h"
 #include "video/decoder.h"
 #include "audio/renderers/renderer.h"
+#include "video/overlaymanager.h"
 
 class Session : public QObject
 {
@@ -32,6 +32,16 @@ public:
     int getDecoderCapabilities(StreamingPreferences::VideoDecoderSelection vds,
                                int videoFormat, int width, int height, int frameRate);
 
+    static Session* get()
+    {
+        return s_ActiveSession;
+    }
+
+    OverlayManager& getOverlayManager()
+    {
+        return m_OverlayManager;
+    }
+
 signals:
     void stageStarting(QString stage);
 
@@ -134,6 +144,8 @@ private:
     OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig;
     int m_AudioSampleCount;
 
+    OverlayManager m_OverlayManager;
+
     static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks;
     static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks;
     static Session* s_ActiveSession;

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -2,6 +2,7 @@
 #include "dxva2.h"
 #include "../ffmpeg.h"
 #include <streaming/streamutils.h>
+#include <streaming/session.h>
 
 #include <SDL_syswm.h>
 
@@ -25,7 +26,8 @@ DXVA2Renderer::DXVA2Renderer() :
     m_RenderTarget(nullptr),
     m_ProcService(nullptr),
     m_Processor(nullptr),
-    m_FrameIndex(0)
+    m_FrameIndex(0),
+    m_OverlayFont(nullptr)
 {
     RtlZeroMemory(m_DecSurfaces, sizeof(m_DecSurfaces));
     RtlZeroMemory(&m_DXVAContext, sizeof(m_DXVAContext));
@@ -44,6 +46,7 @@ DXVA2Renderer::~DXVA2Renderer()
     SAFE_COM_RELEASE(m_RenderTarget);
     SAFE_COM_RELEASE(m_ProcService);
     SAFE_COM_RELEASE(m_Processor);
+    SAFE_COM_RELEASE(m_OverlayFont);
 
     for (int i = 0; i < ARRAYSIZE(m_DecSurfaces); i++) {
         SAFE_COM_RELEASE(m_DecSurfaces[i]);
@@ -318,6 +321,32 @@ bool DXVA2Renderer::initializeRenderer()
     return true;
 }
 
+bool DXVA2Renderer::initializeOverlay()
+{
+    HRESULT hr;
+
+    hr = D3DXCreateFontA(m_Device,
+                         20,
+                         0,
+                         FW_HEAVY,
+                         1,
+                         false,
+                         ANSI_CHARSET,
+                         OUT_DEFAULT_PRECIS,
+                         DEFAULT_QUALITY,
+                         DEFAULT_PITCH | FF_DONTCARE,
+                         "",
+                         &m_OverlayFont);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "D3DXCreateFontA() failed: %x",
+                     hr);
+        return false;
+    }
+
+    return true;
+}
+
 bool DXVA2Renderer::isDecoderBlacklisted()
 {
     IDirect3D9* d3d9;
@@ -622,6 +651,9 @@ bool DXVA2Renderer::initialize(SDL_Window* window, int videoFormat, int width, i
         return false;
     }
 
+    // It's okay if this fails
+    initializeOverlay();
+
     // For some reason, using Direct3D9Ex breaks this with multi-monitor setups.
     // When focus is lost, the window is minimized then immediately restored without
     // input focus. This glitches out the renderer and a bunch of other stuff.
@@ -785,7 +817,27 @@ void DXVA2Renderer::renderFrameAtVsync(AVFrame *frame)
 
     bltParams.Alpha = DXVA2_Fixed32OpaqueAlpha();
 
-    m_Device->Clear(0, nullptr, D3DCLEAR_TARGET, D3DCOLOR_ARGB(255, 0, 0, 0), 0.0f, 0);
+    hr = m_Device->Clear(0, nullptr, D3DCLEAR_TARGET, D3DCOLOR_ARGB(255, 0, 0, 0), 0.0f, 0);
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Clear() failed: %x",
+                     hr);
+        SDL_Event event;
+        event.type = SDL_RENDER_TARGETS_RESET;
+        SDL_PushEvent(&event);
+        return;
+    }
+
+    hr = m_Device->BeginScene();
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "BeginScene() failed: %x",
+                     hr);
+        SDL_Event event;
+        event.type = SDL_RENDER_TARGETS_RESET;
+        SDL_PushEvent(&event);
+        return;
+    }
 
     hr = m_Processor->VideoProcessBlt(m_RenderTarget, &bltParams, &sample, 1, nullptr);
     if (FAILED(hr)) {
@@ -798,6 +850,28 @@ void DXVA2Renderer::renderFrameAtVsync(AVFrame *frame)
         return;
     }
 
+    if (m_OverlayFont != nullptr) {
+        if (Session::get()->getOverlayManager().isOverlayEnabled(OverlayManager::OverlayDebug)) {
+            m_OverlayFont->DrawTextA(nullptr,
+                                     Session::get()->getOverlayManager().getOverlayText(OverlayManager::OverlayDebug),
+                                     -1,
+                                     &sample.DstRect,
+                                     DT_LEFT | DT_NOCLIP,
+                                     D3DCOLOR_ARGB(255, 255, 255, 255));
+        }
+    }
+
+    hr = m_Device->EndScene();
+    if (FAILED(hr)) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "EndScene() failed: %x",
+                     hr);
+        SDL_Event event;
+        event.type = SDL_RENDER_TARGETS_RESET;
+        SDL_PushEvent(&event);
+        return;
+    }
+
     hr = m_Device->PresentEx(nullptr, nullptr, nullptr, nullptr, 0);
     if (FAILED(hr)) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,

app/streaming/video/ffmpeg-renderers/dxva2.h

@@ -4,6 +4,7 @@
 #include "pacer/pacer.h"
 
 #include <d3d9.h>
+#include <d3dx9.h>
 #include <dxva2api.h>
 
 extern "C" {
@@ -31,6 +32,7 @@ private:
     bool initializeDecoder();
     bool initializeRenderer();
     bool initializeDevice(SDL_Window* window, bool enableVsync);
+    bool initializeOverlay();
     bool isDecoderBlacklisted();
 
     static
@@ -67,4 +69,5 @@ private:
     DXVA2_ValueRange m_SaturationRange;
     DXVA2_VideoDesc m_Desc;
     REFERENCE_TIME m_FrameIndex;
+    LPD3DXFONT m_OverlayFont;
 };

app/streaming/video/ffmpeg.cpp

@@ -1,6 +1,8 @@
 #include <Limelight.h>
 #include "ffmpeg.h"
 #include "streaming/streamutils.h"
+#include "streaming/session.h"
+
 #include <h264_stream.h>
 
 #ifdef Q_OS_WIN32
@@ -276,54 +278,51 @@ void FFmpegVideoDecoder::addVideoStats(VIDEO_STATS& src, VIDEO_STATS& dst)
     dst.renderedFps = (float)dst.renderedFrames / ((float)(now - dst.measurementStartTimestamp) / 1000);
 }
 
+void FFmpegVideoDecoder::stringifyVideoStats(VIDEO_STATS& stats, char* output)
+{
+    int offset = 0;
+
+    // Start with an empty string
+    output[offset] = 0;
+
+    if (stats.receivedFps > 0) {
+        offset += sprintf(&output[offset],
+                          "Incoming frame rate from host PC: %.2f FPS\n"
+                          "Decoding frame rate: %.2f FPS\n"
+                          "Rendering frame rate: %.2f FPS\n",
+                          stats.receivedFps,
+                          stats.decodedFps,
+                          stats.renderedFps);
+    }
+
+    if (stats.renderedFrames != 0) {
+        offset += sprintf(&output[offset],
+                          "Average reassembly time: %.2f ms\n"
+                          "Average decode time: %.2f ms\n"
+                          "Average frame pacing delay: %.2f ms\n"
+                          "Average render time: %.2f ms\n"
+                          "Frames dropped by your network connection: %.2f%%\n"
+                          "Frames dropped by frame pacing: %.2f%%",
+                          (float)stats.totalReassemblyTime / stats.decodedFrames,
+                          (float)stats.totalDecodeTime / stats.decodedFrames,
+                          (float)stats.totalPacerTime / stats.renderedFrames,
+                          (float)stats.totalRenderTime / stats.renderedFrames,
+                          (float)stats.networkDroppedFrames / stats.decodedFrames,
+                          (float)stats.pacerDroppedFrames / stats.decodedFrames);
+    }
+}
+
 void FFmpegVideoDecoder::logVideoStats(VIDEO_STATS& stats, const char* title)
 {
     if (stats.renderedFps > 0 || stats.renderedFrames != 0) {
+        char videoStatsStr[512];
+        stringifyVideoStats(stats, videoStatsStr);
+
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                     "%s", title);
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "----------------------------------------------------------");
+                    "----------------------------------------------------------\n%s",
-    }
+                    videoStatsStr);
-
-    if (stats.renderedFps > 0) {
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Incoming frame rate from network: %.2f FPS",
-                    stats.receivedFps);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Decoding frame rate: %.2f FPS",
-                    stats.decodedFps);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Rendering frame rate: %.2f FPS",
-                    stats.renderedFps);
-    }
-    if (stats.renderedFrames != 0) {
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Total frames received: %u",
-                    stats.receivedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Total frames decoded: %u",
-                    stats.decodedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Total frames rendered: %u",
-                    stats.renderedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Average reassembly time: %.2f ms",
-                    (float)stats.totalReassemblyTime / stats.decodedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Average decode time: %.2f ms",
-                    (float)stats.totalDecodeTime / stats.decodedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Average frame pacing delay: %.2f ms",
-                    (float)stats.totalPacerTime / stats.renderedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Average render time: %.2f ms",
-                    (float)stats.totalRenderTime / stats.renderedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Frames lost during network transmission: %.2f%%",
-                    (float)stats.networkDroppedFrames / stats.decodedFrames);
-        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
-                    "Frames dropped by frame pacing: %.2f%%",
-                    (float)stats.pacerDroppedFrames / stats.decodedFrames);
     }
 }
 
@@ -492,16 +491,26 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
     PLENTRY entry = du->bufferList;
     int err;
 
-    // Flip stats windows roughly every second
+    if (!m_LastFrameNumber) {
-    if (m_ActiveWndVideoStats.receivedFrames == m_StreamFps) {
+        m_ActiveWndVideoStats.measurementStartTimestamp = SDL_GetTicks();
-#if 0
+        m_LastFrameNumber = du->frameNumber;
-        VIDEO_STATS lastTwoWndStats = {};
+    }
-        addVideoStats(m_LastWndVideoStats, lastTwoWndStats);
+    else {
-        addVideoStats(m_ActiveWndVideoStats, lastTwoWndStats);
+        // Any frame number greater than m_LastFrameNumber + 1 represents a dropped frame
+        m_ActiveWndVideoStats.networkDroppedFrames += du->frameNumber - (m_LastFrameNumber + 1);
+        m_LastFrameNumber = du->frameNumber;
+    }
 
-        // Print stats from the last 2 windows
+    // Flip stats windows roughly every second
-        logVideoStats(lastTwoWndStats, "Periodic video stats");
+    if (SDL_TICKS_PASSED(SDL_GetTicks(), m_ActiveWndVideoStats.measurementStartTimestamp + 1000)) {
-#endif
+        // Update overlay stats if it's enabled
+        if (Session::get()->getOverlayManager().isOverlayEnabled(OverlayManager::OverlayDebug)) {
+            VIDEO_STATS lastTwoWndStats = {};
+            addVideoStats(m_LastWndVideoStats, lastTwoWndStats);
+            addVideoStats(m_ActiveWndVideoStats, lastTwoWndStats);
+
+            stringifyVideoStats(lastTwoWndStats, Session::get()->getOverlayManager().getOverlayText(OverlayManager::OverlayDebug));
+        }
 
         // Accumulate these values into the global stats
         addVideoStats(m_ActiveWndVideoStats, m_GlobalVideoStats);
@@ -514,16 +523,6 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
 
     m_ActiveWndVideoStats.receivedFrames++;
 
-    if (!m_LastFrameNumber) {
-        m_ActiveWndVideoStats.measurementStartTimestamp = SDL_GetTicks();
-        m_LastFrameNumber = du->frameNumber;
-    }
-    else {
-        // Any frame number greater than m_LastFrameNumber + 1 represents a dropped frame
-        m_ActiveWndVideoStats.networkDroppedFrames += du->frameNumber - (m_LastFrameNumber + 1);
-        m_LastFrameNumber = du->frameNumber;
-    }
-
     int requiredBufferSize = du->fullLength;
     if (du->frameType == FRAME_TYPE_IDR) {
         // Add some extra space in case we need to do an SPS fixup

app/streaming/video/ffmpeg.h

@@ -33,6 +33,8 @@ private:
                                 int videoFormat, int width, int height,
                                 int maxFps, bool enableFramePacing, bool testOnly);
 
+    void stringifyVideoStats(VIDEO_STATS& stats, char* output);
+
     void logVideoStats(VIDEO_STATS& stats, const char* title);
 
     void addVideoStats(VIDEO_STATS& src, VIDEO_STATS& dst);
@@ -58,6 +60,7 @@ private:
     VIDEO_STATS m_ActiveWndVideoStats;
     VIDEO_STATS m_LastWndVideoStats;
     VIDEO_STATS m_GlobalVideoStats;
+
     int m_LastFrameNumber;
     int m_StreamFps;
     bool m_NeedsSpsFixup;

app/streaming/video/overlaymanager.cpp

@@ -0,0 +1,25 @@
+#include "overlaymanager.h"
+
+OverlayManager::OverlayManager()
+{
+    memset(m_Overlays, 0, sizeof(m_Overlays));
+}
+
+bool OverlayManager::isOverlayEnabled(OverlayManager::OverlayType type)
+{
+    return m_Overlays[type].enabled;
+}
+
+char* OverlayManager::getOverlayText(OverlayType type)
+{
+    return m_Overlays[type].text;
+}
+
+void OverlayManager::setOverlayState(OverlayManager::OverlayType type, bool enabled)
+{
+    m_Overlays[type].enabled = enabled;
+    if (!enabled) {
+        // Set the text to empty string on disable
+        m_Overlays[type].text[0] = 0;
+    }
+}

app/streaming/video/overlaymanager.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <QString>
+
+class OverlayManager
+{
+public:
+    enum OverlayType {
+        OverlayDebug,
+        OverlayMinorNotification,
+        OverlayMajorNotification,
+        OverlayMax
+    };
+
+    OverlayManager();
+
+    bool isOverlayEnabled(OverlayType type);
+    char* getOverlayText(OverlayType type);
+    void setOverlayState(OverlayType type, bool enabled);
+
+    struct {
+        bool enabled;
+        int updateSeq;
+        char text[512];
+    } m_Overlays[OverlayMax];
+};

scripts/generate-installers.bat

@@ -115,6 +115,10 @@ echo Copying AntiHooking.dll
 copy %BUILD_FOLDER%\AntiHooking\%BUILD_CONFIG%\AntiHooking.dll %DEPLOY_FOLDER%
 if !ERRORLEVEL! NEQ 0 goto Error
 
+echo Copying d3dx9_43.dll from DirectX SDK
+expand "%DXSDK_DIR%\Redist\Jun2010_d3dx9_43_%ARCH%.cab" -F:d3dx9_43.dll %DEPLOY_FOLDER%
+if !ERRORLEVEL! NEQ 0 goto Error
+
 echo Copying GC mapping list
 copy %SOURCE_ROOT%\app\SDL_GameControllerDB\gamecontrollerdb.txt %DEPLOY_FOLDER%
 if !ERRORLEVEL! NEQ 0 goto Error