WIP: D3D11VA support

Overlays work, but drawing the actual video is unimplemented

app/streaming/video/ffmpeg-renderers/d3d11va.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "renderer.h"
+#include "pacer/pacer.h"
+
+#include <d3d11_1.h>
+#include <dxgi1_5.h>
+
+extern "C" {
+#include <libavutil/hwcontext_d3d11va.h>
+}
+
+class D3D11VARenderer : public IFFmpegRenderer
+{
+public:
+    D3D11VARenderer();
+    virtual ~D3D11VARenderer() override;
+    virtual bool initialize(PDECODER_PARAMETERS params) override;
+    virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary**) override;
+    virtual void renderFrame(AVFrame* frame) override;
+    virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
+    virtual void setHdrMode(bool enabled) override;
+    virtual int getRendererAttributes() override;
+
+private:
+    static void lockContext(void* lock_ctx);
+    static void unlockContext(void* lock_ctx);
+
+    bool setupRenderingResources();
+    void renderOverlay(Overlay::OverlayType type);
+    bool checkDecoderSupport(IDXGIAdapter* adapter);
+
+    IDXGIFactory5* m_Factory;
+    ID3D11Device* m_Device;
+    IDXGISwapChain4* m_SwapChain;
+    ID3D11DeviceContext* m_DeviceContext;
+    ID3D11RenderTargetView* m_RenderTargetView;
+    SDL_mutex* m_ContextLock;
+
+    DECODER_PARAMETERS m_DecoderParams;
+    int m_DisplayWidth;
+    int m_DisplayHeight;
+    bool m_Windowed;
+
+    bool m_AllowTearing;
+    HANDLE m_FrameWaitableObject;
+    ID3D11PixelShader* m_VideoPixelShader;
+
+    SDL_SpinLock m_OverlayLock;
+    ID3D11Buffer* m_OverlayVertexBuffers[Overlay::OverlayMax];
+    ID3D11Texture2D* m_OverlayTextures[Overlay::OverlayMax];
+    ID3D11ShaderResourceView* m_OverlayTextureResourceViews[Overlay::OverlayMax];
+    ID3D11PixelShader* m_OverlayPixelShader;
+
+    AVBufferRef* m_HwContext;
+};
+

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -669,6 +669,12 @@ bool DXVA2Renderer::initializeDevice(SDL_Window* window, bool enableVsync)
 
 bool DXVA2Renderer::initialize(PDECODER_PARAMETERS params)
 {
+    // Don't use DXVA2 for HDR10. While it can render 10-bit color, it doesn't support
+    // the HDR colorspace and HDR display metadata required to enable HDR mode properly.
+    if (params->videoFormat == VIDEO_FORMAT_H265_MAIN10) {
+        return false;
+    }
+
     m_VideoFormat = params->videoFormat;
     m_VideoWidth = params->width;
     m_VideoHeight = params->height;

app/streaming/video/ffmpeg.cpp

@@ -9,6 +9,7 @@
 
 #ifdef Q_OS_WIN32
 #include "ffmpeg-renderers/dxva2.h"
+#include "ffmpeg-renderers/d3d11va.h"
 #endif
 
 #ifdef Q_OS_DARWIN
@@ -599,8 +600,13 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
     if (pass == 0) {
         switch (hwDecodeCfg->device_type) {
 #ifdef Q_OS_WIN32
+        // DXVA2 appears in the hwaccel list before D3D11VA, so we will implicitly
+        // prefer it. When we want to switch to D3D11VA by default, we'll need to
+        // move it into the second pass set below.
         case AV_HWDEVICE_TYPE_DXVA2:
             return new DXVA2Renderer();
+        case AV_HWDEVICE_TYPE_D3D11VA:
+            return new D3D11VARenderer();
 #endif
 #ifdef Q_OS_DARWIN
         case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: