WIP: D3D11VA support

Overlays work, but drawing the actual video is unimplemented

app/app.pro

@@ -108,7 +108,7 @@ unix:!macx {
     }
 }
 win32 {
-    LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lopus
+    LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lopus -ldxgi -ld3d11
     CONFIG += ffmpeg
 }
 win32:!winrt {
@@ -300,14 +300,16 @@ win32 {
     HEADERS += streaming/video/ffmpeg-renderers/dxutil.h
 }
 win32:!winrt {
-    message(DXVA2 renderer selected)
+    message(DXVA2 and D3D11VA renderers selected)
 
     SOURCES += \
         streaming/video/ffmpeg-renderers/dxva2.cpp \
+        streaming/video/ffmpeg-renderers/d3d11va.cpp \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.cpp
 
     HEADERS += \
         streaming/video/ffmpeg-renderers/dxva2.h \
+        streaming/video/ffmpeg-renderers/d3d11va.h \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.h
 }
 macx {

app/resources.qrc

@@ -59,5 +59,7 @@
         <file alias="egl_opaque.vert">shaders/egl_opaque.vert</file>
         <file alias="egl_overlay.frag">shaders/egl_overlay.frag</file>
         <file alias="egl_overlay.vert">shaders/egl_overlay.vert</file>
+        <file alias="d3d11_vertex.fxc">shaders/d3d11_vertex.fxc</file>
+        <file alias="d3d11_overlay_pixel.fxc">shaders/d3d11_overlay_pixel.fxc</file>
     </qresource>
 </RCC>

app/shaders/build_hlsl.bat

@@ -0,0 +1,3 @@
+fxc /T vs_4_0_level_9_3 /Fo d3d11_vertex.fxc d3d11_vertex.hlsl
+
+fxc /T ps_4_0_level_9_3 /Fo d3d11_overlay_pixel.fxc d3d11_overlay_pixel.hlsl

app/shaders/d3d11_overlay_pixel.hlsl

@@ -0,0 +1,13 @@
+Texture2D theTexture : register(t0);
+SamplerState theSampler : register(s0);
+
+struct ShaderInput
+{
+    float4 pos : SV_POSITION;
+    float2 tex : TEXCOORD0;
+};
+
+float4 main(ShaderInput input) : SV_TARGET
+{
+    return theTexture.Sample(theSampler, input.tex);
+}

app/shaders/d3d11_vertex.hlsl

@@ -0,0 +1,19 @@
+struct ShaderInput
+{
+    float2 pos : POSITION;
+    float2 tex : TEXCOORD0;
+};
+
+struct ShaderOutput
+{
+    float4 pos : SV_POSITION;
+    float2 tex : TEXCOORD0;
+};
+
+ShaderOutput main(ShaderInput input) 
+{
+    ShaderOutput output;
+    output.pos = float4(input.pos, 0.0f, 1.0f);
+    output.tex = input.tex;
+    return output;
+}

app/streaming/video/ffmpeg-renderers/d3d11va.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "renderer.h"
+#include "pacer/pacer.h"
+
+#include <d3d11_1.h>
+#include <dxgi1_5.h>
+
+extern "C" {
+#include <libavutil/hwcontext_d3d11va.h>
+}
+
+class D3D11VARenderer : public IFFmpegRenderer
+{
+public:
+    D3D11VARenderer();
+    virtual ~D3D11VARenderer() override;
+    virtual bool initialize(PDECODER_PARAMETERS params) override;
+    virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary**) override;
+    virtual void renderFrame(AVFrame* frame) override;
+    virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
+    virtual void setHdrMode(bool enabled) override;
+    virtual int getRendererAttributes() override;
+
+private:
+    static void lockContext(void* lock_ctx);
+    static void unlockContext(void* lock_ctx);
+
+    bool setupRenderingResources();
+    void renderOverlay(Overlay::OverlayType type);
+    bool checkDecoderSupport(IDXGIAdapter* adapter);
+
+    IDXGIFactory5* m_Factory;
+    ID3D11Device* m_Device;
+    IDXGISwapChain4* m_SwapChain;
+    ID3D11DeviceContext* m_DeviceContext;
+    ID3D11RenderTargetView* m_RenderTargetView;
+    SDL_mutex* m_ContextLock;
+
+    DECODER_PARAMETERS m_DecoderParams;
+    int m_DisplayWidth;
+    int m_DisplayHeight;
+    bool m_Windowed;
+
+    bool m_AllowTearing;
+    HANDLE m_FrameWaitableObject;
+    ID3D11PixelShader* m_VideoPixelShader;
+
+    SDL_SpinLock m_OverlayLock;
+    ID3D11Buffer* m_OverlayVertexBuffers[Overlay::OverlayMax];
+    ID3D11Texture2D* m_OverlayTextures[Overlay::OverlayMax];
+    ID3D11ShaderResourceView* m_OverlayTextureResourceViews[Overlay::OverlayMax];
+    ID3D11PixelShader* m_OverlayPixelShader;
+
+    AVBufferRef* m_HwContext;
+};
+

app/streaming/video/ffmpeg-renderers/dxva2.cpp

@@ -669,6 +669,12 @@ bool DXVA2Renderer::initializeDevice(SDL_Window* window, bool enableVsync)
 
 bool DXVA2Renderer::initialize(PDECODER_PARAMETERS params)
 {
+    // Don't use DXVA2 for HDR10. While it can render 10-bit color, it doesn't support
+    // the HDR colorspace and HDR display metadata required to enable HDR mode properly.
+    if (params->videoFormat == VIDEO_FORMAT_H265_MAIN10) {
+        return false;
+    }
+
     m_VideoFormat = params->videoFormat;
     m_VideoWidth = params->width;
     m_VideoHeight = params->height;

app/streaming/video/ffmpeg.cpp

@@ -9,6 +9,7 @@
 
 #ifdef Q_OS_WIN32
 #include "ffmpeg-renderers/dxva2.h"
+#include "ffmpeg-renderers/d3d11va.h"
 #endif
 
 #ifdef Q_OS_DARWIN
@@ -599,8 +600,13 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
     if (pass == 0) {
         switch (hwDecodeCfg->device_type) {
 #ifdef Q_OS_WIN32
+        // DXVA2 appears in the hwaccel list before D3D11VA, so we will implicitly
+        // prefer it. When we want to switch to D3D11VA by default, we'll need to
+        // move it into the second pass set below.
         case AV_HWDEVICE_TYPE_DXVA2:
             return new DXVA2Renderer();
+        case AV_HWDEVICE_TYPE_D3D11VA:
+            return new D3D11VARenderer();
 #endif
 #ifdef Q_OS_DARWIN
         case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: