Simplify codec selection and prioritization logic

app/streaming/session.h

@@ -11,6 +11,69 @@
 #include "audio/renderers/renderer.h"
 #include "video/overlaymanager.h"
 
+class SupportedVideoFormatList : public QList<int>
+{
+public:
+    operator int() const
+    {
+        int value = 0;
+
+        for (const int & v : *this) {
+            value |= v;
+        }
+
+        return value;
+    }
+
+    void
+    removeByMask(int mask)
+    {
+        int i = 0;
+        while (i < this->length()) {
+            if (this->value(i) & mask) {
+                this->removeAt(i);
+            }
+            else {
+                i++;
+            }
+        }
+    }
+
+    int maskByServerCodecModes(int serverCodecModes)
+    {
+        int val = *this;
+
+        // Make sure nobody forgets to update this for new SCM values
+        SDL_assert((serverCodecModes & ~(SCM_MASK_H264 | SCM_MASK_HEVC | SCM_MASK_AV1)) == 0);
+
+        // H.264 SCM masks
+        if (!(serverCodecModes & SCM_H264)) {
+            val &= ~VIDEO_FORMAT_H264;
+        }
+        SDL_assert((serverCodecModes & SCM_MASK_H264 & ~SCM_H264) == 0);
+
+        // HEVC SCM masks
+        if (!(serverCodecModes & SCM_HEVC)) {
+            val &= ~VIDEO_FORMAT_H265;
+        }
+        if (!(serverCodecModes & SCM_HEVC_MAIN10)) {
+            val &= ~VIDEO_FORMAT_H265_MAIN10;
+        }
+        SDL_assert((serverCodecModes & SCM_MASK_HEVC & ~(SCM_HEVC | SCM_HEVC_MAIN10)) == 0);
+
+        // AV1 SCM masks
+        if (!(serverCodecModes & SCM_AV1_MAIN8)) {
+            val &= ~VIDEO_FORMAT_AV1_MAIN8;
+        }
+        if (!(serverCodecModes & SCM_AV1_MAIN10)) {
+            val &= ~VIDEO_FORMAT_AV1_MAIN10;
+        }
+        SDL_assert((serverCodecModes & SCM_MASK_AV1 & ~(SCM_AV1_MAIN8 | SCM_AV1_MAIN10)) == 0);
+
+        return val;
+    }
+};
+
 class Session : public QObject
 {
     Q_OBJECT
@@ -158,6 +221,7 @@ private:
 
     StreamingPreferences* m_Preferences;
     bool m_IsFullScreen;
+    SupportedVideoFormatList m_SupportedVideoFormats; // Sorted in order of descending priority
     STREAM_CONFIGURATION m_StreamConfig;
     DECODER_RENDERER_CALLBACKS m_VideoCallbacks;
     AUDIO_RENDERER_CALLBACKS m_AudioCallbacks;