Refactor audio rendering into a separate class

app/app.pro

@@ -87,7 +87,8 @@ SOURCES += \
     settings/streamingpreferences.cpp \
     streaming/input.cpp \
     streaming/session.cpp \
-    streaming/audio.cpp \
+    streaming/audio/audio.cpp \
+    streaming/audio/renderers/sdlaud.cpp \
     gui/computermodel.cpp \
     gui/appmodel.cpp \
     streaming/streamutils.cpp \
@@ -104,6 +105,8 @@ HEADERS += \
     settings/streamingpreferences.h \
     streaming/input.hpp \
     streaming/session.hpp \
+    streaming/audio/renderers/renderer.h \
+    streaming/audio/renderers/sdl.h \
     gui/computermodel.h \
     gui/appmodel.h \
     streaming/video/decoder.h \
@@ -118,7 +121,7 @@ ffmpeg {
     DEFINES += HAVE_FFMPEG
     SOURCES += \
         streaming/video/ffmpeg.cpp \
-        streaming/video/ffmpeg-renderers/sdl.cpp \
+        streaming/video/ffmpeg-renderers/sdlvid.cpp \
         streaming/video/ffmpeg-renderers/pacer/pacer.cpp \
         streaming/video/ffmpeg-renderers/pacer/nullthreadedvsyncsource.cpp
 

app/streaming/audio/audio.cpp

@@ -0,0 +1,111 @@
+#include "../session.hpp"
+#include "renderers/renderer.h"
+#include "renderers/sdl.h"
+
+#include <Limelight.h>
+
+IAudioRenderer* Session::createAudioRenderer()
+{
+    return new SdlAudioRenderer();
+}
+
+bool Session::testAudio(int audioConfiguration)
+{
+    IAudioRenderer* audioRenderer;
+
+    audioRenderer = createAudioRenderer();
+    if (audioRenderer == nullptr) {
+        return false;
+    }
+
+    bool ret = audioRenderer->testAudio(audioConfiguration);
+
+    delete audioRenderer;
+
+    return ret;
+}
+
+int Session::detectAudioConfiguration()
+{
+    IAudioRenderer* audioRenderer;
+
+    audioRenderer = createAudioRenderer();
+    if (audioRenderer == nullptr) {
+        // Hope for the best
+        return AUDIO_CONFIGURATION_STEREO;
+    }
+
+    int audioConfig = audioRenderer->detectAudioConfiguration();
+
+    delete audioRenderer;
+
+    return audioConfig;
+}
+
+int Session::arInit(int /* audioConfiguration */,
+                    const POPUS_MULTISTREAM_CONFIGURATION opusConfig,
+                    void* /* arContext */, int /* arFlags */)
+{
+    int error;
+
+    SDL_memcpy(&s_ActiveSession->m_AudioConfig, opusConfig, sizeof(*opusConfig));
+
+    s_ActiveSession->m_OpusDecoder =
+            opus_multistream_decoder_create(opusConfig->sampleRate,
+                                            opusConfig->channelCount,
+                                            opusConfig->streams,
+                                            opusConfig->coupledStreams,
+                                            opusConfig->mapping,
+                                            &error);
+    if (s_ActiveSession->m_OpusDecoder == NULL) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to create decoder: %d",
+                     error);
+        return -1;
+    }
+
+    s_ActiveSession->m_AudioRenderer = s_ActiveSession->createAudioRenderer();
+    if (s_ActiveSession->m_AudioRenderer == nullptr) {
+        opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
+        return -2;
+    }
+
+    s_ActiveSession->m_AudioRenderer->prepareForPlayback(opusConfig);
+    if (s_ActiveSession->m_AudioRenderer == nullptr) {
+        delete s_ActiveSession->m_AudioRenderer;
+        opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
+        return -3;
+    }
+
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Audio stream has %d channels",
+                opusConfig->channelCount);
+
+    return 0;
+}
+
+void Session::arCleanup()
+{
+    delete s_ActiveSession->m_AudioRenderer;
+
+    opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
+    s_ActiveSession->m_OpusDecoder = nullptr;
+}
+
+void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength)
+{
+    int samplesDecoded;
+
+    samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder,
+                                             (unsigned char*)sampleData,
+                                             sampleLength,
+                                             s_ActiveSession->m_OpusDecodeBuffer,
+                                             SAMPLES_PER_FRAME,
+                                             0);
+    if (samplesDecoded > 0) {
+        s_ActiveSession->m_AudioRenderer->submitAudio(s_ActiveSession->m_OpusDecodeBuffer,
+                                                      static_cast<int>(sizeof(short) *
+                                                      samplesDecoded *
+                                                      s_ActiveSession->m_AudioConfig.channelCount));
+    }
+}

app/streaming/audio/renderers/renderer.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <Limelight.h>
+
+#define MAX_CHANNELS 6
+#define SAMPLES_PER_FRAME 240
+
+class IAudioRenderer
+{
+public:
+    virtual ~IAudioRenderer() {}
+
+    virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) = 0;
+
+    virtual void submitAudio(short* audioBuffer, int audioSize) = 0;
+
+    virtual bool testAudio(int audioConfiguration) = 0;
+
+    virtual int detectAudioConfiguration() = 0;
+};

app/streaming/audio/renderers/sdl.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "renderer.h"
+#include <SDL.h>
+
+class SdlAudioRenderer : public IAudioRenderer
+{
+public:
+    SdlAudioRenderer();
+
+    virtual ~SdlAudioRenderer();
+
+    virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);
+
+    virtual void submitAudio(short* audioBuffer, int audioSize);
+
+    virtual bool testAudio(int audioConfiguration);
+
+    virtual int detectAudioConfiguration();
+
+private:
+    SDL_AudioDeviceID m_AudioDevice;
+    int m_ChannelCount;
+    int m_PendingDrops;
+    int m_PendingHardDrops;
+    unsigned int m_SampleIndex;
+    Uint32 m_BaselinePendingData;
+};

app/streaming/audio/renderers/sdlaud.cpp

@@ -1,25 +1,16 @@
-#include "session.hpp"
+#include "sdl.h"
 
 #include <Limelight.h>
 #include <SDL.h>
 
-#define MAX_CHANNELS 6
+#include <QtGlobal>
-#define SAMPLES_PER_FRAME 240
+
 #define MIN_QUEUED_FRAMES 2
 #define MAX_QUEUED_FRAMES 4
 #define STOP_THE_WORLD_LIMIT 20
 #define DROP_RATIO_DENOM 32
 
-SDL_AudioDeviceID Session::s_AudioDevice;
+int SdlAudioRenderer::detectAudioConfiguration()
-OpusMSDecoder* Session::s_OpusDecoder;
-short Session::s_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME];
-int Session::s_ChannelCount;
-int Session::s_PendingDrops;
-int Session::s_PendingHardDrops;
-unsigned int Session::s_SampleIndex;
-Uint32 Session::s_BaselinePendingData;
-
-int Session::sdlDetermineAudioConfiguration()
 {
     SDL_AudioSpec want, have;
     SDL_AudioDeviceID dev;
@@ -70,7 +61,7 @@ Exit:
     return ret;
 }
 
-bool Session::testAudio(int audioConfiguration)
+bool SdlAudioRenderer::testAudio(int audioConfiguration)
 {
     SDL_AudioSpec want, have;
     SDL_AudioDeviceID dev;
@@ -124,12 +115,20 @@ Exit:
     return ret;
 }
 
-int Session::sdlAudioInit(int /* audioConfiguration */,
+SdlAudioRenderer::SdlAudioRenderer()
-                          POPUS_MULTISTREAM_CONFIGURATION opusConfig,
+    : m_AudioDevice(0),
-                          void* /* arContext */, int /* arFlags */)
+      m_ChannelCount(0),
+      m_PendingDrops(0),
+      m_PendingHardDrops(0),
+      m_SampleIndex(0),
+      m_BaselinePendingData(0)
+{
+
+}
+
+bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig)
 {
     SDL_AudioSpec want, have;
-    int error;
 
     SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
     if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) {
@@ -150,8 +149,8 @@ int Session::sdlAudioInit(int /* audioConfiguration */,
     // Specifying non-Po2 seems to work for our supported platforms.
     want.samples = SAMPLES_PER_FRAME;
 
-    s_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0);
+    m_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0);
-    if (s_AudioDevice == 0) {
+    if (m_AudioDevice == 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to open audio device: %s",
                      SDL_GetError());
@@ -159,126 +158,83 @@ int Session::sdlAudioInit(int /* audioConfiguration */,
         return -1;
     }
 
-    s_OpusDecoder = opus_multistream_decoder_create(opusConfig->sampleRate,
-                                                    opusConfig->channelCount,
-                                                    opusConfig->streams,
-                                                    opusConfig->coupledStreams,
-                                                    opusConfig->mapping,
-                                                    &error);
-    if (s_OpusDecoder == NULL) {
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                     "Failed to create decoder: %d",
-                     error);
-        SDL_CloseAudioDevice(s_AudioDevice);
-        s_AudioDevice = 0;
-        SDL_QuitSubSystem(SDL_INIT_AUDIO);
-        return -2;
-    }
-
     // SDL counts pending samples in the queued
     // audio size using the WASAPI backend. This
     // includes silence, which can throw off our
     // pending data count. Get a baseline so we
     // can exclude that data.
-    s_BaselinePendingData = 0;
+    m_BaselinePendingData = 0;
 #ifdef Q_OS_WIN32
     for (int i = 0; i < 100; i++) {
-        s_BaselinePendingData = qMax(s_BaselinePendingData, SDL_GetQueuedAudioSize(s_AudioDevice));
+        m_BaselinePendingData = qMax(m_BaselinePendingData, SDL_GetQueuedAudioSize(m_AudioDevice));
         SDL_Delay(10);
     }
 #endif
-    s_BaselinePendingData *= 2;
+    m_BaselinePendingData *= 2;
     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                 "Baseline pending audio data: %d bytes",
-                s_BaselinePendingData);
+                m_BaselinePendingData);
 
-    s_ChannelCount = opusConfig->channelCount;
+    m_ChannelCount = opusConfig->channelCount;
-    s_SampleIndex = 0;
+    m_SampleIndex = 0;
-    s_PendingDrops = s_PendingHardDrops = 0;
+    m_PendingDrops = m_PendingHardDrops = 0;
 
-    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+    // Start playback
-                "Audio stream has %d channels",
+    SDL_PauseAudioDevice(m_AudioDevice, 0);
-                opusConfig->channelCount);
 
     return 0;
 }
 
-void Session::sdlAudioStart()
+SdlAudioRenderer::~SdlAudioRenderer()
 {
-    // Unpause the audio device
+    // Stop playback
-    SDL_PauseAudioDevice(s_AudioDevice, 0);
+    SDL_PauseAudioDevice(m_AudioDevice, 1);
-}
+    SDL_CloseAudioDevice(m_AudioDevice);
-
-void Session::sdlAudioStop()
-{
-    // Pause the audio device
-    SDL_PauseAudioDevice(s_AudioDevice, 1);
-}
-
-void Session::sdlAudioCleanup()
-{
-    SDL_CloseAudioDevice(s_AudioDevice);
-    s_AudioDevice = 0;
-
-    opus_multistream_decoder_destroy(s_OpusDecoder);
-    s_OpusDecoder = nullptr;
 
     SDL_QuitSubSystem(SDL_INIT_AUDIO);
     SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
 }
 
-void Session::sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength)
+void SdlAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
 {
-    int samplesDecoded;
+    m_SampleIndex++;
 
-    s_SampleIndex++;
+    Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(m_AudioDevice) - (int)m_BaselinePendingData, 0);
-
+    Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * m_ChannelCount * sizeof(short));
-    Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(s_AudioDevice) - (int)s_BaselinePendingData, 0);
-    Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * s_ChannelCount * sizeof(short));
 
     // We must check this prior to the below checks to ensure we don't
-    // underflow if framesQueued - s_PendingHardDrops < 0.
+    // underflow if framesQueued - m_PendingHardDrops < 0.
     if (framesQueued <= MIN_QUEUED_FRAMES) {
-        s_PendingDrops = s_PendingHardDrops = 0;
+        m_PendingDrops = m_PendingHardDrops = 0;
     }
     // Pend enough drops to get us back to MIN_QUEUED_FRAMES
-    else if (framesQueued - s_PendingHardDrops > STOP_THE_WORLD_LIMIT) {
+    else if (framesQueued - m_PendingHardDrops > STOP_THE_WORLD_LIMIT) {
-        s_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES;
+        m_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES;
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                     "Pending hard drop of %u audio frames",
-                    s_PendingHardDrops);
+                    m_PendingHardDrops);
     }
-    else if (framesQueued - s_PendingHardDrops - s_PendingDrops > MAX_QUEUED_FRAMES) {
+    else if (framesQueued - m_PendingHardDrops - m_PendingDrops > MAX_QUEUED_FRAMES) {
-        s_PendingDrops = framesQueued - MIN_QUEUED_FRAMES;
+        m_PendingDrops = framesQueued - MIN_QUEUED_FRAMES;
     }
 
     // Determine if this frame should be dropped
-    if (s_PendingHardDrops != 0) {
+    if (m_PendingHardDrops != 0) {
         // Hard drops happen all at once to forcefully
         // resync with the source.
-        s_PendingHardDrops--;
+        m_PendingHardDrops--;
         return;
     }
-    else if (s_PendingDrops != 0 && s_SampleIndex % DROP_RATIO_DENOM == 0) {
+    else if (m_PendingDrops != 0 && m_SampleIndex % DROP_RATIO_DENOM == 0) {
         // Normal drops are interspersed with the audio data
         // to hide the glitches.
-        s_PendingDrops--;
+        m_PendingDrops--;
         return;
     }
 
-    samplesDecoded = opus_multistream_decode(s_OpusDecoder,
+    if (SDL_QueueAudio(m_AudioDevice, audioBuffer, audioSize) < 0) {
-                                             (unsigned char*)sampleData,
-                                             sampleLength,
-                                             s_OpusDecodeBuffer,
-                                             SAMPLES_PER_FRAME,
-                                             0);
-    if (samplesDecoded > 0) {
-        if (SDL_QueueAudio(s_AudioDevice,
-                           s_OpusDecodeBuffer,
-                           sizeof(short) * samplesDecoded * s_ChannelCount) < 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to queue audio sample: %s",
                      SDL_GetError());
     }
-    }
 }

app/streaming/session.cpp

@@ -48,11 +48,11 @@ CONNECTION_LISTENER_CALLBACKS Session::k_ConnCallbacks = {
 };
 
 AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = {
-    Session::sdlAudioInit,
+    Session::arInit,
-    Session::sdlAudioStart,
+    nullptr,
-    Session::sdlAudioStop,
+    nullptr,
-    Session::sdlAudioCleanup,
+    Session::arCleanup,
-    Session::sdlAudioDecodeAndPlaySample,
+    Session::arDecodeAndPlaySample,
     CAPABILITY_DIRECT_SUBMIT
 };
 
@@ -277,7 +277,9 @@ Session::Session(NvComputer* computer, NvApp& app)
       m_AudioDisabled(false),
       m_DisplayOriginX(0),
       m_DisplayOriginY(0),
-      m_PendingWindowedTransition(false)
+      m_PendingWindowedTransition(false),
+      m_OpusDecoder(nullptr),
+      m_AudioRenderer(nullptr)
 {
 
 }
@@ -321,7 +323,7 @@ void Session::initialize()
     {
     case StreamingPreferences::AC_AUTO:
         SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Autodetecting audio configuration");
-        m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration();
+        m_StreamConfig.audioConfiguration = detectAudioConfiguration();
         break;
     case StreamingPreferences::AC_FORCE_STEREO:
         m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_STEREO;

app/streaming/session.hpp

@@ -8,6 +8,7 @@
 #include "settings/streamingpreferences.h"
 #include "input.hpp"
 #include "video/decoder.h"
+#include "audio/renderers/renderer.h"
 
 class Session : public QObject
 {
@@ -49,7 +50,9 @@ private:
 
     int getDecoderCapabilities();
 
-    int sdlDetermineAudioConfiguration();
+    IAudioRenderer* createAudioRenderer();
+
+    int detectAudioConfiguration();
 
     bool testAudio(int audioConfiguration);
 
@@ -77,6 +80,17 @@ private:
     static
     void clLogMessage(const char* format, ...);
 
+    static
+    int arInit(int audioConfiguration,
+               const POPUS_MULTISTREAM_CONFIGURATION opusConfig,
+               void* arContext, int arFlags);
+
+    static
+    void arCleanup();
+
+    static
+    void arDecodeAndPlaySample(char* sampleData, int sampleLength);
+
     static
     int drSetup(int videoFormat, int width, int height, int frameRate, void*, int);
 
@@ -86,23 +100,6 @@ private:
     static
     int drSubmitDecodeUnit(PDECODE_UNIT du);
 
-    static
-    int sdlAudioInit(int audioConfiguration,
-                     POPUS_MULTISTREAM_CONFIGURATION opusConfig,
-                     void* arContext, int arFlags);
-
-    static
-    void sdlAudioStart();
-
-    static
-    void sdlAudioStop();
-
-    static
-    void sdlAudioCleanup();
-
-    static
-    void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength);
-
     StreamingPreferences m_Preferences;
     STREAM_CONFIGURATION m_StreamConfig;
     DECODER_RENDERER_CALLBACKS m_VideoCallbacks;
@@ -123,14 +120,10 @@ private:
     int m_ActiveVideoHeight;
     int m_ActiveVideoFrameRate;
 
-    static SDL_AudioDeviceID s_AudioDevice;
+    OpusMSDecoder* m_OpusDecoder;
-    static OpusMSDecoder* s_OpusDecoder;
+    short m_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME];
-    static short s_OpusDecodeBuffer[];
+    IAudioRenderer* m_AudioRenderer;
-    static int s_ChannelCount;
+    OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig;
-    static int s_PendingDrops;
-    static int s_PendingHardDrops;
-    static unsigned int s_SampleIndex;
-    static Uint32 s_BaselinePendingData;
 
     static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks;
     static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks;

moonlight-common-c/moonlight-common-c

@@ -1 +1 @@
-Subproject commit 8743ce5d083407c33c8b3c23529eaa9630361e7c
+Subproject commit 718d6a4b28a209902848c8678207ab6a48474a63