Switch to 32-bit floating point audio

Excluding Steam Link due to CPU and API limitations

app/streaming/audio/audio.cpp

@@ -205,29 +205,41 @@ void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength)
     }
 
     if (s_ActiveSession->m_AudioRenderer != nullptr) {
-        int desiredSize = sizeof(short) * s_ActiveSession->m_ActiveAudioConfig.samplesPerFrame * s_ActiveSession->m_ActiveAudioConfig.channelCount;
+        int sampleSize = s_ActiveSession->m_AudioRenderer->getAudioBufferSampleSize();
-        void* buffer = s_ActiveSession->m_AudioRenderer->getAudioBuffer(&desiredSize);
+        int frameSize = sampleSize * s_ActiveSession->m_ActiveAudioConfig.channelCount;
+        int desiredBufferSize = frameSize * s_ActiveSession->m_ActiveAudioConfig.samplesPerFrame;
+        void* buffer = s_ActiveSession->m_AudioRenderer->getAudioBuffer(&desiredBufferSize);
         if (buffer == nullptr) {
             return;
         }
 
-        samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder,
+        if (s_ActiveSession->m_AudioRenderer->getAudioBufferFormat() == IAudioRenderer::AudioFormat::Float32NE) {
-                                                 (unsigned char*)sampleData,
+            samplesDecoded = opus_multistream_decode_float(s_ActiveSession->m_OpusDecoder,
-                                                 sampleLength,
+                                                           (unsigned char*)sampleData,
-                                                 (short*)buffer,
+                                                           sampleLength,
-                                                 desiredSize / sizeof(short) / s_ActiveSession->m_ActiveAudioConfig.channelCount,
+                                                           (float*)buffer,
-                                                 0);
+                                                           desiredBufferSize / frameSize,
+                                                           0);
+        }
+        else {
+            samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder,
+                                                     (unsigned char*)sampleData,
+                                                     sampleLength,
+                                                     (short*)buffer,
+                                                     desiredBufferSize / frameSize,
+                                                     0);
+        }
 
         // Update desiredSize with the number of bytes actually populated by the decoding operation
         if (samplesDecoded > 0) {
-            SDL_assert(desiredSize >= (int)(sizeof(short) * samplesDecoded * s_ActiveSession->m_ActiveAudioConfig.channelCount));
+            SDL_assert(desiredBufferSize >= frameSize * samplesDecoded);
-            desiredSize = sizeof(short) * samplesDecoded * s_ActiveSession->m_ActiveAudioConfig.channelCount;
+            desiredBufferSize = frameSize * samplesDecoded;
         }
         else {
-            desiredSize = 0;
+            desiredBufferSize = 0;
         }
 
-        if (!s_ActiveSession->m_AudioRenderer->submitAudio(desiredSize)) {
+        if (!s_ActiveSession->m_AudioRenderer->submitAudio(desiredBufferSize)) {
             SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                         "Reinitializing audio renderer after failure");
 

app/streaming/audio/renderers/renderer.h

@@ -25,4 +25,19 @@ public:
         // 4 - Surround Left
         // 5 - Surround Right
     }
+
+    enum class AudioFormat {
+        Sint16NE,  // 16-bit signed integer (native endian)
+        Float32NE, // 32-bit floating point (native endian)
+    };
+    virtual AudioFormat getAudioBufferFormat() = 0;
+
+    int getAudioBufferSampleSize() {
+        switch (getAudioBufferFormat()) {
+        case IAudioRenderer::AudioFormat::Sint16NE:
+            return sizeof(short);
+        case IAudioRenderer::AudioFormat::Float32NE:
+            return sizeof(float);
+        }
+    }
 };

app/streaming/audio/renderers/sdl.h

@@ -18,6 +18,8 @@ public:
 
     virtual int getCapabilities();
 
+    virtual AudioFormat getAudioBufferFormat();
+
 private:
     SDL_AudioDeviceID m_AudioDevice;
     void* m_AudioBuffer;

app/streaming/audio/renderers/sdlaud.cpp

@@ -23,7 +23,7 @@ bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION*
 
     SDL_zero(want);
     want.freq = opusConfig->sampleRate;
-    want.format = AUDIO_S16;
+    want.format = AUDIO_F32SYS;
     want.channels = opusConfig->channelCount;
 
     // On PulseAudio systems, setting a value too small can cause underruns for other
@@ -40,7 +40,9 @@ bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION*
     want.samples = SDL_max(480, opusConfig->samplesPerFrame);
 #endif
 
-    m_FrameSize = opusConfig->samplesPerFrame * sizeof(short) * opusConfig->channelCount;
+    m_FrameSize = opusConfig->samplesPerFrame *
+                  opusConfig->channelCount *
+                  getAudioBufferSampleSize();
 
     m_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0);
     if (m_AudioDevice == 0) {
@@ -60,7 +62,7 @@ bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION*
     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                 "Desired audio buffer: %u samples (%u bytes)",
                 want.samples,
-                want.samples * (Uint32)sizeof(short) * want.channels);
+                want.samples * want.channels * getAudioBufferSampleSize());
 
     SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                 "Obtained audio buffer: %u samples (%u bytes)",
@@ -143,3 +145,8 @@ int SdlAudioRenderer::getCapabilities()
     // Direct submit can't be used because we use LiGetPendingAudioDuration()
     return CAPABILITY_SUPPORTS_ARBITRARY_AUDIO_DURATION;
 }
+
+IAudioRenderer::AudioFormat SdlAudioRenderer::getAudioBufferFormat()
+{
+    return AudioFormat::Float32NE;
+}

app/streaming/audio/renderers/slaud.cpp

@@ -23,7 +23,9 @@ bool SLAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* o
     // it's hard to avoid since we get crushed by CPU limitations.
     m_MaxQueuedAudioMs = 40 * opusConfig->channelCount / 2;
 
-    m_AudioBufferSize = opusConfig->samplesPerFrame * sizeof(short) * opusConfig->channelCount;
+    m_AudioBufferSize = opusConfig->samplesPerFrame *
+                        opusConfig->channelCount *
+                        getAudioBufferSampleSize();
     m_AudioStream = SLAudio_CreateStream(m_AudioContext,
                                          opusConfig->sampleRate,
                                          opusConfig->channelCount,
@@ -117,6 +119,11 @@ int SLAudioRenderer::getCapabilities()
     return CAPABILITY_SLOW_OPUS_DECODER | CAPABILITY_SUPPORTS_ARBITRARY_AUDIO_DURATION;
 }
 
+IAudioRenderer::AudioFormat SLAudioRenderer::getAudioBufferFormat()
+{
+    return AudioFormat::Sint16NE;
+}
+
 void SLAudioRenderer::slLogCallback(void*, ESLAudioLog logLevel, const char *message)
 {
     SDL_LogPriority priority;

app/streaming/audio/renderers/slaud.h

@@ -18,6 +18,8 @@ public:
 
     virtual int getCapabilities();
 
+    virtual AudioFormat getAudioBufferFormat();
+
     virtual void remapChannels(POPUS_MULTISTREAM_CONFIGURATION opusConfig);
 
 private:

app/streaming/audio/renderers/soundioaudiorenderer.cpp

@@ -164,7 +164,7 @@ bool SoundIoAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATI
 
     m_AudioPacketDuration = (opusConfig->samplesPerFrame / (opusConfig->sampleRate / 1000)) / 1000.0;
 
-    m_OutputStream->format = SoundIoFormatS16NE;
+    m_OutputStream->format = SoundIoFormatFloat32NE;
     m_OutputStream->sample_rate = opusConfig->sampleRate;
     m_OutputStream->software_latency = m_AudioPacketDuration;
     m_OutputStream->name = "Moonlight";
@@ -323,6 +323,11 @@ int SoundIoAudioRenderer::getCapabilities()
     return CAPABILITY_DIRECT_SUBMIT /* | CAPABILITY_SUPPORTS_ARBITRARY_AUDIO_DURATION */;
 }
 
+IAudioRenderer::AudioFormat SoundIoAudioRenderer::getAudioBufferFormat()
+{
+    return AudioFormat::Float32NE;
+}
+
 void SoundIoAudioRenderer::sioErrorCallback(SoundIoOutStream* stream, int err)
 {
     auto me = reinterpret_cast<SoundIoAudioRenderer*>(stream->userdata);

app/streaming/audio/renderers/soundioaudiorenderer.h

@@ -19,6 +19,8 @@ public:
 
     virtual int getCapabilities();
 
+    virtual AudioFormat getAudioBufferFormat();
+
 private:
     int scoreChannelLayout(const struct SoundIoChannelLayout* layout, const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);