Decode directly into the audio renderer's buffer to avoid a copy

app/streaming/audio/audio.cpp

@@ -143,19 +143,22 @@ void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength)
     s_ActiveSession->m_AudioSampleCount++;
 
     if (s_ActiveSession->m_AudioRenderer != nullptr) {
+        int desiredSize = sizeof(short) * SAMPLES_PER_FRAME * s_ActiveSession->m_AudioConfig.channelCount;
+        void* buffer = s_ActiveSession->m_AudioRenderer->getAudioBuffer(&desiredSize);
+        if (buffer == nullptr) {
+            return;
+        }
+
         samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder,
                                                  (unsigned char*)sampleData,
                                                  sampleLength,
-                                                 s_ActiveSession->m_OpusDecodeBuffer,
+                                                 (short*)buffer,
-                                                 SAMPLES_PER_FRAME,
+                                                 desiredSize / sizeof(short) / s_ActiveSession->m_AudioConfig.channelCount,
                                                  0);
         if (samplesDecoded > 0) {
-            if (!s_ActiveSession->m_AudioRenderer->submitAudio(s_ActiveSession->m_OpusDecodeBuffer,
+            SDL_assert(desiredSize >= sizeof(short) * samplesDecoded * s_ActiveSession->m_AudioConfig.channelCount);
-                                                               static_cast<int>(
+            desiredSize = sizeof(short) * samplesDecoded * s_ActiveSession->m_AudioConfig.channelCount;
-                                                                   sizeof(short) *
+            if (!s_ActiveSession->m_AudioRenderer->submitAudio(desiredSize)) {
-                                                                   samplesDecoded *
-                                                                   s_ActiveSession->m_AudioConfig.channelCount))) {
-
                 SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                             "Reinitializing audio renderer after failure");
 

app/streaming/audio/renderers/renderer.h

@@ -12,6 +12,8 @@ public:
 
     virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) = 0;
 
+    virtual void* getAudioBuffer(int* size) = 0;
+
     // Return false if an unrecoverable error has occurred and the renderer must be reinitialized
-    virtual bool submitAudio(short* audioBuffer, int audioSize) = 0;
+    virtual bool submitAudio(int bytesWritten) = 0;
 };

app/streaming/audio/renderers/sdl.h

@@ -12,8 +12,11 @@ public:
 
     virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);
 
-    virtual bool submitAudio(short* audioBuffer, int audioSize);
+    virtual void* getAudioBuffer(int* size);
+
+    virtual bool submitAudio(int bytesWritten);
 
 private:
     SDL_AudioDeviceID m_AudioDevice;
+    void* m_AudioBuffer;
 };

app/streaming/audio/renderers/sdlaud.cpp

@@ -6,7 +6,8 @@
 #include <QtGlobal>
 
 SdlAudioRenderer::SdlAudioRenderer()
-    : m_AudioDevice(0)
+    : m_AudioDevice(0),
+      m_AudioBuffer(nullptr)
 {
     SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
     if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) {
@@ -37,7 +38,13 @@ bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION*
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to open audio device: %s",
                      SDL_GetError());
-        SDL_QuitSubSystem(SDL_INIT_AUDIO);
+        return false;
+    }
+
+    m_AudioBuffer = malloc(SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount);
+    if (m_AudioBuffer == nullptr) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to allocate audio buffer");
         return false;
     }
 
@@ -65,13 +72,22 @@ SdlAudioRenderer::~SdlAudioRenderer()
         SDL_CloseAudioDevice(m_AudioDevice);
     }
 
+    if (m_AudioBuffer != nullptr) {
+        free(m_AudioBuffer);
+    }
+
     SDL_QuitSubSystem(SDL_INIT_AUDIO);
     SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
 }
 
-bool SdlAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
+void* SdlAudioRenderer::getAudioBuffer(int*)
 {
-    if (SDL_QueueAudio(m_AudioDevice, audioBuffer, audioSize) < 0) {
+    return m_AudioBuffer;
+}
+
+bool SdlAudioRenderer::submitAudio(int bytesWritten)
+{
+    if (SDL_QueueAudio(m_AudioDevice, m_AudioBuffer, bytesWritten) < 0) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Failed to queue audio sample: %s",
                      SDL_GetError());

app/streaming/audio/renderers/slaud.cpp

@@ -9,9 +9,7 @@
 
 SLAudioRenderer::SLAudioRenderer()
     : m_AudioContext(nullptr),
-      m_AudioStream(nullptr),
+      m_AudioStream(nullptr)
-      m_AudioBuffer(nullptr),
-      m_AudioBufferBytesFilled(0)
 {
     SLAudio_SetLogFunction(SLAudioRenderer::slLogCallback, nullptr);
 }
@@ -25,7 +23,7 @@ bool SLAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* o
         return false;
     }
 
-    m_AudioBufferSize = SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount * FRAMES_PER_SUBMISSION;
+    m_AudioBufferSize = SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount;
     m_AudioStream = SLAudio_CreateStream(m_AudioContext,
                                          opusConfig->sampleRate,
                                          opusConfig->channelCount,
@@ -43,16 +41,14 @@ bool SLAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* o
     return true;
 }
 
+void* SLAudioRenderer::getAudioBuffer(int* size)
+{
+    SDL_assert(*size == m_AudioBufferSize);
+    return SLAudio_BeginFrame(m_AudioStream);
+}
+
 SLAudioRenderer::~SLAudioRenderer()
 {
-    if (m_AudioBufferBytesFilled != 0) {
-        // We had a buffer in flight when we quit. Just in case
-        // SLAudio doesn't handle this properly, we'll zero and submit
-        // it just to be safe.
-        memset(m_AudioBuffer, 0, m_AudioBufferSize);
-        SLAudio_SubmitFrame(m_AudioStream);
-    }
-
     if (m_AudioStream != nullptr) {
         SLAudio_FreeStream(m_AudioStream);
     }
@@ -62,28 +58,9 @@ SLAudioRenderer::~SLAudioRenderer()
     }
 }
 
-bool SLAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
+bool SLAudioRenderer::submitAudio(int)
 {
-    if (m_AudioBufferBytesFilled == 0) {
+    SLAudio_SubmitFrame(m_AudioStream);
-        // Get a new audio buffer from SLAudio
-        m_AudioBuffer = (char*)SLAudio_BeginFrame(m_AudioStream);
-        if (m_AudioBuffer == nullptr) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "SLAudio_BeginFrame() failed");
-            return true;
-        }
-    }
-
-    // Accumulate several frames of audio before submitting to reduce CPU load
-    SDL_assert(audioSize <= m_AudioBufferSize - m_AudioBufferBytesFilled);
-    memcpy(&m_AudioBuffer[m_AudioBufferBytesFilled], audioBuffer, audioSize);
-    m_AudioBufferBytesFilled += audioSize;
-
-    // Submit the buffer when it's full
-    if (m_AudioBufferBytesFilled == m_AudioBufferSize) {
-        SLAudio_SubmitFrame(m_AudioStream);
-        m_AudioBufferBytesFilled = 0;
-    }
-
     return true;
 }
 

app/streaming/audio/renderers/slaud.h

@@ -12,14 +12,14 @@ public:
 
     virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);
 
-    virtual bool submitAudio(short* audioBuffer, int audioSize);
+    virtual void* getAudioBuffer(int* size);
+
+    virtual bool submitAudio(int bytesWritten);
 
 private:
     static void slLogCallback(void* context, ESLAudioLog logLevel, const char* message);
 
     CSLAudioContext* m_AudioContext;
     CSLAudioStream* m_AudioStream;
-    char* m_AudioBuffer;
     int m_AudioBufferSize;
-    int m_AudioBufferBytesFilled;
 };

app/streaming/audio/renderers/soundioaudiorenderer.cpp

@@ -284,7 +284,20 @@ bool SoundIoAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATI
     return true;
 }
 
-bool SoundIoAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
+void* SoundIoAudioRenderer::getAudioBuffer(int* size)
+{
+    // We must always write a full frame of audio. If we don't,
+    // the reader will get out of sync with the writer and our
+    // channels will get all mixed up. To ensure this is always
+    // the case, round our bytes free down to the next multiple
+    // of our frame size.
+    int bytesFree = soundio_ring_buffer_free_count(m_RingBuffer);
+    int bytesPerFrame = m_OpusChannelCount * m_OutputStream->bytes_per_sample;
+    *size = qMin(*size, (bytesFree / bytesPerFrame) * bytesPerFrame);
+    return soundio_ring_buffer_write_ptr(m_RingBuffer);
+}
+
+bool SoundIoAudioRenderer::submitAudio(int bytesWritten)
 {
     if (m_Errored) {
         return false;
@@ -293,16 +306,8 @@ bool SoundIoAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
     // Flush events to update with new device arrivals
     soundio_flush_events(m_SoundIo);
 
-    // We must always write a full frame of audio. If we don't,
+    // Advance the write pointer
-    // the reader will get out of sync with the writer and our
+    soundio_ring_buffer_advance_write_ptr(m_RingBuffer, bytesWritten);
-    // channels will get all mixed up. To ensure this is always
-    // the case, round our bytes free down to the next multiple
-    // of our frame size.
-    int bytesFree = soundio_ring_buffer_free_count(m_RingBuffer);
-    int bytesPerFrame = m_OpusChannelCount * m_OutputStream->bytes_per_sample;
-    int bytesToWrite = qMin(audioSize, (bytesFree / bytesPerFrame) * bytesPerFrame);
-    memcpy(soundio_ring_buffer_write_ptr(m_RingBuffer), audioBuffer, bytesToWrite);
-    soundio_ring_buffer_advance_write_ptr(m_RingBuffer, bytesToWrite);
 
     return true;
 }

app/streaming/audio/renderers/soundioaudiorenderer.h

@@ -13,7 +13,9 @@ public:
 
     virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);
 
-    virtual bool submitAudio(short* audioBuffer, int audioSize);
+    virtual void* getAudioBuffer(int* size);
+
+    virtual bool submitAudio(int bytesWritten);
 
 private:
     int scoreChannelLayout(const struct SoundIoChannelLayout* layout, const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);

app/streaming/session.h

@@ -148,7 +148,6 @@ private:
     int m_ActiveVideoFrameRate;
 
     OpusMSDecoder* m_OpusDecoder;
-    short m_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME];
     IAudioRenderer* m_AudioRenderer;
     OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig;
     int m_AudioSampleCount;