From 83747e501da1f88cbec7fd6238bcb7014448c291 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 20 Dec 2013 15:20:48 -0500 Subject: [PATCH 1/5] Move audio buffering responsibility onto us and simply fill whatever the runtime gives us --- .../binding/audio/JavaxAudioRenderer.java | 33 +++++++----- .../limelight/binding/audio/SoundBuffer.java | 53 +++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 src/com/limelight/binding/audio/SoundBuffer.java diff --git a/src/com/limelight/binding/audio/JavaxAudioRenderer.java b/src/com/limelight/binding/audio/JavaxAudioRenderer.java index e6f94f2..134b390 100644 --- a/src/com/limelight/binding/audio/JavaxAudioRenderer.java +++ b/src/com/limelight/binding/audio/JavaxAudioRenderer.java @@ -1,30 +1,36 @@ package com.limelight.binding.audio; -import java.nio.ByteBuffer; - import javax.sound.sampled.AudioFormat; import javax.sound.sampled.AudioSystem; import javax.sound.sampled.DataLine; import javax.sound.sampled.LineUnavailableException; import javax.sound.sampled.SourceDataLine; +import com.limelight.nvstream.av.ShortBufferDescriptor; import com.limelight.nvstream.av.audio.AudioRenderer; -import com.limelight.nvstream.av.audio.OpusDecoder; public class JavaxAudioRenderer implements AudioRenderer { private SourceDataLine soundLine; + private SoundBuffer soundBuffer; + private byte[] lineBuffer; + + public static final int STAGING_BUFFERS = 3; // 3 complete frames of audio @Override public void playDecodedAudio(short[] pcmData, int offset, int length) { if (soundLine != null) { - byte[] pcmDataBytes = new byte[length * 2]; - ByteBuffer.wrap(pcmDataBytes).asShortBuffer().put(pcmData, offset, length); - if (soundLine.available() < length) { - soundLine.write(pcmDataBytes, 0, soundLine.available()); - } - else { - soundLine.write(pcmDataBytes, 0, pcmDataBytes.length); + // Queue the decoded samples into the staging sound buffer + soundBuffer.queue(new ShortBufferDescriptor(pcmData, offset, length)); + + // If there's space available in the sound line, pull some data out + // of the staging buffer and write it to the sound line + int available = soundLine.available(); + if (available > 0) { + int written = soundBuffer.fill(lineBuffer, 0, available); + if (written > 0) { + soundLine.write(lineBuffer, 0, written); + } } } } @@ -39,11 +45,14 @@ public class JavaxAudioRenderer implements AudioRenderer { @Override public void streamInitialized(int channelCount, int sampleRate) { AudioFormat audioFormat = new AudioFormat(sampleRate, 16, channelCount, true, true); - DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat, OpusDecoder.getMaxOutputShorts()); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); try { soundLine = (SourceDataLine) AudioSystem.getLine(info); - soundLine.open(audioFormat, OpusDecoder.getMaxOutputShorts()*4*2); + soundLine.open(audioFormat); soundLine.start(); + + lineBuffer = new byte[soundLine.getBufferSize()]; + soundBuffer = new SoundBuffer(STAGING_BUFFERS); } catch (LineUnavailableException e) { soundLine = null; } diff --git a/src/com/limelight/binding/audio/SoundBuffer.java b/src/com/limelight/binding/audio/SoundBuffer.java new file mode 100644 index 0000000..9197ebb --- /dev/null +++ b/src/com/limelight/binding/audio/SoundBuffer.java @@ -0,0 +1,53 @@ +package com.limelight.binding.audio; + +import java.nio.ByteBuffer; +import java.nio.ShortBuffer; +import java.util.LinkedList; + +import com.limelight.nvstream.av.ShortBufferDescriptor; + +public class SoundBuffer { + + private LinkedList bufferList; + private int maxBuffers; + + public SoundBuffer(int maxBuffers) { + this.bufferList = new LinkedList(); + this.maxBuffers = maxBuffers; + } + + public void queue(ShortBufferDescriptor buff) { + if (bufferList.size() > maxBuffers) { + bufferList.removeFirst(); + } + + bufferList.addLast(buff); + } + + public int fill(byte[] data, int offset, int length) { + int filled = 0; + + // Convert offset and length to be relative to shorts + offset /= 2; + length /= 2; + + ShortBuffer sb = ByteBuffer.wrap(data).asShortBuffer(); + sb.position(offset); + while (length > 0 && !bufferList.isEmpty()) { + ShortBufferDescriptor buff = bufferList.getFirst(); + + if (buff.length > length) { + break; + } + + sb.put(buff.data, buff.offset, buff.length); + length -= buff.length; + filled += buff.length; + + bufferList.removeFirst(); + } + + // Return bytes instead of shorts + return filled * 2; + } +} From 8390d0bbd7f94b0cefb9ffa0efef15c176a401ca Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 20 Dec 2013 19:41:52 -0500 Subject: [PATCH 2/5] Try to workaround Java's poor OS X mixer --- .../limelight/binding/audio/JavaxAudioRenderer.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/com/limelight/binding/audio/JavaxAudioRenderer.java b/src/com/limelight/binding/audio/JavaxAudioRenderer.java index 134b390..09620d4 100644 --- a/src/com/limelight/binding/audio/JavaxAudioRenderer.java +++ b/src/com/limelight/binding/audio/JavaxAudioRenderer.java @@ -48,7 +48,15 @@ public class JavaxAudioRenderer implements AudioRenderer { DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); try { soundLine = (SourceDataLine) AudioSystem.getLine(info); - soundLine.open(audioFormat); + + // Java's OS X mixer performs very badly with the default buffer size + if (System.getProperty("os.name").contains("Mac OS X")) { + soundLine.open(audioFormat, 16384); + } + else { + soundLine.open(audioFormat); + } + soundLine.start(); lineBuffer = new byte[soundLine.getBufferSize()]; From 958e9a3c72b225ee6c144be0ba5b40c3f660041d Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 20 Dec 2013 19:44:35 -0500 Subject: [PATCH 3/5] Revert "Try to workaround Java's poor OS X mixer" This reverts commit 8390d0bbd7f94b0cefb9ffa0efef15c176a401ca. --- .../limelight/binding/audio/JavaxAudioRenderer.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/com/limelight/binding/audio/JavaxAudioRenderer.java b/src/com/limelight/binding/audio/JavaxAudioRenderer.java index 09620d4..134b390 100644 --- a/src/com/limelight/binding/audio/JavaxAudioRenderer.java +++ b/src/com/limelight/binding/audio/JavaxAudioRenderer.java @@ -48,15 +48,7 @@ public class JavaxAudioRenderer implements AudioRenderer { DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); try { soundLine = (SourceDataLine) AudioSystem.getLine(info); - - // Java's OS X mixer performs very badly with the default buffer size - if (System.getProperty("os.name").contains("Mac OS X")) { - soundLine.open(audioFormat, 16384); - } - else { - soundLine.open(audioFormat); - } - + soundLine.open(audioFormat); soundLine.start(); lineBuffer = new byte[soundLine.getBufferSize()]; From caa329790678ea0c1c99f78eeec7a837b5197eb8 Mon Sep 17 00:00:00 2001 From: Diego Waxemberg Date: Fri, 20 Dec 2013 19:51:58 -0500 Subject: [PATCH 4/5] dynamically increase the buffer size if we build up to large of a queue --- .../binding/audio/JavaxAudioRenderer.java | 30 +++++++++++++++---- .../limelight/binding/audio/SoundBuffer.java | 8 +++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/com/limelight/binding/audio/JavaxAudioRenderer.java b/src/com/limelight/binding/audio/JavaxAudioRenderer.java index 134b390..7e0ca5e 100644 --- a/src/com/limelight/binding/audio/JavaxAudioRenderer.java +++ b/src/com/limelight/binding/audio/JavaxAudioRenderer.java @@ -14,7 +14,10 @@ public class JavaxAudioRenderer implements AudioRenderer { private SourceDataLine soundLine; private SoundBuffer soundBuffer; private byte[] lineBuffer; + private int channelCount; + private int sampleRate; + public static final int DEFAULT_BUFFER_SIZE = 4096; public static final int STAGING_BUFFERS = 3; // 3 complete frames of audio @Override @@ -26,6 +29,18 @@ public class JavaxAudioRenderer implements AudioRenderer { // If there's space available in the sound line, pull some data out // of the staging buffer and write it to the sound line int available = soundLine.available(); + + // Kinda jank. If the queued is larger than available, we are going to have a delay + // so we increase the buffer size + if (available < soundBuffer.size()) { + System.out.println("buffer too full, buffer size: " + soundLine.getBufferSize()); + int currentBuffer = soundLine.getBufferSize(); + soundLine.close(); + createSoundLine(currentBuffer*2); + available = soundLine.available(); + System.out.println("creating new line with buffer size: " + soundLine.getBufferSize()); + } + if (available > 0) { int written = soundBuffer.fill(lineBuffer, 0, available); if (written > 0) { @@ -41,16 +56,14 @@ public class JavaxAudioRenderer implements AudioRenderer { soundLine.close(); } } - - @Override - public void streamInitialized(int channelCount, int sampleRate) { + + private void createSoundLine(int bufferSize) { AudioFormat audioFormat = new AudioFormat(sampleRate, 16, channelCount, true, true); DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); try { soundLine = (SourceDataLine) AudioSystem.getLine(info); - soundLine.open(audioFormat); + soundLine.open(audioFormat, bufferSize); soundLine.start(); - lineBuffer = new byte[soundLine.getBufferSize()]; soundBuffer = new SoundBuffer(STAGING_BUFFERS); } catch (LineUnavailableException e) { @@ -58,4 +71,11 @@ public class JavaxAudioRenderer implements AudioRenderer { } } + @Override + public void streamInitialized(int channelCount, int sampleRate) { + this.channelCount = channelCount; + this.sampleRate = sampleRate; + createSoundLine(DEFAULT_BUFFER_SIZE); + } + } diff --git a/src/com/limelight/binding/audio/SoundBuffer.java b/src/com/limelight/binding/audio/SoundBuffer.java index 9197ebb..8859bad 100644 --- a/src/com/limelight/binding/audio/SoundBuffer.java +++ b/src/com/limelight/binding/audio/SoundBuffer.java @@ -24,6 +24,14 @@ public class SoundBuffer { bufferList.addLast(buff); } + public int size() { + int size = 0; + for (ShortBufferDescriptor desc : bufferList) { + size += desc.length; + } + return size; + } + public int fill(byte[] data, int offset, int length) { int filled = 0; From 78f48b26b0e647e9937630130791d39f8c2e0ea8 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sat, 21 Dec 2013 16:21:52 -0500 Subject: [PATCH 5/5] Only use the audio buffer hack on OS X --- .../binding/audio/JavaxAudioRenderer.java | 66 ++++++++++++++----- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/src/com/limelight/binding/audio/JavaxAudioRenderer.java b/src/com/limelight/binding/audio/JavaxAudioRenderer.java index 7e0ca5e..dfd84ce 100644 --- a/src/com/limelight/binding/audio/JavaxAudioRenderer.java +++ b/src/com/limelight/binding/audio/JavaxAudioRenderer.java @@ -16,8 +16,10 @@ public class JavaxAudioRenderer implements AudioRenderer { private byte[] lineBuffer; private int channelCount; private int sampleRate; + private boolean reallocateLines; - public static final int DEFAULT_BUFFER_SIZE = 4096; + public static final int DEFAULT_BUFFER_SIZE = 0; + public static final int STARING_BUFFER_SIZE = 4096; public static final int STAGING_BUFFERS = 3; // 3 complete frames of audio @Override @@ -26,21 +28,28 @@ public class JavaxAudioRenderer implements AudioRenderer { // Queue the decoded samples into the staging sound buffer soundBuffer.queue(new ShortBufferDescriptor(pcmData, offset, length)); - // If there's space available in the sound line, pull some data out - // of the staging buffer and write it to the sound line int available = soundLine.available(); - - // Kinda jank. If the queued is larger than available, we are going to have a delay - // so we increase the buffer size - if (available < soundBuffer.size()) { - System.out.println("buffer too full, buffer size: " + soundLine.getBufferSize()); - int currentBuffer = soundLine.getBufferSize(); - soundLine.close(); - createSoundLine(currentBuffer*2); - available = soundLine.available(); - System.out.println("creating new line with buffer size: " + soundLine.getBufferSize()); + if (reallocateLines) { + // Kinda jank. If the queued is larger than available, we are going to have a delay + // so we increase the buffer size + if (available < soundBuffer.size()) { + System.out.println("buffer too full, buffer size: " + soundLine.getBufferSize()); + int currentBuffer = soundLine.getBufferSize(); + soundLine.close(); + createSoundLine(currentBuffer*2); + if (soundLine != null) { + available = soundLine.available(); + System.out.println("creating new line with buffer size: " + soundLine.getBufferSize()); + } + else { + available = 0; + System.out.println("failed to create sound line"); + } + } } + // If there's space available in the sound line, pull some data out + // of the staging buffer and write it to the sound line if (available > 0) { int written = soundBuffer.fill(lineBuffer, 0, available); if (written > 0) { @@ -59,10 +68,26 @@ public class JavaxAudioRenderer implements AudioRenderer { private void createSoundLine(int bufferSize) { AudioFormat audioFormat = new AudioFormat(sampleRate, 16, channelCount, true, true); - DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); + + DataLine.Info info; + + if (bufferSize == DEFAULT_BUFFER_SIZE) { + info = new DataLine.Info(SourceDataLine.class, audioFormat); + } + else { + info = new DataLine.Info(SourceDataLine.class, audioFormat, bufferSize); + } + try { soundLine = (SourceDataLine) AudioSystem.getLine(info); - soundLine.open(audioFormat, bufferSize); + + if (bufferSize == DEFAULT_BUFFER_SIZE) { + soundLine.open(audioFormat); + } + else { + soundLine.open(audioFormat, bufferSize); + } + soundLine.start(); lineBuffer = new byte[soundLine.getBufferSize()]; soundBuffer = new SoundBuffer(STAGING_BUFFERS); @@ -75,7 +100,16 @@ public class JavaxAudioRenderer implements AudioRenderer { public void streamInitialized(int channelCount, int sampleRate) { this.channelCount = channelCount; this.sampleRate = sampleRate; - createSoundLine(DEFAULT_BUFFER_SIZE); + + // Workaround OS X's bad Java mixer + if (System.getProperty("os.name").contains("Mac OS X")) { + createSoundLine(STARING_BUFFER_SIZE); + reallocateLines = true; + } + else { + createSoundLine(DEFAULT_BUFFER_SIZE); + reallocateLines = false; + } } }