diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java index bfff091f..f7dd583d 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java @@ -10,9 +10,16 @@ public class AudioDepacketizer { private static final int DU_LIMIT = 15; private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(DU_LIMIT); - + + private AudioRenderer directSubmitRenderer; + // Sequencing state private short lastSequenceNumber; + + public AudioDepacketizer(AudioRenderer directSubmitRenderer) + { + this.directSubmitRenderer = directSubmitRenderer; + } private void decodeData(byte[] data, int off, int len) { @@ -24,8 +31,10 @@ public class AudioDepacketizer { // Return value of decode is frames (shorts) decoded per channel decodeLen *= 2*OpusDecoder.getChannelCount(); - // Put it on the decoded queue - if (!decodedUnits.offer(new ByteBufferDescriptor(pcmData, 0, decodeLen))) { + if (directSubmitRenderer != null) { + directSubmitRenderer.playDecodedAudio(pcmData, 0, decodeLen); + } + else if (!decodedUnits.offer(new ByteBufferDescriptor(pcmData, 0, decodeLen))) { System.out.println("Audio player too slow! Forced to drop decoded samples"); // Clear out the queue decodedUnits.clear(); diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java index a7125998..48fb88af 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java @@ -1,6 +1,11 @@ package com.limelight.nvstream.av.audio; public interface AudioRenderer { + // playDecodedAudio() is lightweight, so don't use an extra thread for playback + public static final int CAPABILITY_DIRECT_SUBMIT = 0x1; + + public int getCapabilities(); + public void streamInitialized(int channelCount, int sampleRate); public void playDecodedAudio(byte[] audioData, int offset, int length); diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java index fb7604e1..3ae8c5e6 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java @@ -23,7 +23,7 @@ public class AudioStream { private DatagramSocket rtp; - private AudioDepacketizer depacketizer = new AudioDepacketizer(); + private AudioDepacketizer depacketizer; private LinkedList threads = new LinkedList(); @@ -79,7 +79,9 @@ public class AudioStream { startDepacketizerThread(); - startDecoderThread(); + if ((streamListener.getCapabilities() & AudioRenderer.CAPABILITY_DIRECT_SUBMIT) == 0) { + startDecoderThread(); + } startUdpPingThread(); } @@ -102,6 +104,13 @@ public class AudioStream { } streamListener.streamInitialized(OpusDecoder.getChannelCount(), OpusDecoder.getSampleRate()); + + if ((streamListener.getCapabilities() & AudioRenderer.CAPABILITY_DIRECT_SUBMIT) != 0) { + depacketizer = new AudioDepacketizer(streamListener); + } + else { + depacketizer = new AudioDepacketizer(null); + } } private void startDepacketizerThread() diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java index 7fe0be00..335c53d2 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java @@ -6,6 +6,11 @@ public interface VideoDecoderRenderer { public static final int FLAG_PREFER_QUALITY = 0x1; public static final int FLAG_FORCE_HARDWARE_DECODING = 0x2; public static final int FLAG_FORCE_SOFTWARE_DECODING = 0x4; + + // SubmitDecodeUnit() is lightweight, so don't use an extra thread for decoding + public static final int CAPABILITY_DIRECT_SUBMIT = 0x1; + + public int getCapabilities(); public void setup(int width, int height, int redrawRate, Object renderTarget, int drFlags); diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java index 60ea4852..4215ae6b 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java @@ -22,12 +22,14 @@ public class VideoDepacketizer { private ByteBufferDescriptor cachedDesc = new ByteBufferDescriptor(null, 0, 0); private ConnectionStatusListener controlListener; + private VideoDecoderRenderer directSubmitDr; - private static final int DU_LIMIT = 7; + private static final int DU_LIMIT = 15; private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(DU_LIMIT); - public VideoDepacketizer(ConnectionStatusListener controlListener) + public VideoDepacketizer(VideoDecoderRenderer directSubmitDr, ConnectionStatusListener controlListener) { + this.directSubmitDr = directSubmitDr; this.controlListener = controlListener; } @@ -43,7 +45,11 @@ public class VideoDepacketizer { if (avcNalDataChain != null && avcNalDataLength != 0) { // Construct the H264 decode unit DecodeUnit du = new DecodeUnit(DecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, 0); - if (!decodedUnits.offer(du)) { + if (directSubmitDr != null) { + // Submit directly to the decoder + directSubmitDr.submitDecodeUnit(du); + } + else if (!decodedUnits.offer(du)) { // We need a new IDR frame since we're discarding data now System.out.println("Video decoder is too slow! Forced to drop decode units"); decodedUnits.clear(); diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java index 7414e0ef..b197f64d 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java @@ -35,6 +35,7 @@ public class VideoStream { private LinkedList threads = new LinkedList(); private NvConnectionListener listener; + private ConnectionStatusListener avConnListener; private VideoDepacketizer depacketizer; private StreamConfiguration streamConfig; @@ -47,7 +48,7 @@ public class VideoStream { { this.host = host; this.listener = listener; - this.depacketizer = new VideoDepacketizer(avConnListener); + this.avConnListener = avConnListener; this.streamConfig = streamConfig; } @@ -134,6 +135,13 @@ public class VideoStream { if (decRend != null) { decRend.setup(streamConfig.getWidth(), streamConfig.getHeight(), 60, renderTarget, drFlags); + + if ((decRend.getCapabilities() & VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT) != 0) { + depacketizer = new VideoDepacketizer(decRend, avConnListener); + } + else { + depacketizer = new VideoDepacketizer(null, avConnListener); + } } } @@ -164,8 +172,10 @@ public class VideoStream { // Start the depacketizer thread to deal with the RTP data startDepacketizerThread(); - // Start decoding the data we're receiving - startDecoderThread(); + // Start a decode thread if we're not doing direct submit + if ((decRend.getCapabilities() & VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT) == 0) { + startDecoderThread(); + } // Start the renderer decRend.start();