Add a Opus decoder JNI library. Write an audio depacketizer. Audio works!

src/com/limelight/nvstream/NvAudioStream.java

@@ -13,8 +13,14 @@ import jlibrtp.RTPSession;
 
 import com.limelight.nvstream.av.AvBufferDescriptor;
 import com.limelight.nvstream.av.AvBufferPool;
+import com.limelight.nvstream.av.AvDecodeUnit;
 import com.limelight.nvstream.av.AvRtpPacket;
+import com.limelight.nvstream.av.audio.AvAudioDepacketizer;
+import com.limelight.nvstream.av.audio.OpusDecoder;
 
+import android.media.AudioFormat;
+import android.media.AudioManager;
+import android.media.AudioTrack;
 import android.media.MediaCodec;
 import android.media.MediaFormat;
 import android.net.rtp.AudioGroup;
@@ -27,20 +33,132 @@ public class NvAudioStream {
 	
 	private LinkedBlockingQueue<AvRtpPacket> packets = new LinkedBlockingQueue<AvRtpPacket>();
 	
+	private AudioTrack track;
+	
 	private RTPSession session;
 	private DatagramSocket rtp;
 	
+	private AvAudioDepacketizer depacketizer = new AvAudioDepacketizer();
+	
 	private AvBufferPool pool = new AvBufferPool(1500);
 	
-	public void setupRtpSession(String host) throws SocketException
-	{
-		DatagramSocket rtcp;
+	public void startAudioStream(final String host)
+	{		
+		new Thread(new Runnable() {
 
+			@Override
+			public void run() {
+				try {
+					setupRtpSession(host);
+				} catch (SocketException e) {
+					e.printStackTrace();
+					return;
+				}
+				
+				setupAudio();
+				
+				startReceiveThread();
+				
+				startDepacketizerThread();
+				
+				startUdpPingThread();
+				
+				startDecoderThread();
+			}
+			
+		}).start();
+	}
+	
+	private void setupRtpSession(String host) throws SocketException
+	{
 		rtp = new DatagramSocket(RTP_PORT);
-		rtcp = new DatagramSocket(RTCP_PORT);
 		
-		session = new RTPSession(rtp, rtcp);
-		session.addParticipant(new Participant(host, RTP_PORT, RTCP_PORT));
+		session = new RTPSession(rtp, null);
+		session.addParticipant(new Participant(host, RTP_PORT, 0));
+	}
+	
+	private void setupAudio()
+	{
+		int channelConfig;
+		int err;
+		
+		err = OpusDecoder.init();
+		if (err == 0) {
+			System.out.println("Opus decoder initialized");
+		}
+		else {
+			System.err.println("Opus decoder init failed: "+err);
+			return;
+		}
+		
+		switch (OpusDecoder.getChannelCount())
+		{
+		case 1:
+			channelConfig = AudioFormat.CHANNEL_OUT_MONO;
+			break;
+		case 2:
+			channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
+			break;
+		default:
+			System.err.println("Unsupported channel count");
+			return;
+		}
+				
+		track = new AudioTrack(AudioManager.STREAM_MUSIC,
+				OpusDecoder.getSampleRate(),
+				channelConfig,
+				AudioFormat.ENCODING_PCM_16BIT,
+				1024, // 1KB buffer
+				AudioTrack.MODE_STREAM);
+		
+		track.play();
+	}
+	
+	private void startDepacketizerThread()
+	{
+		// This thread lessens the work on the receive thread
+		// so it can spend more time waiting for data
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				for (;;)
+				{
+					AvRtpPacket packet;
+					
+					try {
+						packet = packets.take();
+					} catch (InterruptedException e) {
+						e.printStackTrace();
+						return;
+					}
+					
+					// !!! We no longer own the data buffer at this point !!!
+					depacketizer.decodeInputData(packet);
+				}
+			}
+		}).start();
+	}
+	
+	private void startDecoderThread()
+	{
+		// Decoder thread
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				for (;;)
+				{
+					short[] samples;
+					try {
+						samples = depacketizer.getNextDecodedData();
+					} catch (InterruptedException e) {
+						e.printStackTrace();
+						return;
+					}
+					
+					track.write(samples, 0, samples.length);
+				}
+			}
+		}).start();
 	}
 	
 	private void startReceiveThread()
@@ -101,24 +219,4 @@ public class NvAudioStream {
 			}
 		}).start();
 	}
-	
-	/*public void startStream(String host) throws SocketException, UnknownHostException
-	{
-		System.out.println("Starting audio group");
-		group = new AudioGroup();
-		group.setMode(AudioGroup.MODE_NORMAL);
-		
-		System.out.println("Starting audio stream");
-		stream = new AudioStream(InetAddress.getByAddress(new byte[]{0,0,0,0}));
-		stream.setMode(AudioStream.MODE_NORMAL);
-		stream.associate(InetAddress.getByName(host), PORT);
-		stream.setCodec(AudioCodec.PCMA);
-		stream.join(group);
-		
-		for (AudioCodec c : AudioCodec.getCodecs())
-			System.out.println(c.type + " " + c.fmtp + " " + c.rtpmap);
-		
-		System.out.println("Joined");
-	}*/
-
 }

src/com/limelight/nvstream/NvConnection.java

@@ -82,6 +82,7 @@ public class NvConnection {
 					startSteamBigPicture();
 					performHandshake();
 					startVideo(video);
+					startAudio();
 					beginControlStream();
 					controlStream.startJitterPackets();
 					startController();
@@ -101,6 +102,11 @@ public class NvConnection {
 		new NvVideoStream().startVideoStream(host, surface);
 	}
 	
+	public void startAudio()
+	{
+		new NvAudioStream().startAudioStream(host);
+	}
+	
 	public void sendMouseMove(final short deltaX, final short deltaY)
 	{
 		if (inputStream == null)

src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java

@@ -0,0 +1,59 @@
+package com.limelight.nvstream.av.audio;
+
+import java.util.concurrent.LinkedBlockingQueue;
+
+import com.limelight.nvstream.av.AvBufferDescriptor;
+import com.limelight.nvstream.av.AvRtpPacket;
+
+public class AvAudioDepacketizer {
+	private LinkedBlockingQueue<short[]> decodedUnits = new LinkedBlockingQueue<short[]>();
+	
+	// Sequencing state
+	private short lastSequenceNumber;
+	
+	public void decodeInputData(AvRtpPacket packet)
+	{
+		short seq = packet.getSequenceNumber();
+		
+		if (packet.getPacketType() != 97) {
+			// Only type 97 is audio
+			return;
+		}
+		
+		// Toss out the current NAL if we receive a packet that is
+		// out of sequence
+		if (lastSequenceNumber != 0 &&
+			(short)(lastSequenceNumber + 1) != seq)
+		{
+			System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
+			
+			// Tell the decoder about this
+			//OpusDecoder.decode(null, 0, 0, null);
+		}
+		
+		lastSequenceNumber = seq;		
+		
+		// This is all the depacketizing we need to do
+		AvBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor();
+
+		// Submit this data to the decoder
+		short[] pcmData = new short[OpusDecoder.getMaxOutputShorts()];
+		
+		int decodeLen = OpusDecoder.decode(rtpPayload.data, rtpPayload.offset, rtpPayload.length, pcmData);
+		
+		// Return value of decode is frames decoded per channel
+		decodeLen *= OpusDecoder.getChannelCount();
+		
+		if (decodeLen > 0) {
+			// Jank!
+			short[] trimmedPcmData = new short[decodeLen];
+			System.arraycopy(pcmData, 0, trimmedPcmData, 0, decodeLen);
+			decodedUnits.add(trimmedPcmData);
+		}
+	}
+	
+	public short[] getNextDecodedData() throws InterruptedException
+	{
+		return decodedUnits.take();
+	}
+}

src/com/limelight/nvstream/av/audio/OpusDecoder.java

@@ -0,0 +1,14 @@
+package com.limelight.nvstream.av.audio;
+
+public class OpusDecoder {
+	static {
+		System.loadLibrary("nv_opus_dec");
+	}
+	
+	public static native int init();
+	public static native void destroy();
+	public static native int getChannelCount();
+	public static native int getMaxOutputShorts();
+	public static native int getSampleRate();
+	public static native int decode(byte[] indata, int inoff, int inlen, short[] outpcmdata);
+}

src/com/limelight/nvstream/av/video/AvVideoDepacketizer.java

@@ -161,7 +161,7 @@ public class AvVideoDepacketizer {
 		if (lastSequenceNumber != 0 &&
 			(short)(lastSequenceNumber + 1) != seq)
 		{
-			System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
+			System.out.println("Received OOS video data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
 			
 			// Reset the depacketizer state
 			currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;