Stage WIP audio support

src/com/limelight/nvstream/NvVideoStream.java

@@ -30,8 +30,8 @@ public class NvVideoStream {
 	public static final int RTCP_PORT = 47999;
 	public static final int FIRST_FRAME_PORT = 47996;
 	
-	private ByteBuffer[] videoDecoderInputBuffers = null;
+	private ByteBuffer[] videoDecoderInputBuffers, audioDecoderInputBuffers;
-	private MediaCodec videoDecoder;
+	private MediaCodec videoDecoder, audioDecoder;
 	
 	private LinkedBlockingQueue<AvRtpPacket> packets = new LinkedBlockingQueue<AvRtpPacket>();
 	
@@ -94,10 +94,19 @@ public class NvVideoStream {
 		videoDecoder = MediaCodec.createDecoderByType("video/avc");
 		MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720);
 
+		audioDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm");
+		MediaFormat audioFormat = MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2);
+
 		videoDecoder.configure(videoFormat, surface, null, 0);
+		audioDecoder.configure(audioFormat, null, null, 0);
+
 		videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT);
+		
 		videoDecoder.start();
+		audioDecoder.start();
+
 		videoDecoderInputBuffers = videoDecoder.getInputBuffers();
+		audioDecoderInputBuffers = audioDecoder.getInputBuffers();
 	}
 
 	public void startVideoStream(final String host, final Surface surface)
@@ -130,6 +139,9 @@ public class NvVideoStream {
 				// Start decoding the data we're receiving
 				startDecoderThread();
 				
+				// Start playing back audio data
+				startAudioPlaybackThread();
+				
 				// Read the first frame to start the UDP video stream
 				try {
 					readFirstFrame(host);
@@ -188,6 +200,32 @@ public class NvVideoStream {
 							}
 						}
 						break;
+						
+						case AvDecodeUnit.TYPE_AAC:
+						{
+							int inputIndex = audioDecoder.dequeueInputBuffer(0);
+							if (inputIndex == -4)
+							{
+								ByteBuffer buf = audioDecoderInputBuffers[inputIndex];
+								
+								// Clear old input data
+								buf.clear();
+								
+								// Copy data from our buffer list into the input buffer
+								for (AvBufferDescriptor desc : du.getBufferList())
+								{
+									buf.put(desc.data, desc.offset, desc.length);
+									
+									// Release the buffer back to the buffer pool
+									pool.free(desc.data);
+								}
+
+								audioDecoder.queueInputBuffer(inputIndex,
+											0, du.getDataLength(),
+											0, du.getFlags());
+							}
+						}
+						break;
 					
 						default:
 						{
@@ -284,6 +322,40 @@ public class NvVideoStream {
 		}).start();
 	}
 	
+	private void startAudioPlaybackThread()
+	{
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				for (;;)
+				{
+					BufferInfo info = new BufferInfo();
+					System.out.println("Waiting for audio");
+					int outIndex = audioDecoder.dequeueOutputBuffer(info, -1);
+					System.out.println("Got audio");
+				    switch (outIndex) {
+				    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
+				    	System.out.println("Output buffers changed");
+					    break;
+				    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
+				    	System.out.println("Output format changed");
+				    	System.out.println("New output Format: " + videoDecoder.getOutputFormat());
+				    	break;
+				    case MediaCodec.INFO_TRY_AGAIN_LATER:
+				    	System.out.println("Try again later");
+				    	break;
+				    default:
+				      break;
+				    }
+				    if (outIndex >= 0) {
+				    	audioDecoder.releaseOutputBuffer(outIndex, true);
+				    }
+			    	
+				}
+			}
+		}).start();
+	}
+	
 	private void outputDisplayLoop()
 	{
 		for (;;)

src/com/limelight/nvstream/av/AvDecodeUnit.java

@@ -5,6 +5,7 @@ import java.util.List;
 public class AvDecodeUnit {
 	public static final int TYPE_UNKNOWN = 0;
 	public static final int TYPE_H264 = 1;
+	public static final int TYPE_AAC = 2;
 	
 	private int type;
 	private List<AvBufferDescriptor> bufferList;

src/com/limelight/nvstream/av/AvDepacketizer.java

@@ -8,24 +8,49 @@ import android.media.MediaCodec;
 public class AvDepacketizer {
 	
 	// Current NAL state
-	private LinkedList<AvBufferDescriptor> nalDataChain = null;
+	private LinkedList<AvBufferDescriptor> avcNalDataChain = null;
-	private int nalDataLength = 0;
+	private int avcNalDataLength = 0;
+	private LinkedList<AvBufferDescriptor> aacNalDataChain = null;
+	private int aacNalDataLength = 0;
+	private int currentlyDecoding;
 	
 	// Sequencing state
 	private short lastSequenceNumber;
 	
 	private LinkedBlockingQueue<AvDecodeUnit> decodedUnits = new LinkedBlockingQueue<AvDecodeUnit>();
 	
-	private void reassembleNal()
+	private void reassembleAacNal()
+	{
+		// This is the start of a new AAC NAL
+		if (aacNalDataChain != null && aacNalDataLength != 0)
+		{
+			System.out.println("Assembling AAC NAL: "+aacNalDataLength);
+			
+			/*AvBufferDescriptor header = aacNalDataChain.getFirst();
+			for (int i = 0; i < header.length; i++)
+				System.out.printf("%02x ", header.data[header.offset+i]);
+			System.out.println();*/
+			
+			// Construct the AAC decode unit
+			AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_AAC, aacNalDataChain, aacNalDataLength, 0);
+			decodedUnits.add(du);
+			
+			// Clear old state
+			aacNalDataChain = null;
+			aacNalDataLength = 0;
+		}
+	}
+	
+	private void reassembleAvcNal()
 	{
 		// This is the start of a new NAL
-		if (nalDataChain != null && nalDataLength != 0)
+		if (avcNalDataChain != null && avcNalDataLength != 0)
 		{
 			int flags = 0;
 			
 			// Check if this is a special NAL unit
-			AvBufferDescriptor header = nalDataChain.getFirst();
+			AvBufferDescriptor header = avcNalDataChain.getFirst();
-			AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header);
+			AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(header);
 			
 			if (specialSeq != null)
 			{
@@ -69,105 +94,98 @@ public class AvDepacketizer {
 			}
 
 			// Construct the H264 decode unit
-			AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags);
+			AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags);
 			decodedUnits.add(du);
 			
 			// Clear old state
-			nalDataChain = null;
+			avcNalDataChain = null;
-			nalDataLength = 0;
+			avcNalDataLength = 0;
 		}
 	}
 	
 	public void addInputData(AvPacket packet)
 	{
 		AvBufferDescriptor location = packet.getNewPayloadDescriptor();
-		int payloadLength = location.length;
-		boolean terminateNal = false;
 		
 		while (location.length != 0)
 		{
 			// Remember the start of the NAL data in this packet
 			int start = location.offset;
 			
-			// Check for the start sequence
+			// Check for a special sequence
-			AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
+			AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(location);
-			if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
+			if (specialSeq != null)
 			{
-				// Reassemble any pending NAL
+				if (NAL.isAvcStartSequence(specialSeq))
-				reassembleNal();
+				{
-				
+					// We're decoding H264 now
-				// Setup state for the new NAL
+					currentlyDecoding = AvDecodeUnit.TYPE_H264;
-				nalDataChain = new LinkedList<AvBufferDescriptor>();
+					
-				nalDataLength = 0;
+					// Check if it's the end of the last frame
+					if (NAL.isAvcFrameStart(specialSeq))
+					{
+						// Reassemble any pending AVC NAL
+						reassembleAvcNal();
+						
+						// Setup state for the new NAL
+						avcNalDataChain = new LinkedList<AvBufferDescriptor>();
+						avcNalDataLength = 0;
+					}
+				}
+				else if (NAL.isAacStartSequence(specialSeq))
+				{
+					// We're decoding AAC now
+					currentlyDecoding = AvDecodeUnit.TYPE_AAC;
+					
+					// Reassemble any pending AAC NAL
+					reassembleAacNal();
+					
+					// Setup state for the new NAL
+					aacNalDataChain = new LinkedList<AvBufferDescriptor>();
+					aacNalDataLength = 0;
+				}
+				else
+				{
+					// Not either sequence we want
+					//currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
+				}
 				
 				// Skip the start sequence
 				location.length -= specialSeq.length;
 				location.offset += specialSeq.length;
 			}
 			
-			// If there's a NAL assembly in progress, add the current data
+			// Move to the next special sequence
-			if (nalDataChain != null)
+			while (location.length != 0)
 			{
-				// FIXME: This is a hack to make parsing full packets
+				specialSeq = NAL.getSpecialSequenceDescriptor(location);
-				// take less time. We assume if they don't start with
+				
-				// a NAL start sequence, they're full of NAL data
+				// Check if this should end the current NAL
-				if (payloadLength == 968)
+				if (specialSeq != null)
 				{
-					location.offset += location.length;
+					break;
-					location.length = 0;
 				}
 				else
 				{
-					//System.out.println("Using slow parsing case");
+					// This byte is part of the NAL data
-					while (location.length != 0)
+					location.offset++;
-					{
+					location.length--;
-						specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
-						
-						// Check if this should end the current NAL
-						//if (specialSeq != null)
-						if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
-						{
-							//terminateNal = true;
-							break;
-						}
-						else
-						{
-							// This byte is part of the NAL data
-							location.offset++;
-							location.length--;
-						}
-					}
-				}
-				
-				int endSub;
-				
-				// If parsing was finished due to reaching new start sequence,
-				// remove the last byte from the NAL (since it's the first byte of the
-				// start of the next one)
-				if (location.length != 0)
-				{
-					endSub = 1;
-				}
-				else
-				{
-					endSub = 0;
-				}
-				
-				// Add a buffer descriptor describing the NAL data in this packet
-				nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub));
-				nalDataLength += location.offset-start-endSub;
-				
-				// Terminate the NAL if asked
-				if (terminateNal)
-				{
-					reassembleNal();
 				}
 			}
-			else
+			
+			AvBufferDescriptor data = new AvBufferDescriptor(location.data, start, location.offset-start);
+			
+			if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null)
 			{
-				// Otherwise, skip the data
+				// Add a buffer descriptor describing the NAL data in this packet
-				location.offset++;
+				avcNalDataChain.add(data);
-				location.length--;
+				avcNalDataLength += location.offset-start;
+			}
+			else if (currentlyDecoding == AvDecodeUnit.TYPE_AAC && aacNalDataChain != null)
+			{
+				// Add a buffer descriptor describing the NAL data in this packet
+				aacNalDataChain.add(data);
+				aacNalDataLength += location.offset-start;
 			}
 		}
 	}
@@ -182,8 +200,13 @@ public class AvDepacketizer {
 			lastSequenceNumber + 1 != seq)
 		{
 			System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
-			nalDataChain = null;
+			
-			nalDataLength = 0;
+			// Reset the depacketizer state
+			currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
+			avcNalDataChain = null;
+			avcNalDataLength = 0;
+			aacNalDataChain = null;
+			aacNalDataLength = 0;
 		}
 		
 		lastSequenceNumber = seq;
@@ -199,18 +222,38 @@ public class AvDepacketizer {
 	}
 }
 
-class H264NAL {
+class NAL {
 	
-	// This assume's that the buffer passed in is already a special sequence
+	// This assumes that the buffer passed in is already a special sequence
-	public static boolean isStartSequence(AvBufferDescriptor specialSeq)
+	public static boolean isAvcStartSequence(AvBufferDescriptor specialSeq)
 	{
-		if (/*specialSeq.length != 3 && */specialSeq.length != 4)
+		if (specialSeq.length != 3 && specialSeq.length != 4)
 			return false;
 		
 		// The start sequence is 00 00 01 or 00 00 00 01
 		return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
 	}
 	
+	// This assumes that the buffer passed in is already a special sequence
+	public static boolean isAacStartSequence(AvBufferDescriptor specialSeq)
+	{
+		if (specialSeq.length != 3)
+			return false;
+		
+		// The start sequence is 00 00 03
+		return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x03);
+	}
+	
+	// This assumes that the buffer passed in is already a special sequence
+	public static boolean isAvcFrameStart(AvBufferDescriptor specialSeq)
+	{
+		if (specialSeq.length != 4)
+			return false;
+		
+		// The frame start sequence is 00 00 00 01
+		return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
+	}
+	
 	// Returns a buffer descriptor describing the start sequence
 	public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer)
 	{