From e0676690445e17c31caf1a79d04793cdf3eed4df Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 30 Oct 2013 22:23:11 -0400 Subject: [PATCH] Stage WIP audio support --- src/com/limelight/nvstream/NvVideoStream.java | 76 ++++++- .../limelight/nvstream/av/AvDecodeUnit.java | 1 + .../limelight/nvstream/av/AvDepacketizer.java | 205 +++++++++++------- 3 files changed, 199 insertions(+), 83 deletions(-) diff --git a/src/com/limelight/nvstream/NvVideoStream.java b/src/com/limelight/nvstream/NvVideoStream.java index c4871f37..1c1695b8 100644 --- a/src/com/limelight/nvstream/NvVideoStream.java +++ b/src/com/limelight/nvstream/NvVideoStream.java @@ -30,8 +30,8 @@ public class NvVideoStream { public static final int RTCP_PORT = 47999; public static final int FIRST_FRAME_PORT = 47996; - private ByteBuffer[] videoDecoderInputBuffers = null; - private MediaCodec videoDecoder; + private ByteBuffer[] videoDecoderInputBuffers, audioDecoderInputBuffers; + private MediaCodec videoDecoder, audioDecoder; private LinkedBlockingQueue packets = new LinkedBlockingQueue(); @@ -94,10 +94,19 @@ public class NvVideoStream { videoDecoder = MediaCodec.createDecoderByType("video/avc"); MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720); + audioDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm"); + MediaFormat audioFormat = MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2); + videoDecoder.configure(videoFormat, surface, null, 0); + audioDecoder.configure(audioFormat, null, null, 0); + videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); + videoDecoder.start(); + audioDecoder.start(); + videoDecoderInputBuffers = videoDecoder.getInputBuffers(); + audioDecoderInputBuffers = audioDecoder.getInputBuffers(); } public void startVideoStream(final String host, final Surface surface) @@ -130,6 +139,9 @@ public class NvVideoStream { // Start decoding the data we're receiving startDecoderThread(); + // Start playing back audio data + startAudioPlaybackThread(); + // Read the first frame to start the UDP video stream try { readFirstFrame(host); @@ -188,6 +200,32 @@ public class NvVideoStream { } } break; + + case AvDecodeUnit.TYPE_AAC: + { + int inputIndex = audioDecoder.dequeueInputBuffer(0); + if (inputIndex == -4) + { + ByteBuffer buf = audioDecoderInputBuffers[inputIndex]; + + // Clear old input data + buf.clear(); + + // Copy data from our buffer list into the input buffer + for (AvBufferDescriptor desc : du.getBufferList()) + { + buf.put(desc.data, desc.offset, desc.length); + + // Release the buffer back to the buffer pool + pool.free(desc.data); + } + + audioDecoder.queueInputBuffer(inputIndex, + 0, du.getDataLength(), + 0, du.getFlags()); + } + } + break; default: { @@ -284,6 +322,40 @@ public class NvVideoStream { }).start(); } + private void startAudioPlaybackThread() + { + new Thread(new Runnable() { + @Override + public void run() { + for (;;) + { + BufferInfo info = new BufferInfo(); + System.out.println("Waiting for audio"); + int outIndex = audioDecoder.dequeueOutputBuffer(info, -1); + System.out.println("Got audio"); + switch (outIndex) { + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + System.out.println("Output buffers changed"); + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + System.out.println("Output format changed"); + System.out.println("New output Format: " + videoDecoder.getOutputFormat()); + break; + case MediaCodec.INFO_TRY_AGAIN_LATER: + System.out.println("Try again later"); + break; + default: + break; + } + if (outIndex >= 0) { + audioDecoder.releaseOutputBuffer(outIndex, true); + } + + } + } + }).start(); + } + private void outputDisplayLoop() { for (;;) diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java index ef3d8552..4574c696 100644 --- a/src/com/limelight/nvstream/av/AvDecodeUnit.java +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -5,6 +5,7 @@ import java.util.List; public class AvDecodeUnit { public static final int TYPE_UNKNOWN = 0; public static final int TYPE_H264 = 1; + public static final int TYPE_AAC = 2; private int type; private List bufferList; diff --git a/src/com/limelight/nvstream/av/AvDepacketizer.java b/src/com/limelight/nvstream/av/AvDepacketizer.java index 06e2390c..4874cb47 100644 --- a/src/com/limelight/nvstream/av/AvDepacketizer.java +++ b/src/com/limelight/nvstream/av/AvDepacketizer.java @@ -8,24 +8,49 @@ import android.media.MediaCodec; public class AvDepacketizer { // Current NAL state - private LinkedList nalDataChain = null; - private int nalDataLength = 0; + private LinkedList avcNalDataChain = null; + private int avcNalDataLength = 0; + private LinkedList aacNalDataChain = null; + private int aacNalDataLength = 0; + private int currentlyDecoding; // Sequencing state private short lastSequenceNumber; private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); - private void reassembleNal() + private void reassembleAacNal() + { + // This is the start of a new AAC NAL + if (aacNalDataChain != null && aacNalDataLength != 0) + { + System.out.println("Assembling AAC NAL: "+aacNalDataLength); + + /*AvBufferDescriptor header = aacNalDataChain.getFirst(); + for (int i = 0; i < header.length; i++) + System.out.printf("%02x ", header.data[header.offset+i]); + System.out.println();*/ + + // Construct the AAC decode unit + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_AAC, aacNalDataChain, aacNalDataLength, 0); + decodedUnits.add(du); + + // Clear old state + aacNalDataChain = null; + aacNalDataLength = 0; + } + } + + private void reassembleAvcNal() { // This is the start of a new NAL - if (nalDataChain != null && nalDataLength != 0) + if (avcNalDataChain != null && avcNalDataLength != 0) { int flags = 0; // Check if this is a special NAL unit - AvBufferDescriptor header = nalDataChain.getFirst(); - AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header); + AvBufferDescriptor header = avcNalDataChain.getFirst(); + AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(header); if (specialSeq != null) { @@ -69,105 +94,98 @@ public class AvDepacketizer { } // Construct the H264 decode unit - AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags); + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags); decodedUnits.add(du); // Clear old state - nalDataChain = null; - nalDataLength = 0; + avcNalDataChain = null; + avcNalDataLength = 0; } } public void addInputData(AvPacket packet) { AvBufferDescriptor location = packet.getNewPayloadDescriptor(); - int payloadLength = location.length; - boolean terminateNal = false; while (location.length != 0) { // Remember the start of the NAL data in this packet int start = location.offset; - // Check for the start sequence - AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location); - if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) + // Check for a special sequence + AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(location); + if (specialSeq != null) { - // Reassemble any pending NAL - reassembleNal(); - - // Setup state for the new NAL - nalDataChain = new LinkedList(); - nalDataLength = 0; + if (NAL.isAvcStartSequence(specialSeq)) + { + // We're decoding H264 now + currentlyDecoding = AvDecodeUnit.TYPE_H264; + + // Check if it's the end of the last frame + if (NAL.isAvcFrameStart(specialSeq)) + { + // Reassemble any pending AVC NAL + reassembleAvcNal(); + + // Setup state for the new NAL + avcNalDataChain = new LinkedList(); + avcNalDataLength = 0; + } + } + else if (NAL.isAacStartSequence(specialSeq)) + { + // We're decoding AAC now + currentlyDecoding = AvDecodeUnit.TYPE_AAC; + + // Reassemble any pending AAC NAL + reassembleAacNal(); + + // Setup state for the new NAL + aacNalDataChain = new LinkedList(); + aacNalDataLength = 0; + } + else + { + // Not either sequence we want + //currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN; + } // Skip the start sequence location.length -= specialSeq.length; location.offset += specialSeq.length; } - // If there's a NAL assembly in progress, add the current data - if (nalDataChain != null) + // Move to the next special sequence + while (location.length != 0) { - // FIXME: This is a hack to make parsing full packets - // take less time. We assume if they don't start with - // a NAL start sequence, they're full of NAL data - if (payloadLength == 968) + specialSeq = NAL.getSpecialSequenceDescriptor(location); + + // Check if this should end the current NAL + if (specialSeq != null) { - location.offset += location.length; - location.length = 0; + break; } else { - //System.out.println("Using slow parsing case"); - while (location.length != 0) - { - specialSeq = H264NAL.getSpecialSequenceDescriptor(location); - - // Check if this should end the current NAL - //if (specialSeq != null) - if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) - { - //terminateNal = true; - break; - } - else - { - // This byte is part of the NAL data - location.offset++; - location.length--; - } - } - } - - int endSub; - - // If parsing was finished due to reaching new start sequence, - // remove the last byte from the NAL (since it's the first byte of the - // start of the next one) - if (location.length != 0) - { - endSub = 1; - } - else - { - endSub = 0; - } - - // Add a buffer descriptor describing the NAL data in this packet - nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub)); - nalDataLength += location.offset-start-endSub; - - // Terminate the NAL if asked - if (terminateNal) - { - reassembleNal(); + // This byte is part of the NAL data + location.offset++; + location.length--; } } - else + + AvBufferDescriptor data = new AvBufferDescriptor(location.data, start, location.offset-start); + + if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null) { - // Otherwise, skip the data - location.offset++; - location.length--; + // Add a buffer descriptor describing the NAL data in this packet + avcNalDataChain.add(data); + avcNalDataLength += location.offset-start; + } + else if (currentlyDecoding == AvDecodeUnit.TYPE_AAC && aacNalDataChain != null) + { + // Add a buffer descriptor describing the NAL data in this packet + aacNalDataChain.add(data); + aacNalDataLength += location.offset-start; } } } @@ -182,8 +200,13 @@ public class AvDepacketizer { lastSequenceNumber + 1 != seq) { System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); - nalDataChain = null; - nalDataLength = 0; + + // Reset the depacketizer state + currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN; + avcNalDataChain = null; + avcNalDataLength = 0; + aacNalDataChain = null; + aacNalDataLength = 0; } lastSequenceNumber = seq; @@ -199,18 +222,38 @@ public class AvDepacketizer { } } -class H264NAL { +class NAL { - // This assume's that the buffer passed in is already a special sequence - public static boolean isStartSequence(AvBufferDescriptor specialSeq) + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcStartSequence(AvBufferDescriptor specialSeq) { - if (/*specialSeq.length != 3 && */specialSeq.length != 4) + if (specialSeq.length != 3 && specialSeq.length != 4) return false; // The start sequence is 00 00 01 or 00 00 00 01 return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); } + // This assumes that the buffer passed in is already a special sequence + public static boolean isAacStartSequence(AvBufferDescriptor specialSeq) + { + if (specialSeq.length != 3) + return false; + + // The start sequence is 00 00 03 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x03); + } + + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcFrameStart(AvBufferDescriptor specialSeq) + { + if (specialSeq.length != 4) + return false; + + // The frame start sequence is 00 00 00 01 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); + } + // Returns a buffer descriptor describing the start sequence public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer) {