From e6af9df142f60db996bd9daefd59b8b7a22d6739 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 29 Oct 2013 20:18:22 -0400 Subject: [PATCH 1/9] Rewrite video support so that it actually works. Add a janky NAL parser dissector. --- AndroidManifest.xml | 1 + LuaScripts/NALParser.lua | 62 ++++++ src/com/limelight/Connection.java | 12 +- src/com/limelight/nvstream/NvVideoStream.java | 192 ++++++++---------- .../nvstream/av/AvBufferDescriptor.java | 14 ++ .../limelight/nvstream/av/AvDecodeUnit.java | 34 ++++ src/com/limelight/nvstream/av/AvPacket.java | 18 ++ src/com/limelight/nvstream/av/AvParser.java | 131 ++++++++++++ 8 files changed, 350 insertions(+), 114 deletions(-) create mode 100644 LuaScripts/NALParser.lua create mode 100644 src/com/limelight/nvstream/av/AvBufferDescriptor.java create mode 100644 src/com/limelight/nvstream/av/AvDecodeUnit.java create mode 100644 src/com/limelight/nvstream/av/AvPacket.java create mode 100644 src/com/limelight/nvstream/av/AvParser.java diff --git a/AndroidManifest.xml b/AndroidManifest.xml index dc47bbfc..1318cd0f 100644 --- a/AndroidManifest.xml +++ b/AndroidManifest.xml @@ -27,6 +27,7 @@ = 0) - { - ByteBuffer buf = decoderInputBuffers[inputIndex]; - - buf.clear(); - buf.put(firstFrame); - - decoder.queueInputBuffer(inputIndex, - 0, firstFrame.length, - 0, 0); - frameIndex++; - } + videoDecoder.configure(videoFormat, surface, null, 0); + videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); + videoDecoder.start(); + videoDecoderInputBuffers = videoDecoder.getInputBuffers(); final RTPSession session = new RTPSession(rtp, rtcp); session.addParticipant(new Participant(host, RTP_PORT, RTCP_PORT)); - //session.RTPSessionRegister(NvVideoStream.this, null, null); // Ping thread new Thread(new Runnable() { @@ -129,64 +120,103 @@ public class NvVideoStream implements RTPAppIntf { } }).start(); - // Receive thread + // Decoder thread new Thread(new Runnable() { @Override public void run() { - byte[] packet = new byte[1500]; - - // Send PING every 100 ms + // Read the decode units generated from the RTP stream for (;;) { - DatagramPacket dp = new DatagramPacket(packet, 0, packet.length); - + AvDecodeUnit du; try { - rtp.receive(dp); - } catch (IOException e) { + du = parser.getNextDecodeUnit(); + } catch (InterruptedException e) { e.printStackTrace(); - break; + return; } - System.out.println("in receiveData"); - int inputIndex = decoder.dequeueInputBuffer(-1); - if (inputIndex >= 0) + switch (du.getType()) { - ByteBuffer buf = decoderInputBuffers[inputIndex]; - NvVideoPacket nvVideo = new NvVideoPacket(dp.getData()); - - buf.clear(); - buf.put(nvVideo.data); - - System.out.println(nvVideo); - if (nvVideo.length == 0xc803) { - decoder.queueInputBuffer(inputIndex, - 0, nvVideo.length, - 0, 0); - frameIndex++; - } else { - decoder.queueInputBuffer(inputIndex, - 0, 0, - 0, 0); + case AvDecodeUnit.TYPE_H264: + { + int inputIndex = videoDecoder.dequeueInputBuffer(-1); + if (inputIndex >= 0) + { + ByteBuffer buf = videoDecoderInputBuffers[inputIndex]; + + // Clear old input data + buf.clear(); + + // Copy data from our buffer list into the input buffer + for (AvBufferDescriptor desc : du.getBufferList()) + { + buf.put(desc.data, desc.offset, desc.length); + } + + videoDecoder.queueInputBuffer(inputIndex, + 0, du.getDataLength(), + 0, 0); + } } + break; + + default: + { + System.out.println("Unknown decode unit type"); + } + break; } } } }).start(); + // Receive thread + new Thread(new Runnable() { + + @Override + public void run() { + byte[] buffer = new byte[1500]; + AvBufferDescriptor desc = new AvBufferDescriptor(null, 0, 0); + + for (;;) + { + DatagramPacket packet = new DatagramPacket(buffer, buffer.length); + + try { + rtp.receive(packet); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return; + } + + desc.length = packet.getLength(); + desc.offset = packet.getOffset(); + desc.data = packet.getData(); + + // Skip the RTP header + desc.offset += 12; + desc.length -= 12; + + // Give the data to the AV parser + parser.addInputData(new AvPacket(desc)); + + } + } + + }).start(); + for (;;) { BufferInfo info = new BufferInfo(); - System.out.println("dequeuing outputbuffer"); - int outIndex = decoder.dequeueOutputBuffer(info, -1); - System.out.println("done dequeuing output buffer"); + int outIndex = videoDecoder.dequeueOutputBuffer(info, -1); switch (outIndex) { case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: System.out.println("Output buffers changed"); break; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: System.out.println("Output format changed"); - //decoderOutputFormat = decoder.getOutputFormat(); - System.out.println("New output Format: " + decoder.getOutputFormat()); + System.out.println("New output Format: " + videoDecoder.getOutputFormat()); break; case MediaCodec.INFO_TRY_AGAIN_LATER: System.out.println("Try again later"); @@ -195,28 +225,13 @@ public class NvVideoStream implements RTPAppIntf { break; } if (outIndex >= 0) { - System.out.println("releasing output buffer"); - decoder.releaseOutputBuffer(outIndex, true); - System.out.println("output buffer released"); + videoDecoder.releaseOutputBuffer(outIndex, true); } } } }).start(); } - - @Override - public void receiveData(DataFrame frame, Participant participant) { - } - - @Override - public void userEvent(int type, Participant[] participant) { - } - - @Override - public int frameSize(int payloadType) { - return 1; - } /** * Generates the presentation time for frame N, in microseconds. @@ -224,33 +239,4 @@ public class NvVideoStream implements RTPAppIntf { private static long computePresentationTime(int frameIndex) { return 132 + frameIndex * 1000000 / FRAME_RATE; } - - class NvVideoPacket { - byte[] preamble; - short length; - byte[] extra; - byte[] data; - - public NvVideoPacket(byte[] payload) - { - ByteBuffer bb = ByteBuffer.wrap(payload).order(ByteOrder.LITTLE_ENDIAN); - - preamble = new byte[12+16]; - extra = new byte[38]; - - bb.get(preamble); - length = bb.getShort(); - bb.get(extra); - data = new byte[length]; - - if (bb.remaining() + length <= payload.length) - bb.get(data); - } - - public String toString() - { - return "";//String.format("Length: %d | %02x %02x %02x %02x %02x %02x %02x %02x", - //length, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); - } - } } diff --git a/src/com/limelight/nvstream/av/AvBufferDescriptor.java b/src/com/limelight/nvstream/av/AvBufferDescriptor.java new file mode 100644 index 00000000..73090237 --- /dev/null +++ b/src/com/limelight/nvstream/av/AvBufferDescriptor.java @@ -0,0 +1,14 @@ +package com.limelight.nvstream.av; + +public class AvBufferDescriptor { + public byte[] data; + public int offset; + public int length; + + public AvBufferDescriptor(byte[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } +} diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java new file mode 100644 index 00000000..693503dc --- /dev/null +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -0,0 +1,34 @@ +package com.limelight.nvstream.av; + +import java.util.List; + +public class AvDecodeUnit { + public static final int TYPE_UNKNOWN = 0; + public static final int TYPE_H264 = 1; + + private int type; + private List bufferList; + private int dataLength; + + public AvDecodeUnit(int type, List bufferList, int dataLength) + { + this.type = type; + this.bufferList = bufferList; + this.dataLength = dataLength; + } + + public int getType() + { + return type; + } + + public List getBufferList() + { + return bufferList; + } + + public int getDataLength() + { + return dataLength; + } +} diff --git a/src/com/limelight/nvstream/av/AvPacket.java b/src/com/limelight/nvstream/av/AvPacket.java new file mode 100644 index 00000000..cf49aedb --- /dev/null +++ b/src/com/limelight/nvstream/av/AvPacket.java @@ -0,0 +1,18 @@ +package com.limelight.nvstream.av; + +public class AvPacket { + private AvBufferDescriptor buffer; + + public AvPacket(AvBufferDescriptor rtpPayload) + { + byte[] data = new byte[rtpPayload.length]; + System.arraycopy(rtpPayload.data, rtpPayload.offset, data, 0, rtpPayload.length); + buffer = new AvBufferDescriptor(data, 0, data.length); + } + + public AvBufferDescriptor getPayload() + { + int payloadOffset = buffer.offset+56; + return new AvBufferDescriptor(buffer.data, payloadOffset, buffer.length-payloadOffset); + } +} diff --git a/src/com/limelight/nvstream/av/AvParser.java b/src/com/limelight/nvstream/av/AvParser.java new file mode 100644 index 00000000..0878831d --- /dev/null +++ b/src/com/limelight/nvstream/av/AvParser.java @@ -0,0 +1,131 @@ +package com.limelight.nvstream.av; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; + +public class AvParser { + + // Current NAL state + private LinkedList nalDataChain; + private int nalDataLength; + + private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); + + private void reassembleNal() + { + // This is the start of a new NAL + if (nalDataChain != null && nalDataLength != 0) + { + // Construct the H264 decode unit + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength); + decodedUnits.add(du); + + // Clear old state + nalDataChain = null; + nalDataLength = 0; + } + } + + public void addInputData(AvPacket packet) + { + AvBufferDescriptor payload = packet.getPayload(); + AvBufferDescriptor location = new AvBufferDescriptor(payload.data, payload.offset, payload.length); + + while (location.length != 0) + { + // Remember the start of the NAL data in this packet + int start = location.offset; + + // Check for the start sequence + if (H264NAL.hasStartSequence(location)) + { + // Reassemble any pending NAL + reassembleNal(); + + // Setup state for the new NAL + nalDataChain = new LinkedList(); + nalDataLength = 0; + + // Skip the start sequence and the type byte + location.length -= 5; + location.offset += 5; + } + + // If there's a NAL assembly in progress, add the current data + if (nalDataChain != null) + { + // FIXME: This is a hack to make parsing full packets + // take less time. We assume if they don't start with + // a NAL start sequence, they're full of NAL data + if (payload.length == 968) + { + location.offset += location.length; + location.length = 0; + } + else + { + System.out.println("Using slow parsing case"); + while (location.length != 0) + { + // Check if this should end the current NAL + if (H264NAL.hasStartSequence(location)) + { + break; + } + else + { + // This byte is part of the NAL data + location.offset++; + location.length--; + } + } + } + + // Add a buffer descriptor describing the NAL data in this packet + nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start)); + nalDataLength += location.offset-start; + } + else + { + // Otherwise, skip the data + location.offset++; + location.length--; + } + } + } + + public AvDecodeUnit getNextDecodeUnit() throws InterruptedException + { + return decodedUnits.take(); + } +} + +class H264NAL { + public static boolean shouldTerminateNal(AvBufferDescriptor buffer) + { + if (buffer.length < 4) + return false; + + if (buffer.data[buffer.offset] != 0x00 || + buffer.data[buffer.offset+1] != 0x00 || + buffer.data[buffer.offset+2] != 0x00) + { + return false; + } + + return true; + } + + public static boolean hasStartSequence(AvBufferDescriptor buffer) + { + // NAL start sequence is 00 00 00 01 + if (!shouldTerminateNal(buffer)) + return false; + + if (buffer.data[buffer.offset+3] != 0x01) + return false; + + return true; + } +} From e5126ebe012cdceb051227a5cfbfc4bd183a17a5 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 29 Oct 2013 21:39:57 -0400 Subject: [PATCH 2/9] Use a packet buffer pool to reduce memory pressure --- src/com/limelight/nvstream/NvVideoStream.java | 25 +++++++--------- .../limelight/nvstream/av/AvBufferPool.java | 30 +++++++++++++++++++ src/com/limelight/nvstream/av/AvPacket.java | 9 ++---- src/com/limelight/nvstream/av/AvParser.java | 18 +++++------ 4 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 src/com/limelight/nvstream/av/AvBufferPool.java diff --git a/src/com/limelight/nvstream/NvVideoStream.java b/src/com/limelight/nvstream/NvVideoStream.java index 6ec2e3c2..1fe24d0c 100644 --- a/src/com/limelight/nvstream/NvVideoStream.java +++ b/src/com/limelight/nvstream/NvVideoStream.java @@ -16,6 +16,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import com.limelight.nvstream.av.AvBufferDescriptor; +import com.limelight.nvstream.av.AvBufferPool; import com.limelight.nvstream.av.AvDecodeUnit; import com.limelight.nvstream.av.AvPacket; import com.limelight.nvstream.av.AvParser; @@ -35,10 +36,11 @@ public class NvVideoStream { public static final int RTCP_PORT = 47999; public static final int FIRST_FRAME_PORT = 47996; - private static final int FRAME_RATE = 60; private ByteBuffer[] videoDecoderInputBuffers = null; private MediaCodec videoDecoder; + private AvBufferPool pool = new AvBufferPool(1500); + private AvParser parser = new AvParser(); private InputStream getFirstFrame(String host) throws UnknownHostException, IOException @@ -151,6 +153,9 @@ public class NvVideoStream { for (AvBufferDescriptor desc : du.getBufferList()) { buf.put(desc.data, desc.offset, desc.length); + + // Release the buffer back to the buffer pool + pool.free(desc.data); } videoDecoder.queueInputBuffer(inputIndex, @@ -175,13 +180,11 @@ public class NvVideoStream { @Override public void run() { - byte[] buffer = new byte[1500]; + DatagramPacket packet = new DatagramPacket(pool.allocate(), 1500); AvBufferDescriptor desc = new AvBufferDescriptor(null, 0, 0); for (;;) - { - DatagramPacket packet = new DatagramPacket(buffer, buffer.length); - + { try { rtp.receive(packet); } catch (IOException e) { @@ -198,12 +201,13 @@ public class NvVideoStream { desc.offset += 12; desc.length -= 12; - // Give the data to the AV parser + // !!! We no longer own the data buffer at this point !!! parser.addInputData(new AvPacket(desc)); + // Get a new buffer from the buffer pool + packet.setData(pool.allocate(), 0, 1500); } } - }).start(); for (;;) @@ -232,11 +236,4 @@ public class NvVideoStream { } }).start(); } - - /** - * Generates the presentation time for frame N, in microseconds. - */ - private static long computePresentationTime(int frameIndex) { - return 132 + frameIndex * 1000000 / FRAME_RATE; - } } diff --git a/src/com/limelight/nvstream/av/AvBufferPool.java b/src/com/limelight/nvstream/av/AvBufferPool.java new file mode 100644 index 00000000..b0fe335e --- /dev/null +++ b/src/com/limelight/nvstream/av/AvBufferPool.java @@ -0,0 +1,30 @@ +package com.limelight.nvstream.av; + +import java.util.LinkedList; + +public class AvBufferPool { + private LinkedList bufferList = new LinkedList(); + private int bufferSize; + + public AvBufferPool(int size) + { + this.bufferSize = size; + } + + public synchronized byte[] allocate() + { + if (bufferList.isEmpty()) + { + return new byte[bufferSize]; + } + else + { + return bufferList.removeFirst(); + } + } + + public synchronized void free(byte[] buffer) + { + bufferList.addFirst(buffer); + } +} diff --git a/src/com/limelight/nvstream/av/AvPacket.java b/src/com/limelight/nvstream/av/AvPacket.java index cf49aedb..95c83ce1 100644 --- a/src/com/limelight/nvstream/av/AvPacket.java +++ b/src/com/limelight/nvstream/av/AvPacket.java @@ -5,14 +5,11 @@ public class AvPacket { public AvPacket(AvBufferDescriptor rtpPayload) { - byte[] data = new byte[rtpPayload.length]; - System.arraycopy(rtpPayload.data, rtpPayload.offset, data, 0, rtpPayload.length); - buffer = new AvBufferDescriptor(data, 0, data.length); + buffer = new AvBufferDescriptor(rtpPayload.data, rtpPayload.offset, rtpPayload.length); } - public AvBufferDescriptor getPayload() + public AvBufferDescriptor getNewPayloadDescriptor() { - int payloadOffset = buffer.offset+56; - return new AvBufferDescriptor(buffer.data, payloadOffset, buffer.length-payloadOffset); + return new AvBufferDescriptor(buffer.data, buffer.offset+56, buffer.length-56); } } diff --git a/src/com/limelight/nvstream/av/AvParser.java b/src/com/limelight/nvstream/av/AvParser.java index 0878831d..765c1100 100644 --- a/src/com/limelight/nvstream/av/AvParser.java +++ b/src/com/limelight/nvstream/av/AvParser.java @@ -1,7 +1,6 @@ package com.limelight.nvstream.av; import java.util.LinkedList; -import java.util.List; import java.util.concurrent.LinkedBlockingQueue; public class AvParser { @@ -16,7 +15,7 @@ public class AvParser { { // This is the start of a new NAL if (nalDataChain != null && nalDataLength != 0) - { + { // Construct the H264 decode unit AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength); decodedUnits.add(du); @@ -28,9 +27,10 @@ public class AvParser { } public void addInputData(AvPacket packet) - { - AvBufferDescriptor payload = packet.getPayload(); - AvBufferDescriptor location = new AvBufferDescriptor(payload.data, payload.offset, payload.length); + { + // This payload buffer descriptor belongs to us + AvBufferDescriptor location = packet.getNewPayloadDescriptor(); + int payloadLength = location.length; while (location.length != 0) { @@ -47,9 +47,9 @@ public class AvParser { nalDataChain = new LinkedList(); nalDataLength = 0; - // Skip the start sequence and the type byte - location.length -= 5; - location.offset += 5; + // Skip the start sequence + location.length -= 4; + location.offset += 4; } // If there's a NAL assembly in progress, add the current data @@ -58,7 +58,7 @@ public class AvParser { // FIXME: This is a hack to make parsing full packets // take less time. We assume if they don't start with // a NAL start sequence, they're full of NAL data - if (payload.length == 968) + if (payloadLength == 968) { location.offset += location.length; location.length = 0; From 8ba9d8cfc8ab69144bf91bffa14fa5d2c90fe6d0 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 30 Oct 2013 04:32:20 -0400 Subject: [PATCH 3/9] Disable putting buffers back in the pool until the double-free issue gets resolved --- src/com/limelight/nvstream/av/AvBufferPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/limelight/nvstream/av/AvBufferPool.java b/src/com/limelight/nvstream/av/AvBufferPool.java index b0fe335e..d02c75eb 100644 --- a/src/com/limelight/nvstream/av/AvBufferPool.java +++ b/src/com/limelight/nvstream/av/AvBufferPool.java @@ -25,6 +25,6 @@ public class AvBufferPool { public synchronized void free(byte[] buffer) { - bufferList.addFirst(buffer); + //bufferList.addFirst(buffer); } } From 303ffaa8a36ba28bf8d2c0d51eb23365b4859810 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 30 Oct 2013 18:21:12 -0400 Subject: [PATCH 4/9] Add a copy constructor for the buffer descriptor class --- src/com/limelight/nvstream/av/AvBufferDescriptor.java | 7 +++++++ src/com/limelight/nvstream/av/AvPacket.java | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/com/limelight/nvstream/av/AvBufferDescriptor.java b/src/com/limelight/nvstream/av/AvBufferDescriptor.java index 73090237..415ae758 100644 --- a/src/com/limelight/nvstream/av/AvBufferDescriptor.java +++ b/src/com/limelight/nvstream/av/AvBufferDescriptor.java @@ -11,4 +11,11 @@ public class AvBufferDescriptor { this.offset = offset; this.length = length; } + + public AvBufferDescriptor(AvBufferDescriptor desc) + { + this.data = desc.data; + this.offset = desc.offset; + this.length = desc.length; + } } diff --git a/src/com/limelight/nvstream/av/AvPacket.java b/src/com/limelight/nvstream/av/AvPacket.java index 95c83ce1..554f1778 100644 --- a/src/com/limelight/nvstream/av/AvPacket.java +++ b/src/com/limelight/nvstream/av/AvPacket.java @@ -5,7 +5,7 @@ public class AvPacket { public AvPacket(AvBufferDescriptor rtpPayload) { - buffer = new AvBufferDescriptor(rtpPayload.data, rtpPayload.offset, rtpPayload.length); + buffer = new AvBufferDescriptor(rtpPayload); } public AvBufferDescriptor getNewPayloadDescriptor() From 9e09ca2b7afdc9ab629854d1fc2d04462d067b07 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 30 Oct 2013 20:14:54 -0400 Subject: [PATCH 5/9] WIP: Add RTP class. Massive refactoring of NvVideoStream. Rename AvParser to AvDepacketizer. Begin parsing other special NAL sequences. --- src/com/limelight/nvstream/NvVideoStream.java | 424 ++++++++++-------- .../limelight/nvstream/av/AvDecodeUnit.java | 9 +- .../limelight/nvstream/av/AvDepacketizer.java | 274 +++++++++++ src/com/limelight/nvstream/av/AvParser.java | 131 ------ .../limelight/nvstream/av/AvRtpPacket.java | 32 ++ 5 files changed, 563 insertions(+), 307 deletions(-) create mode 100644 src/com/limelight/nvstream/av/AvDepacketizer.java delete mode 100644 src/com/limelight/nvstream/av/AvParser.java create mode 100644 src/com/limelight/nvstream/av/AvRtpPacket.java diff --git a/src/com/limelight/nvstream/NvVideoStream.java b/src/com/limelight/nvstream/NvVideoStream.java index 1fe24d0c..c4871f37 100644 --- a/src/com/limelight/nvstream/NvVideoStream.java +++ b/src/com/limelight/nvstream/NvVideoStream.java @@ -1,29 +1,23 @@ package com.limelight.nvstream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.DatagramPacket; import java.net.DatagramSocket; -import java.net.InetAddress; import java.net.Socket; import java.net.SocketException; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.util.concurrent.LinkedBlockingQueue; import com.limelight.nvstream.av.AvBufferDescriptor; import com.limelight.nvstream.av.AvBufferPool; import com.limelight.nvstream.av.AvDecodeUnit; import com.limelight.nvstream.av.AvPacket; -import com.limelight.nvstream.av.AvParser; +import com.limelight.nvstream.av.AvDepacketizer; +import com.limelight.nvstream.av.AvRtpPacket; -import jlibrtp.DataFrame; import jlibrtp.Participant; -import jlibrtp.RTPAppIntf; import jlibrtp.RTPSession; import android.media.MediaCodec; @@ -39,15 +33,72 @@ public class NvVideoStream { private ByteBuffer[] videoDecoderInputBuffers = null; private MediaCodec videoDecoder; + private LinkedBlockingQueue packets = new LinkedBlockingQueue(); + + private RTPSession session; + private DatagramSocket rtp; + private AvBufferPool pool = new AvBufferPool(1500); - private AvParser parser = new AvParser(); + private AvDepacketizer depacketizer = new AvDepacketizer(); - private InputStream getFirstFrame(String host) throws UnknownHostException, IOException + private InputStream openFirstFrameInputStream(String host) throws UnknownHostException, IOException { Socket s = new Socket(host, FIRST_FRAME_PORT); return s.getInputStream(); } + + private void readFirstFrame(String host) throws IOException + { + byte[] firstFrame = pool.allocate(); + System.out.println("VID: Waiting for first frame"); + InputStream firstFrameStream = openFirstFrameInputStream(host); + + int offset = 0; + for (;;) + { + int bytesRead = firstFrameStream.read(firstFrame, offset, firstFrame.length-offset); + + if (bytesRead == -1) + break; + + offset += bytesRead; + } + + System.out.println("VID: First frame read ("+offset+" bytes)"); + + // FIXME: Investigate: putting these NALs into the data stream + // causes the picture to get messed up + //depacketizer.addInputData(new AvPacket(new AvBufferDescriptor(firstFrame, 0, offset))); + } + + public void setupRtpSession(String host) throws SocketException + { + DatagramSocket rtcp; + + rtp = new DatagramSocket(RTP_PORT); + + rtp.setReceiveBufferSize(2097152); + System.out.println("RECV BUF: "+rtp.getReceiveBufferSize()); + System.out.println("SEND BUF: "+rtp.getSendBufferSize()); + + + rtcp = new DatagramSocket(RTCP_PORT); + + session = new RTPSession(rtp, rtcp); + session.addParticipant(new Participant(host, RTP_PORT, RTCP_PORT)); + } + + public void setupDecoders(Surface surface) + { + videoDecoder = MediaCodec.createDecoderByType("video/avc"); + MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720); + + videoDecoder.configure(videoFormat, surface, null, 0); + videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); + videoDecoder.start(); + videoDecoderInputBuffers = videoDecoder.getInputBuffers(); + } public void startVideoStream(final String host, final Surface surface) { @@ -55,185 +106,208 @@ public class NvVideoStream { @Override public void run() { + // Setup the decoder context + setupDecoders(surface); - byte[] firstFrame = new byte[98]; + // Open RTP sockets and start session try { - System.out.println("VID: Waiting for first frame"); - InputStream firstFrameStream = getFirstFrame(host); - - int offset = 0; - do - { - offset = firstFrameStream.read(firstFrame, offset, firstFrame.length-offset); - } while (offset != firstFrame.length); - System.out.println("VID: First frame read "); - } catch (UnknownHostException e2) { - // TODO Auto-generated catch block - e2.printStackTrace(); - return; - } catch (IOException e2) { - // TODO Auto-generated catch block - e2.printStackTrace(); - return; - } - - final DatagramSocket rtp, rtcp; - try { - rtp = new DatagramSocket(RTP_PORT); - rtcp = new DatagramSocket(RTCP_PORT); + setupRtpSession(host); } catch (SocketException e1) { - // TODO Auto-generated catch block e1.printStackTrace(); return; } - - videoDecoder = MediaCodec.createDecoderByType("video/avc"); - MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720); - - videoDecoder.configure(videoFormat, surface, null, 0); - videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); - videoDecoder.start(); - videoDecoderInputBuffers = videoDecoder.getInputBuffers(); - final RTPSession session = new RTPSession(rtp, rtcp); - session.addParticipant(new Participant(host, RTP_PORT, RTCP_PORT)); + // Start the receive thread early to avoid missing + // early packets + startReceiveThread(); - // Ping thread - new Thread(new Runnable() { - @Override - public void run() { - // PING in ASCII - final byte[] pingPacket = new byte[] {0x50, 0x49, 0x4E, 0x47}; - - // RTP payload type is 127 (dynamic) - session.payloadType(127); - - // Send PING every 100 ms - for (;;) - { - session.sendData(pingPacket); - - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - } - }).start(); + // Start the keepalive ping to keep the stream going + startUdpPingThread(); - // Decoder thread - new Thread(new Runnable() { - @Override - public void run() { - // Read the decode units generated from the RTP stream - for (;;) - { - AvDecodeUnit du; - try { - du = parser.getNextDecodeUnit(); - } catch (InterruptedException e) { - e.printStackTrace(); - return; - } - - switch (du.getType()) - { - case AvDecodeUnit.TYPE_H264: - { - int inputIndex = videoDecoder.dequeueInputBuffer(-1); - if (inputIndex >= 0) - { - ByteBuffer buf = videoDecoderInputBuffers[inputIndex]; - - // Clear old input data - buf.clear(); - - // Copy data from our buffer list into the input buffer - for (AvBufferDescriptor desc : du.getBufferList()) - { - buf.put(desc.data, desc.offset, desc.length); - - // Release the buffer back to the buffer pool - pool.free(desc.data); - } - - videoDecoder.queueInputBuffer(inputIndex, - 0, du.getDataLength(), - 0, 0); - } - } - break; - - default: - { - System.out.println("Unknown decode unit type"); - } - break; - } - } - } - }).start(); + // Start the depacketizer thread to deal with the RTP data + startDepacketizerThread(); - // Receive thread - new Thread(new Runnable() { - - @Override - public void run() { - DatagramPacket packet = new DatagramPacket(pool.allocate(), 1500); - AvBufferDescriptor desc = new AvBufferDescriptor(null, 0, 0); - - for (;;) - { - try { - rtp.receive(packet); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - return; - } - - desc.length = packet.getLength(); - desc.offset = packet.getOffset(); - desc.data = packet.getData(); - - // Skip the RTP header - desc.offset += 12; - desc.length -= 12; - - // !!! We no longer own the data buffer at this point !!! - parser.addInputData(new AvPacket(desc)); - - // Get a new buffer from the buffer pool - packet.setData(pool.allocate(), 0, 1500); - } - } - }).start(); + // Start decoding the data we're receiving + startDecoderThread(); + // Read the first frame to start the UDP video stream + try { + readFirstFrame(host); + } catch (IOException e2) { + e2.printStackTrace(); + return; + } + + // Render the frames that are coming out of the decoder + outputDisplayLoop(); + } + }).start(); + } + + private void startDecoderThread() + { + // Decoder thread + new Thread(new Runnable() { + @Override + public void run() { + // Read the decode units generated from the RTP stream for (;;) { - BufferInfo info = new BufferInfo(); - int outIndex = videoDecoder.dequeueOutputBuffer(info, -1); - switch (outIndex) { - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - System.out.println("Output buffers changed"); - break; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - System.out.println("Output format changed"); - System.out.println("New output Format: " + videoDecoder.getOutputFormat()); - break; - case MediaCodec.INFO_TRY_AGAIN_LATER: - System.out.println("Try again later"); - break; - default: - break; - } - if (outIndex >= 0) { - videoDecoder.releaseOutputBuffer(outIndex, true); - } - + AvDecodeUnit du; + try { + du = depacketizer.getNextDecodeUnit(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + + switch (du.getType()) + { + case AvDecodeUnit.TYPE_H264: + { + int inputIndex = videoDecoder.dequeueInputBuffer(-1); + if (inputIndex >= 0) + { + ByteBuffer buf = videoDecoderInputBuffers[inputIndex]; + + // Clear old input data + buf.clear(); + + // Copy data from our buffer list into the input buffer + for (AvBufferDescriptor desc : du.getBufferList()) + { + buf.put(desc.data, desc.offset, desc.length); + + // Release the buffer back to the buffer pool + pool.free(desc.data); + } + + videoDecoder.queueInputBuffer(inputIndex, + 0, du.getDataLength(), + 0, du.getFlags()); + } + } + break; + + default: + { + System.out.println("Unknown decode unit type"); + } + break; + } } } }).start(); } + + private void startDepacketizerThread() + { + // This thread lessens the work on the receive thread + // so it can spend more time waiting for data + new Thread(new Runnable() { + @Override + public void run() { + for (;;) + { + AvRtpPacket packet; + + try { + packet = packets.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + + // !!! We no longer own the data buffer at this point !!! + depacketizer.addInputData(packet); + } + } + }).start(); + } + + private void startReceiveThread() + { + // Receive thread + new Thread(new Runnable() { + @Override + public void run() { + DatagramPacket packet = new DatagramPacket(pool.allocate(), 1500); + AvBufferDescriptor desc = new AvBufferDescriptor(null, 0, 0); + + for (;;) + { + try { + rtp.receive(packet); + } catch (IOException e) { + e.printStackTrace(); + return; + } + + desc.length = packet.getLength(); + desc.offset = packet.getOffset(); + desc.data = packet.getData(); + + // Give the packet to the depacketizer thread + packets.add(new AvRtpPacket(desc)); + + // Get a new buffer from the buffer pool + packet.setData(pool.allocate(), 0, 1500); + } + } + }).start(); + } + + private void startUdpPingThread() + { + // Ping thread + new Thread(new Runnable() { + @Override + public void run() { + // PING in ASCII + final byte[] pingPacket = new byte[] {0x50, 0x49, 0x4E, 0x47}; + + // RTP payload type is 127 (dynamic) + session.payloadType(127); + + // Send PING every 100 ms + for (;;) + { + session.sendData(pingPacket); + + try { + Thread.sleep(100); + } catch (InterruptedException e) { + break; + } + } + } + }).start(); + } + + private void outputDisplayLoop() + { + for (;;) + { + BufferInfo info = new BufferInfo(); + int outIndex = videoDecoder.dequeueOutputBuffer(info, -1); + switch (outIndex) { + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + System.out.println("Output buffers changed"); + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + System.out.println("Output format changed"); + System.out.println("New output Format: " + videoDecoder.getOutputFormat()); + break; + case MediaCodec.INFO_TRY_AGAIN_LATER: + System.out.println("Try again later"); + break; + default: + break; + } + if (outIndex >= 0) { + videoDecoder.releaseOutputBuffer(outIndex, true); + } + + } + } } diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java index 693503dc..ef3d8552 100644 --- a/src/com/limelight/nvstream/av/AvDecodeUnit.java +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -9,12 +9,14 @@ public class AvDecodeUnit { private int type; private List bufferList; private int dataLength; + private int flags; - public AvDecodeUnit(int type, List bufferList, int dataLength) + public AvDecodeUnit(int type, List bufferList, int dataLength, int flags) { this.type = type; this.bufferList = bufferList; this.dataLength = dataLength; + this.flags = flags; } public int getType() @@ -22,6 +24,11 @@ public class AvDecodeUnit { return type; } + public int getFlags() + { + return flags; + } + public List getBufferList() { return bufferList; diff --git a/src/com/limelight/nvstream/av/AvDepacketizer.java b/src/com/limelight/nvstream/av/AvDepacketizer.java new file mode 100644 index 00000000..06e2390c --- /dev/null +++ b/src/com/limelight/nvstream/av/AvDepacketizer.java @@ -0,0 +1,274 @@ +package com.limelight.nvstream.av; + +import java.util.LinkedList; +import java.util.concurrent.LinkedBlockingQueue; + +import android.media.MediaCodec; + +public class AvDepacketizer { + + // Current NAL state + private LinkedList nalDataChain = null; + private int nalDataLength = 0; + + // Sequencing state + private short lastSequenceNumber; + + private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); + + private void reassembleNal() + { + // This is the start of a new NAL + if (nalDataChain != null && nalDataLength != 0) + { + int flags = 0; + + // Check if this is a special NAL unit + AvBufferDescriptor header = nalDataChain.getFirst(); + AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header); + + if (specialSeq != null) + { + // The next byte after the special sequence is the NAL header + byte nalHeader = specialSeq.data[specialSeq.offset+specialSeq.length]; + + switch (nalHeader) + { + // SPS and PPS + case 0x67: + case 0x68: + System.out.println("Codec config"); + flags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + break; + + // IDR + case 0x65: + System.out.println("Reference frame"); + flags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME; + break; + + // non-IDR frame + case 0x61: + break; + + // Unknown type + default: + System.out.printf("Unknown NAL header: %02x %02x %02x %02x %02x\n", + header.data[header.offset], header.data[header.offset+1], + header.data[header.offset+2], header.data[header.offset+3], + header.data[header.offset+4]); + break; + } + } + else + { + System.out.printf("Invalid NAL: %02x %02x %02x %02x %02x\n", + header.data[header.offset], header.data[header.offset+1], + header.data[header.offset+2], header.data[header.offset+3], + header.data[header.offset+4]); + } + + // Construct the H264 decode unit + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags); + decodedUnits.add(du); + + // Clear old state + nalDataChain = null; + nalDataLength = 0; + } + } + + public void addInputData(AvPacket packet) + { + AvBufferDescriptor location = packet.getNewPayloadDescriptor(); + int payloadLength = location.length; + boolean terminateNal = false; + + while (location.length != 0) + { + // Remember the start of the NAL data in this packet + int start = location.offset; + + // Check for the start sequence + AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location); + if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) + { + // Reassemble any pending NAL + reassembleNal(); + + // Setup state for the new NAL + nalDataChain = new LinkedList(); + nalDataLength = 0; + + // Skip the start sequence + location.length -= specialSeq.length; + location.offset += specialSeq.length; + } + + // If there's a NAL assembly in progress, add the current data + if (nalDataChain != null) + { + // FIXME: This is a hack to make parsing full packets + // take less time. We assume if they don't start with + // a NAL start sequence, they're full of NAL data + if (payloadLength == 968) + { + location.offset += location.length; + location.length = 0; + } + else + { + //System.out.println("Using slow parsing case"); + while (location.length != 0) + { + specialSeq = H264NAL.getSpecialSequenceDescriptor(location); + + // Check if this should end the current NAL + //if (specialSeq != null) + if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) + { + //terminateNal = true; + break; + } + else + { + // This byte is part of the NAL data + location.offset++; + location.length--; + } + } + } + + int endSub; + + // If parsing was finished due to reaching new start sequence, + // remove the last byte from the NAL (since it's the first byte of the + // start of the next one) + if (location.length != 0) + { + endSub = 1; + } + else + { + endSub = 0; + } + + // Add a buffer descriptor describing the NAL data in this packet + nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub)); + nalDataLength += location.offset-start-endSub; + + // Terminate the NAL if asked + if (terminateNal) + { + reassembleNal(); + } + } + else + { + // Otherwise, skip the data + location.offset++; + location.length--; + } + } + } + + public void addInputData(AvRtpPacket packet) + { + short seq = packet.getSequenceNumber(); + + // Toss out the current NAL if we receive a packet that is + // out of sequence + if (lastSequenceNumber != 0 && + lastSequenceNumber + 1 != seq) + { + System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); + nalDataChain = null; + nalDataLength = 0; + } + + lastSequenceNumber = seq; + + // Pass the payload to the non-sequencing parser + AvBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); + addInputData(new AvPacket(rtpPayload)); + } + + public AvDecodeUnit getNextDecodeUnit() throws InterruptedException + { + return decodedUnits.take(); + } +} + +class H264NAL { + + // This assume's that the buffer passed in is already a special sequence + public static boolean isStartSequence(AvBufferDescriptor specialSeq) + { + if (/*specialSeq.length != 3 && */specialSeq.length != 4) + return false; + + // The start sequence is 00 00 01 or 00 00 00 01 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); + } + + // Returns a buffer descriptor describing the start sequence + public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer) + { + // NAL start sequence is 00 00 00 01 or 00 00 01 + if (buffer.length < 3) + return null; + + // 00 00 is magic + if (buffer.data[buffer.offset] == 0x00 && + buffer.data[buffer.offset+1] == 0x00) + { + // Another 00 could be the end of the special sequence + // 00 00 00 or the middle of 00 00 00 01 + if (buffer.data[buffer.offset+2] == 0x00) + { + if (buffer.length >= 4 && + buffer.data[buffer.offset+3] == 0x01) + { + // It's the AVC start sequence 00 00 00 01 + return new AvBufferDescriptor(buffer.data, buffer.offset, 4); + } + else + { + // It's 00 00 00 + return new AvBufferDescriptor(buffer.data, buffer.offset, 3); + } + } + else if (buffer.data[buffer.offset+2] == 0x01 || + buffer.data[buffer.offset+2] == 0x02) + { + // These are easy: 00 00 01 or 00 00 02 + return new AvBufferDescriptor(buffer.data, buffer.offset, 3); + } + else if (buffer.data[buffer.offset+2] == 0x03) + { + // 00 00 03 is special because it's a subsequence of the + // NAL wrapping substitute for 00 00 00, 00 00 01, 00 00 02, + // or 00 00 03 in the RBSP sequence. We need to check the next + // byte to see whether it's 00, 01, 02, or 03 (a valid RBSP substitution) + // or whether it's something else + + if (buffer.length < 4) + return null; + + if (buffer.data[buffer.offset+3] >= 0x00 && + buffer.data[buffer.offset+3] <= 0x03) + { + // It's not really a special sequence after all + return null; + } + else + { + // It's not a standard replacement so it's a special sequence + return new AvBufferDescriptor(buffer.data, buffer.offset, 3); + } + } + } + + return null; + } +} diff --git a/src/com/limelight/nvstream/av/AvParser.java b/src/com/limelight/nvstream/av/AvParser.java deleted file mode 100644 index 765c1100..00000000 --- a/src/com/limelight/nvstream/av/AvParser.java +++ /dev/null @@ -1,131 +0,0 @@ -package com.limelight.nvstream.av; - -import java.util.LinkedList; -import java.util.concurrent.LinkedBlockingQueue; - -public class AvParser { - - // Current NAL state - private LinkedList nalDataChain; - private int nalDataLength; - - private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); - - private void reassembleNal() - { - // This is the start of a new NAL - if (nalDataChain != null && nalDataLength != 0) - { - // Construct the H264 decode unit - AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength); - decodedUnits.add(du); - - // Clear old state - nalDataChain = null; - nalDataLength = 0; - } - } - - public void addInputData(AvPacket packet) - { - // This payload buffer descriptor belongs to us - AvBufferDescriptor location = packet.getNewPayloadDescriptor(); - int payloadLength = location.length; - - while (location.length != 0) - { - // Remember the start of the NAL data in this packet - int start = location.offset; - - // Check for the start sequence - if (H264NAL.hasStartSequence(location)) - { - // Reassemble any pending NAL - reassembleNal(); - - // Setup state for the new NAL - nalDataChain = new LinkedList(); - nalDataLength = 0; - - // Skip the start sequence - location.length -= 4; - location.offset += 4; - } - - // If there's a NAL assembly in progress, add the current data - if (nalDataChain != null) - { - // FIXME: This is a hack to make parsing full packets - // take less time. We assume if they don't start with - // a NAL start sequence, they're full of NAL data - if (payloadLength == 968) - { - location.offset += location.length; - location.length = 0; - } - else - { - System.out.println("Using slow parsing case"); - while (location.length != 0) - { - // Check if this should end the current NAL - if (H264NAL.hasStartSequence(location)) - { - break; - } - else - { - // This byte is part of the NAL data - location.offset++; - location.length--; - } - } - } - - // Add a buffer descriptor describing the NAL data in this packet - nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start)); - nalDataLength += location.offset-start; - } - else - { - // Otherwise, skip the data - location.offset++; - location.length--; - } - } - } - - public AvDecodeUnit getNextDecodeUnit() throws InterruptedException - { - return decodedUnits.take(); - } -} - -class H264NAL { - public static boolean shouldTerminateNal(AvBufferDescriptor buffer) - { - if (buffer.length < 4) - return false; - - if (buffer.data[buffer.offset] != 0x00 || - buffer.data[buffer.offset+1] != 0x00 || - buffer.data[buffer.offset+2] != 0x00) - { - return false; - } - - return true; - } - - public static boolean hasStartSequence(AvBufferDescriptor buffer) - { - // NAL start sequence is 00 00 00 01 - if (!shouldTerminateNal(buffer)) - return false; - - if (buffer.data[buffer.offset+3] != 0x01) - return false; - - return true; - } -} diff --git a/src/com/limelight/nvstream/av/AvRtpPacket.java b/src/com/limelight/nvstream/av/AvRtpPacket.java new file mode 100644 index 00000000..e9f8bfd0 --- /dev/null +++ b/src/com/limelight/nvstream/av/AvRtpPacket.java @@ -0,0 +1,32 @@ +package com.limelight.nvstream.av; + +import java.nio.ByteBuffer; + +public class AvRtpPacket { + + private short seqNum; + private AvBufferDescriptor buffer; + + public AvRtpPacket(AvBufferDescriptor buffer) + { + this.buffer = new AvBufferDescriptor(buffer); + + ByteBuffer bb = ByteBuffer.wrap(buffer.data, buffer.offset, buffer.length); + + // Discard the first couple of bytes + bb.getShort(); + + // Get the sequence number + seqNum = bb.getShort(); + } + + public short getSequenceNumber() + { + return seqNum; + } + + public AvBufferDescriptor getNewPayloadDescriptor() + { + return new AvBufferDescriptor(buffer.data, buffer.offset+12, buffer.length-12); + } +} From e0676690445e17c31caf1a79d04793cdf3eed4df Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 30 Oct 2013 22:23:11 -0400 Subject: [PATCH 6/9] Stage WIP audio support --- src/com/limelight/nvstream/NvVideoStream.java | 76 ++++++- .../limelight/nvstream/av/AvDecodeUnit.java | 1 + .../limelight/nvstream/av/AvDepacketizer.java | 205 +++++++++++------- 3 files changed, 199 insertions(+), 83 deletions(-) diff --git a/src/com/limelight/nvstream/NvVideoStream.java b/src/com/limelight/nvstream/NvVideoStream.java index c4871f37..1c1695b8 100644 --- a/src/com/limelight/nvstream/NvVideoStream.java +++ b/src/com/limelight/nvstream/NvVideoStream.java @@ -30,8 +30,8 @@ public class NvVideoStream { public static final int RTCP_PORT = 47999; public static final int FIRST_FRAME_PORT = 47996; - private ByteBuffer[] videoDecoderInputBuffers = null; - private MediaCodec videoDecoder; + private ByteBuffer[] videoDecoderInputBuffers, audioDecoderInputBuffers; + private MediaCodec videoDecoder, audioDecoder; private LinkedBlockingQueue packets = new LinkedBlockingQueue(); @@ -94,10 +94,19 @@ public class NvVideoStream { videoDecoder = MediaCodec.createDecoderByType("video/avc"); MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720); + audioDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm"); + MediaFormat audioFormat = MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2); + videoDecoder.configure(videoFormat, surface, null, 0); + audioDecoder.configure(audioFormat, null, null, 0); + videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); + videoDecoder.start(); + audioDecoder.start(); + videoDecoderInputBuffers = videoDecoder.getInputBuffers(); + audioDecoderInputBuffers = audioDecoder.getInputBuffers(); } public void startVideoStream(final String host, final Surface surface) @@ -130,6 +139,9 @@ public class NvVideoStream { // Start decoding the data we're receiving startDecoderThread(); + // Start playing back audio data + startAudioPlaybackThread(); + // Read the first frame to start the UDP video stream try { readFirstFrame(host); @@ -188,6 +200,32 @@ public class NvVideoStream { } } break; + + case AvDecodeUnit.TYPE_AAC: + { + int inputIndex = audioDecoder.dequeueInputBuffer(0); + if (inputIndex == -4) + { + ByteBuffer buf = audioDecoderInputBuffers[inputIndex]; + + // Clear old input data + buf.clear(); + + // Copy data from our buffer list into the input buffer + for (AvBufferDescriptor desc : du.getBufferList()) + { + buf.put(desc.data, desc.offset, desc.length); + + // Release the buffer back to the buffer pool + pool.free(desc.data); + } + + audioDecoder.queueInputBuffer(inputIndex, + 0, du.getDataLength(), + 0, du.getFlags()); + } + } + break; default: { @@ -284,6 +322,40 @@ public class NvVideoStream { }).start(); } + private void startAudioPlaybackThread() + { + new Thread(new Runnable() { + @Override + public void run() { + for (;;) + { + BufferInfo info = new BufferInfo(); + System.out.println("Waiting for audio"); + int outIndex = audioDecoder.dequeueOutputBuffer(info, -1); + System.out.println("Got audio"); + switch (outIndex) { + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + System.out.println("Output buffers changed"); + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + System.out.println("Output format changed"); + System.out.println("New output Format: " + videoDecoder.getOutputFormat()); + break; + case MediaCodec.INFO_TRY_AGAIN_LATER: + System.out.println("Try again later"); + break; + default: + break; + } + if (outIndex >= 0) { + audioDecoder.releaseOutputBuffer(outIndex, true); + } + + } + } + }).start(); + } + private void outputDisplayLoop() { for (;;) diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java index ef3d8552..4574c696 100644 --- a/src/com/limelight/nvstream/av/AvDecodeUnit.java +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -5,6 +5,7 @@ import java.util.List; public class AvDecodeUnit { public static final int TYPE_UNKNOWN = 0; public static final int TYPE_H264 = 1; + public static final int TYPE_AAC = 2; private int type; private List bufferList; diff --git a/src/com/limelight/nvstream/av/AvDepacketizer.java b/src/com/limelight/nvstream/av/AvDepacketizer.java index 06e2390c..4874cb47 100644 --- a/src/com/limelight/nvstream/av/AvDepacketizer.java +++ b/src/com/limelight/nvstream/av/AvDepacketizer.java @@ -8,24 +8,49 @@ import android.media.MediaCodec; public class AvDepacketizer { // Current NAL state - private LinkedList nalDataChain = null; - private int nalDataLength = 0; + private LinkedList avcNalDataChain = null; + private int avcNalDataLength = 0; + private LinkedList aacNalDataChain = null; + private int aacNalDataLength = 0; + private int currentlyDecoding; // Sequencing state private short lastSequenceNumber; private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); - private void reassembleNal() + private void reassembleAacNal() + { + // This is the start of a new AAC NAL + if (aacNalDataChain != null && aacNalDataLength != 0) + { + System.out.println("Assembling AAC NAL: "+aacNalDataLength); + + /*AvBufferDescriptor header = aacNalDataChain.getFirst(); + for (int i = 0; i < header.length; i++) + System.out.printf("%02x ", header.data[header.offset+i]); + System.out.println();*/ + + // Construct the AAC decode unit + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_AAC, aacNalDataChain, aacNalDataLength, 0); + decodedUnits.add(du); + + // Clear old state + aacNalDataChain = null; + aacNalDataLength = 0; + } + } + + private void reassembleAvcNal() { // This is the start of a new NAL - if (nalDataChain != null && nalDataLength != 0) + if (avcNalDataChain != null && avcNalDataLength != 0) { int flags = 0; // Check if this is a special NAL unit - AvBufferDescriptor header = nalDataChain.getFirst(); - AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header); + AvBufferDescriptor header = avcNalDataChain.getFirst(); + AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(header); if (specialSeq != null) { @@ -69,105 +94,98 @@ public class AvDepacketizer { } // Construct the H264 decode unit - AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags); + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags); decodedUnits.add(du); // Clear old state - nalDataChain = null; - nalDataLength = 0; + avcNalDataChain = null; + avcNalDataLength = 0; } } public void addInputData(AvPacket packet) { AvBufferDescriptor location = packet.getNewPayloadDescriptor(); - int payloadLength = location.length; - boolean terminateNal = false; while (location.length != 0) { // Remember the start of the NAL data in this packet int start = location.offset; - // Check for the start sequence - AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location); - if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) + // Check for a special sequence + AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(location); + if (specialSeq != null) { - // Reassemble any pending NAL - reassembleNal(); - - // Setup state for the new NAL - nalDataChain = new LinkedList(); - nalDataLength = 0; + if (NAL.isAvcStartSequence(specialSeq)) + { + // We're decoding H264 now + currentlyDecoding = AvDecodeUnit.TYPE_H264; + + // Check if it's the end of the last frame + if (NAL.isAvcFrameStart(specialSeq)) + { + // Reassemble any pending AVC NAL + reassembleAvcNal(); + + // Setup state for the new NAL + avcNalDataChain = new LinkedList(); + avcNalDataLength = 0; + } + } + else if (NAL.isAacStartSequence(specialSeq)) + { + // We're decoding AAC now + currentlyDecoding = AvDecodeUnit.TYPE_AAC; + + // Reassemble any pending AAC NAL + reassembleAacNal(); + + // Setup state for the new NAL + aacNalDataChain = new LinkedList(); + aacNalDataLength = 0; + } + else + { + // Not either sequence we want + //currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN; + } // Skip the start sequence location.length -= specialSeq.length; location.offset += specialSeq.length; } - // If there's a NAL assembly in progress, add the current data - if (nalDataChain != null) + // Move to the next special sequence + while (location.length != 0) { - // FIXME: This is a hack to make parsing full packets - // take less time. We assume if they don't start with - // a NAL start sequence, they're full of NAL data - if (payloadLength == 968) + specialSeq = NAL.getSpecialSequenceDescriptor(location); + + // Check if this should end the current NAL + if (specialSeq != null) { - location.offset += location.length; - location.length = 0; + break; } else { - //System.out.println("Using slow parsing case"); - while (location.length != 0) - { - specialSeq = H264NAL.getSpecialSequenceDescriptor(location); - - // Check if this should end the current NAL - //if (specialSeq != null) - if (specialSeq != null && H264NAL.isStartSequence(specialSeq)) - { - //terminateNal = true; - break; - } - else - { - // This byte is part of the NAL data - location.offset++; - location.length--; - } - } - } - - int endSub; - - // If parsing was finished due to reaching new start sequence, - // remove the last byte from the NAL (since it's the first byte of the - // start of the next one) - if (location.length != 0) - { - endSub = 1; - } - else - { - endSub = 0; - } - - // Add a buffer descriptor describing the NAL data in this packet - nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub)); - nalDataLength += location.offset-start-endSub; - - // Terminate the NAL if asked - if (terminateNal) - { - reassembleNal(); + // This byte is part of the NAL data + location.offset++; + location.length--; } } - else + + AvBufferDescriptor data = new AvBufferDescriptor(location.data, start, location.offset-start); + + if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null) { - // Otherwise, skip the data - location.offset++; - location.length--; + // Add a buffer descriptor describing the NAL data in this packet + avcNalDataChain.add(data); + avcNalDataLength += location.offset-start; + } + else if (currentlyDecoding == AvDecodeUnit.TYPE_AAC && aacNalDataChain != null) + { + // Add a buffer descriptor describing the NAL data in this packet + aacNalDataChain.add(data); + aacNalDataLength += location.offset-start; } } } @@ -182,8 +200,13 @@ public class AvDepacketizer { lastSequenceNumber + 1 != seq) { System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); - nalDataChain = null; - nalDataLength = 0; + + // Reset the depacketizer state + currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN; + avcNalDataChain = null; + avcNalDataLength = 0; + aacNalDataChain = null; + aacNalDataLength = 0; } lastSequenceNumber = seq; @@ -199,18 +222,38 @@ public class AvDepacketizer { } } -class H264NAL { +class NAL { - // This assume's that the buffer passed in is already a special sequence - public static boolean isStartSequence(AvBufferDescriptor specialSeq) + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcStartSequence(AvBufferDescriptor specialSeq) { - if (/*specialSeq.length != 3 && */specialSeq.length != 4) + if (specialSeq.length != 3 && specialSeq.length != 4) return false; // The start sequence is 00 00 01 or 00 00 00 01 return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); } + // This assumes that the buffer passed in is already a special sequence + public static boolean isAacStartSequence(AvBufferDescriptor specialSeq) + { + if (specialSeq.length != 3) + return false; + + // The start sequence is 00 00 03 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x03); + } + + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcFrameStart(AvBufferDescriptor specialSeq) + { + if (specialSeq.length != 4) + return false; + + // The frame start sequence is 00 00 00 01 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); + } + // Returns a buffer descriptor describing the start sequence public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer) { From 1454ade3d238ec3942a735a2b97553b2e2679ab1 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Thu, 31 Oct 2013 00:06:54 -0400 Subject: [PATCH 7/9] Fix OOS bug with packet 32768 --- src/com/limelight/nvstream/av/AvDepacketizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/limelight/nvstream/av/AvDepacketizer.java b/src/com/limelight/nvstream/av/AvDepacketizer.java index 4874cb47..afbb0694 100644 --- a/src/com/limelight/nvstream/av/AvDepacketizer.java +++ b/src/com/limelight/nvstream/av/AvDepacketizer.java @@ -197,7 +197,7 @@ public class AvDepacketizer { // Toss out the current NAL if we receive a packet that is // out of sequence if (lastSequenceNumber != 0 && - lastSequenceNumber + 1 != seq) + (short)(lastSequenceNumber + 1) != seq) { System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); From e50f668aafee1bf13117ce347bf21855e0486080 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 4 Nov 2013 01:43:35 -0500 Subject: [PATCH 8/9] Add a pair button to the UI. Fix the hardcoded MAC address string. Pass the device model in the pairing request. --- gen/com/limelight/R.java | 4 +- res/layout/activity_connection.xml | 24 ++++---- src/com/limelight/Connection.java | 65 +++++++++++++++++++- src/com/limelight/nvstream/NvConnection.java | 33 +++++++++- src/com/limelight/nvstream/NvHTTP.java | 5 +- 5 files changed, 113 insertions(+), 18 deletions(-) diff --git a/gen/com/limelight/R.java b/gen/com/limelight/R.java index 0ed4d137..d88967d1 100644 --- a/gen/com/limelight/R.java +++ b/gen/com/limelight/R.java @@ -35,8 +35,8 @@ or to a theme attribute in the form "?[package:][type:]na public static final int ic_launcher=0x7f020000; } public static final class id { - public static final int editText1=0x7f080002; - public static final int hostTextView=0x7f080000; + public static final int hostTextView=0x7f080002; + public static final int pairButton=0x7f080000; public static final int statusButton=0x7f080001; public static final int surfaceView=0x7f080003; } diff --git a/res/layout/activity_connection.xml b/res/layout/activity_connection.xml index b2b265b8..cb35dfeb 100644 --- a/res/layout/activity_connection.xml +++ b/res/layout/activity_connection.xml @@ -7,29 +7,31 @@ android:paddingRight="@dimen/activity_horizontal_margin" android:paddingTop="@dimen/activity_vertical_margin" tools:context=".Connection" > + +