moonlight-stream/moonlight-android
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-android.git synced 2025-07-18 18:42:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stage WIP audio support

Browse Source
This commit is contained in:
Cameron Gutman 2013-10-30 22:23:11 -04:00
parent 9e09ca2b7a
commit e067669044
3 changed files with 199 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/com/limelight/nvstream/NvVideoStream.java
View File

@ -30,8 +30,8 @@ public class NvVideoStream {
public static final int RTCP_PORT = 47999;
public static final int FIRST_FRAME_PORT = 47996;
private ByteBuffer[] videoDecoderInputBuffers = null;
private MediaCodec videoDecoder;
private ByteBuffer[] videoDecoderInputBuffers, audioDecoderInputBuffers;
private MediaCodec videoDecoder, audioDecoder;
private LinkedBlockingQueue<AvRtpPacket> packets = new LinkedBlockingQueue<AvRtpPacket>();
@ -94,10 +94,19 @@ public class NvVideoStream {
videoDecoder = MediaCodec.createDecoderByType("video/avc");
MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720);
audioDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm");
MediaFormat audioFormat = MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2);
videoDecoder.configure(videoFormat, surface, null, 0);
audioDecoder.configure(audioFormat, null, null, 0);
videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT);
videoDecoder.start();
audioDecoder.start();
videoDecoderInputBuffers = videoDecoder.getInputBuffers();
audioDecoderInputBuffers = audioDecoder.getInputBuffers();
}
public void startVideoStream(final String host, final Surface surface)
@ -130,6 +139,9 @@ public class NvVideoStream {
// Start decoding the data we're receiving
startDecoderThread();
// Start playing back audio data
startAudioPlaybackThread();
// Read the first frame to start the UDP video stream
try {
readFirstFrame(host);
@ -189,6 +201,32 @@ public class NvVideoStream {
}
break;
case AvDecodeUnit.TYPE_AAC:
{
int inputIndex = audioDecoder.dequeueInputBuffer(0);
if (inputIndex == -4)
{
ByteBuffer buf = audioDecoderInputBuffers[inputIndex];
// Clear old input data
buf.clear();
// Copy data from our buffer list into the input buffer
for (AvBufferDescriptor desc : du.getBufferList())
{
buf.put(desc.data, desc.offset, desc.length);
// Release the buffer back to the buffer pool
pool.free(desc.data);
}
audioDecoder.queueInputBuffer(inputIndex,
0, du.getDataLength(),
0, du.getFlags());
}
}
break;
default:
{
System.out.println("Unknown decode unit type");
@ -284,6 +322,40 @@ public class NvVideoStream {
}).start();
}
private void startAudioPlaybackThread()
{
new Thread(new Runnable() {
@Override
public void run() {
for (;;)
{
BufferInfo info = new BufferInfo();
System.out.println("Waiting for audio");
int outIndex = audioDecoder.dequeueOutputBuffer(info, -1);
System.out.println("Got audio");
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
System.out.println("Output buffers changed");
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
System.out.println("Output format changed");
System.out.println("New output Format: " + videoDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
System.out.println("Try again later");
break;
default:
break;
}
if (outIndex >= 0) {
audioDecoder.releaseOutputBuffer(outIndex, true);
}
}
}
}).start();
}
private void outputDisplayLoop()
{
for (;;)

1
src/com/limelight/nvstream/av/AvDecodeUnit.java
View File

@ -5,6 +5,7 @@ import java.util.List;
public class AvDecodeUnit {
public static final int TYPE_UNKNOWN = 0;
public static final int TYPE_H264 = 1;
public static final int TYPE_AAC = 2;
private int type;
private List<AvBufferDescriptor> bufferList;

177
src/com/limelight/nvstream/av/AvDepacketizer.java
View File

@ -8,24 +8,49 @@ import android.media.MediaCodec;
public class AvDepacketizer {
// Current NAL state
private LinkedList<AvBufferDescriptor> nalDataChain = null;
private int nalDataLength = 0;
private LinkedList<AvBufferDescriptor> avcNalDataChain = null;
private int avcNalDataLength = 0;
private LinkedList<AvBufferDescriptor> aacNalDataChain = null;
private int aacNalDataLength = 0;
private int currentlyDecoding;
// Sequencing state
private short lastSequenceNumber;
private LinkedBlockingQueue<AvDecodeUnit> decodedUnits = new LinkedBlockingQueue<AvDecodeUnit>();
private void reassembleNal()
private void reassembleAacNal()
{
// This is the start of a new AAC NAL
if (aacNalDataChain != null && aacNalDataLength != 0)
{
System.out.println("Assembling AAC NAL: "+aacNalDataLength);
/*AvBufferDescriptor header = aacNalDataChain.getFirst();
for (int i = 0; i < header.length; i++)
System.out.printf("%02x ", header.data[header.offset+i]);
System.out.println();*/
// Construct the AAC decode unit
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_AAC, aacNalDataChain, aacNalDataLength, 0);
decodedUnits.add(du);
// Clear old state
aacNalDataChain = null;
aacNalDataLength = 0;
}
}
private void reassembleAvcNal()
{
// This is the start of a new NAL
if (nalDataChain != null && nalDataLength != 0)
if (avcNalDataChain != null && avcNalDataLength != 0)
{
int flags = 0;
// Check if this is a special NAL unit
AvBufferDescriptor header = nalDataChain.getFirst();
AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header);
AvBufferDescriptor header = avcNalDataChain.getFirst();
AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(header);
if (specialSeq != null)
{
@ -69,65 +94,75 @@ public class AvDepacketizer {
}
// Construct the H264 decode unit
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags);
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags);
decodedUnits.add(du);
// Clear old state
nalDataChain = null;
nalDataLength = 0;
avcNalDataChain = null;
avcNalDataLength = 0;
}
}
public void addInputData(AvPacket packet)
{
AvBufferDescriptor location = packet.getNewPayloadDescriptor();
int payloadLength = location.length;
boolean terminateNal = false;
while (location.length != 0)
{
// Remember the start of the NAL data in this packet
int start = location.offset;
// Check for the start sequence
AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
// Check for a special sequence
AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(location);
if (specialSeq != null)
{
// Reassemble any pending NAL
reassembleNal();
if (NAL.isAvcStartSequence(specialSeq))
{
// We're decoding H264 now
currentlyDecoding = AvDecodeUnit.TYPE_H264;
// Check if it's the end of the last frame
if (NAL.isAvcFrameStart(specialSeq))
{
// Reassemble any pending AVC NAL
reassembleAvcNal();
// Setup state for the new NAL
nalDataChain = new LinkedList<AvBufferDescriptor>();
nalDataLength = 0;
avcNalDataChain = new LinkedList<AvBufferDescriptor>();
avcNalDataLength = 0;
}
}
else if (NAL.isAacStartSequence(specialSeq))
{
// We're decoding AAC now
currentlyDecoding = AvDecodeUnit.TYPE_AAC;
// Reassemble any pending AAC NAL
reassembleAacNal();
// Setup state for the new NAL
aacNalDataChain = new LinkedList<AvBufferDescriptor>();
aacNalDataLength = 0;
}
else
{
// Not either sequence we want
//currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
}
// Skip the start sequence
location.length -= specialSeq.length;
location.offset += specialSeq.length;
}
// If there's a NAL assembly in progress, add the current data
if (nalDataChain != null)
{
// FIXME: This is a hack to make parsing full packets
// take less time. We assume if they don't start with
// a NAL start sequence, they're full of NAL data
if (payloadLength == 968)
{
location.offset += location.length;
location.length = 0;
}
else
{
//System.out.println("Using slow parsing case");
// Move to the next special sequence
while (location.length != 0)
{
specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
specialSeq = NAL.getSpecialSequenceDescriptor(location);
// Check if this should end the current NAL
//if (specialSeq != null)
if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
if (specialSeq != null)
{
//terminateNal = true;
break;
}
else
@ -137,37 +172,20 @@ public class AvDepacketizer {
location.length--;
}
}
}
int endSub;
AvBufferDescriptor data = new AvBufferDescriptor(location.data, start, location.offset-start);
// If parsing was finished due to reaching new start sequence,
// remove the last byte from the NAL (since it's the first byte of the
// start of the next one)
if (location.length != 0)
if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null)
{
endSub = 1;
}
else
{
endSub = 0;
}
// Add a buffer descriptor describing the NAL data in this packet
nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub));
nalDataLength += location.offset-start-endSub;
// Terminate the NAL if asked
if (terminateNal)
{
reassembleNal();
avcNalDataChain.add(data);
avcNalDataLength += location.offset-start;
}
}
else
else if (currentlyDecoding == AvDecodeUnit.TYPE_AAC && aacNalDataChain != null)
{
// Otherwise, skip the data
location.offset++;
location.length--;
// Add a buffer descriptor describing the NAL data in this packet
aacNalDataChain.add(data);
aacNalDataLength += location.offset-start;
}
}
}
@ -182,8 +200,13 @@ public class AvDepacketizer {
lastSequenceNumber + 1 != seq)
{
System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
nalDataChain = null;
nalDataLength = 0;
// Reset the depacketizer state
currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
avcNalDataChain = null;
avcNalDataLength = 0;
aacNalDataChain = null;
aacNalDataLength = 0;
}
lastSequenceNumber = seq;
@ -199,18 +222,38 @@ public class AvDepacketizer {
}
}
class H264NAL {
class NAL {
// This assume's that the buffer passed in is already a special sequence
public static boolean isStartSequence(AvBufferDescriptor specialSeq)
// This assumes that the buffer passed in is already a special sequence
public static boolean isAvcStartSequence(AvBufferDescriptor specialSeq)
{
if (/*specialSeq.length != 3 && */specialSeq.length != 4)
if (specialSeq.length != 3 && specialSeq.length != 4)
return false;
// The start sequence is 00 00 01 or 00 00 00 01
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
}
// This assumes that the buffer passed in is already a special sequence
public static boolean isAacStartSequence(AvBufferDescriptor specialSeq)
{
if (specialSeq.length != 3)
return false;
// The start sequence is 00 00 03
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x03);
}
// This assumes that the buffer passed in is already a special sequence
public static boolean isAvcFrameStart(AvBufferDescriptor specialSeq)
{
if (specialSeq.length != 4)
return false;
// The frame start sequence is 00 00 00 01
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
}
// Returns a buffer descriptor describing the start sequence
public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer)
{