mirror of
https://github.com/moonlight-stream/moonlight-android.git
synced 2025-07-18 18:42:46 +00:00
Stage WIP audio support
This commit is contained in:
Cameron Gutman
parent
9e09ca2b7a
commit
e067669044
@ -30,8 +30,8 @@ public class NvVideoStream {
|
||||
public static final int RTCP_PORT = 47999;
|
||||
public static final int FIRST_FRAME_PORT = 47996;
|
||||
|
||||
private ByteBuffer[] videoDecoderInputBuffers = null;
|
||||
private MediaCodec videoDecoder;
|
||||
private ByteBuffer[] videoDecoderInputBuffers, audioDecoderInputBuffers;
|
||||
private MediaCodec videoDecoder, audioDecoder;
|
||||
|
||||
private LinkedBlockingQueue<AvRtpPacket> packets = new LinkedBlockingQueue<AvRtpPacket>();
|
||||
|
||||
@ -94,10 +94,19 @@ public class NvVideoStream {
|
||||
videoDecoder = MediaCodec.createDecoderByType("video/avc");
|
||||
MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720);
|
||||
|
||||
audioDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm");
|
||||
MediaFormat audioFormat = MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2);
|
||||
|
||||
videoDecoder.configure(videoFormat, surface, null, 0);
|
||||
audioDecoder.configure(audioFormat, null, null, 0);
|
||||
|
||||
videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT);
|
||||
|
||||
videoDecoder.start();
|
||||
audioDecoder.start();
|
||||
|
||||
videoDecoderInputBuffers = videoDecoder.getInputBuffers();
|
||||
audioDecoderInputBuffers = audioDecoder.getInputBuffers();
|
||||
}
|
||||
|
||||
public void startVideoStream(final String host, final Surface surface)
|
||||
@ -130,6 +139,9 @@ public class NvVideoStream {
|
||||
// Start decoding the data we're receiving
|
||||
startDecoderThread();
|
||||
|
||||
// Start playing back audio data
|
||||
startAudioPlaybackThread();
|
||||
|
||||
// Read the first frame to start the UDP video stream
|
||||
try {
|
||||
readFirstFrame(host);
|
||||
@ -189,6 +201,32 @@ public class NvVideoStream {
|
||||
}
|
||||
break;
|
||||
|
||||
case AvDecodeUnit.TYPE_AAC:
|
||||
{
|
||||
int inputIndex = audioDecoder.dequeueInputBuffer(0);
|
||||
if (inputIndex == -4)
|
||||
{
|
||||
ByteBuffer buf = audioDecoderInputBuffers[inputIndex];
|
||||
|
||||
// Clear old input data
|
||||
buf.clear();
|
||||
|
||||
// Copy data from our buffer list into the input buffer
|
||||
for (AvBufferDescriptor desc : du.getBufferList())
|
||||
{
|
||||
buf.put(desc.data, desc.offset, desc.length);
|
||||
|
||||
// Release the buffer back to the buffer pool
|
||||
pool.free(desc.data);
|
||||
}
|
||||
|
||||
audioDecoder.queueInputBuffer(inputIndex,
|
||||
0, du.getDataLength(),
|
||||
0, du.getFlags());
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
{
|
||||
System.out.println("Unknown decode unit type");
|
||||
@ -284,6 +322,40 @@ public class NvVideoStream {
|
||||
}).start();
|
||||
}
|
||||
|
||||
private void startAudioPlaybackThread()
|
||||
{
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
for (;;)
|
||||
{
|
||||
BufferInfo info = new BufferInfo();
|
||||
System.out.println("Waiting for audio");
|
||||
int outIndex = audioDecoder.dequeueOutputBuffer(info, -1);
|
||||
System.out.println("Got audio");
|
||||
switch (outIndex) {
|
||||
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
|
||||
System.out.println("Output buffers changed");
|
||||
break;
|
||||
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
|
||||
System.out.println("Output format changed");
|
||||
System.out.println("New output Format: " + videoDecoder.getOutputFormat());
|
||||
break;
|
||||
case MediaCodec.INFO_TRY_AGAIN_LATER:
|
||||
System.out.println("Try again later");
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (outIndex >= 0) {
|
||||
audioDecoder.releaseOutputBuffer(outIndex, true);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
private void outputDisplayLoop()
|
||||
{
|
||||
for (;;)
|
||||
|
||||
@ -5,6 +5,7 @@ import java.util.List;
|
||||
public class AvDecodeUnit {
|
||||
public static final int TYPE_UNKNOWN = 0;
|
||||
public static final int TYPE_H264 = 1;
|
||||
public static final int TYPE_AAC = 2;
|
||||
|
||||
private int type;
|
||||
private List<AvBufferDescriptor> bufferList;
|
||||
|
||||
@ -8,24 +8,49 @@ import android.media.MediaCodec;
|
||||
public class AvDepacketizer {
|
||||
|
||||
// Current NAL state
|
||||
private LinkedList<AvBufferDescriptor> nalDataChain = null;
|
||||
private int nalDataLength = 0;
|
||||
private LinkedList<AvBufferDescriptor> avcNalDataChain = null;
|
||||
private int avcNalDataLength = 0;
|
||||
private LinkedList<AvBufferDescriptor> aacNalDataChain = null;
|
||||
private int aacNalDataLength = 0;
|
||||
private int currentlyDecoding;
|
||||
|
||||
// Sequencing state
|
||||
private short lastSequenceNumber;
|
||||
|
||||
private LinkedBlockingQueue<AvDecodeUnit> decodedUnits = new LinkedBlockingQueue<AvDecodeUnit>();
|
||||
|
||||
private void reassembleNal()
|
||||
private void reassembleAacNal()
|
||||
{
|
||||
// This is the start of a new AAC NAL
|
||||
if (aacNalDataChain != null && aacNalDataLength != 0)
|
||||
{
|
||||
System.out.println("Assembling AAC NAL: "+aacNalDataLength);
|
||||
|
||||
/*AvBufferDescriptor header = aacNalDataChain.getFirst();
|
||||
for (int i = 0; i < header.length; i++)
|
||||
System.out.printf("%02x ", header.data[header.offset+i]);
|
||||
System.out.println();*/
|
||||
|
||||
// Construct the AAC decode unit
|
||||
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_AAC, aacNalDataChain, aacNalDataLength, 0);
|
||||
decodedUnits.add(du);
|
||||
|
||||
// Clear old state
|
||||
aacNalDataChain = null;
|
||||
aacNalDataLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private void reassembleAvcNal()
|
||||
{
|
||||
// This is the start of a new NAL
|
||||
if (nalDataChain != null && nalDataLength != 0)
|
||||
if (avcNalDataChain != null && avcNalDataLength != 0)
|
||||
{
|
||||
int flags = 0;
|
||||
|
||||
// Check if this is a special NAL unit
|
||||
AvBufferDescriptor header = nalDataChain.getFirst();
|
||||
AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(header);
|
||||
AvBufferDescriptor header = avcNalDataChain.getFirst();
|
||||
AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(header);
|
||||
|
||||
if (specialSeq != null)
|
||||
{
|
||||
@ -69,105 +94,98 @@ public class AvDepacketizer {
|
||||
}
|
||||
|
||||
// Construct the H264 decode unit
|
||||
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength, flags);
|
||||
AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags);
|
||||
decodedUnits.add(du);
|
||||
|
||||
// Clear old state
|
||||
nalDataChain = null;
|
||||
nalDataLength = 0;
|
||||
avcNalDataChain = null;
|
||||
avcNalDataLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public void addInputData(AvPacket packet)
|
||||
{
|
||||
AvBufferDescriptor location = packet.getNewPayloadDescriptor();
|
||||
int payloadLength = location.length;
|
||||
boolean terminateNal = false;
|
||||
|
||||
while (location.length != 0)
|
||||
{
|
||||
// Remember the start of the NAL data in this packet
|
||||
int start = location.offset;
|
||||
|
||||
// Check for the start sequence
|
||||
AvBufferDescriptor specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
|
||||
if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
|
||||
// Check for a special sequence
|
||||
AvBufferDescriptor specialSeq = NAL.getSpecialSequenceDescriptor(location);
|
||||
if (specialSeq != null)
|
||||
{
|
||||
// Reassemble any pending NAL
|
||||
reassembleNal();
|
||||
if (NAL.isAvcStartSequence(specialSeq))
|
||||
{
|
||||
// We're decoding H264 now
|
||||
currentlyDecoding = AvDecodeUnit.TYPE_H264;
|
||||
|
||||
// Setup state for the new NAL
|
||||
nalDataChain = new LinkedList<AvBufferDescriptor>();
|
||||
nalDataLength = 0;
|
||||
// Check if it's the end of the last frame
|
||||
if (NAL.isAvcFrameStart(specialSeq))
|
||||
{
|
||||
// Reassemble any pending AVC NAL
|
||||
reassembleAvcNal();
|
||||
|
||||
// Setup state for the new NAL
|
||||
avcNalDataChain = new LinkedList<AvBufferDescriptor>();
|
||||
avcNalDataLength = 0;
|
||||
}
|
||||
}
|
||||
else if (NAL.isAacStartSequence(specialSeq))
|
||||
{
|
||||
// We're decoding AAC now
|
||||
currentlyDecoding = AvDecodeUnit.TYPE_AAC;
|
||||
|
||||
// Reassemble any pending AAC NAL
|
||||
reassembleAacNal();
|
||||
|
||||
// Setup state for the new NAL
|
||||
aacNalDataChain = new LinkedList<AvBufferDescriptor>();
|
||||
aacNalDataLength = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Not either sequence we want
|
||||
//currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
// Skip the start sequence
|
||||
location.length -= specialSeq.length;
|
||||
location.offset += specialSeq.length;
|
||||
}
|
||||
|
||||
// If there's a NAL assembly in progress, add the current data
|
||||
if (nalDataChain != null)
|
||||
// Move to the next special sequence
|
||||
while (location.length != 0)
|
||||
{
|
||||
// FIXME: This is a hack to make parsing full packets
|
||||
// take less time. We assume if they don't start with
|
||||
// a NAL start sequence, they're full of NAL data
|
||||
if (payloadLength == 968)
|
||||
specialSeq = NAL.getSpecialSequenceDescriptor(location);
|
||||
|
||||
// Check if this should end the current NAL
|
||||
if (specialSeq != null)
|
||||
{
|
||||
location.offset += location.length;
|
||||
location.length = 0;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
//System.out.println("Using slow parsing case");
|
||||
while (location.length != 0)
|
||||
{
|
||||
specialSeq = H264NAL.getSpecialSequenceDescriptor(location);
|
||||
|
||||
// Check if this should end the current NAL
|
||||
//if (specialSeq != null)
|
||||
if (specialSeq != null && H264NAL.isStartSequence(specialSeq))
|
||||
{
|
||||
//terminateNal = true;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
// This byte is part of the NAL data
|
||||
location.offset++;
|
||||
location.length--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int endSub;
|
||||
|
||||
// If parsing was finished due to reaching new start sequence,
|
||||
// remove the last byte from the NAL (since it's the first byte of the
|
||||
// start of the next one)
|
||||
if (location.length != 0)
|
||||
{
|
||||
endSub = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
endSub = 0;
|
||||
}
|
||||
|
||||
// Add a buffer descriptor describing the NAL data in this packet
|
||||
nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start-endSub));
|
||||
nalDataLength += location.offset-start-endSub;
|
||||
|
||||
// Terminate the NAL if asked
|
||||
if (terminateNal)
|
||||
{
|
||||
reassembleNal();
|
||||
// This byte is part of the NAL data
|
||||
location.offset++;
|
||||
location.length--;
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
AvBufferDescriptor data = new AvBufferDescriptor(location.data, start, location.offset-start);
|
||||
|
||||
if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null)
|
||||
{
|
||||
// Otherwise, skip the data
|
||||
location.offset++;
|
||||
location.length--;
|
||||
// Add a buffer descriptor describing the NAL data in this packet
|
||||
avcNalDataChain.add(data);
|
||||
avcNalDataLength += location.offset-start;
|
||||
}
|
||||
else if (currentlyDecoding == AvDecodeUnit.TYPE_AAC && aacNalDataChain != null)
|
||||
{
|
||||
// Add a buffer descriptor describing the NAL data in this packet
|
||||
aacNalDataChain.add(data);
|
||||
aacNalDataLength += location.offset-start;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -182,8 +200,13 @@ public class AvDepacketizer {
|
||||
lastSequenceNumber + 1 != seq)
|
||||
{
|
||||
System.out.println("Received OOS data (expected "+(lastSequenceNumber + 1)+", got "+seq+")");
|
||||
nalDataChain = null;
|
||||
nalDataLength = 0;
|
||||
|
||||
// Reset the depacketizer state
|
||||
currentlyDecoding = AvDecodeUnit.TYPE_UNKNOWN;
|
||||
avcNalDataChain = null;
|
||||
avcNalDataLength = 0;
|
||||
aacNalDataChain = null;
|
||||
aacNalDataLength = 0;
|
||||
}
|
||||
|
||||
lastSequenceNumber = seq;
|
||||
@ -199,18 +222,38 @@ public class AvDepacketizer {
|
||||
}
|
||||
}
|
||||
|
||||
class H264NAL {
|
||||
class NAL {
|
||||
|
||||
// This assume's that the buffer passed in is already a special sequence
|
||||
public static boolean isStartSequence(AvBufferDescriptor specialSeq)
|
||||
// This assumes that the buffer passed in is already a special sequence
|
||||
public static boolean isAvcStartSequence(AvBufferDescriptor specialSeq)
|
||||
{
|
||||
if (/*specialSeq.length != 3 && */specialSeq.length != 4)
|
||||
if (specialSeq.length != 3 && specialSeq.length != 4)
|
||||
return false;
|
||||
|
||||
// The start sequence is 00 00 01 or 00 00 00 01
|
||||
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
|
||||
}
|
||||
|
||||
// This assumes that the buffer passed in is already a special sequence
|
||||
public static boolean isAacStartSequence(AvBufferDescriptor specialSeq)
|
||||
{
|
||||
if (specialSeq.length != 3)
|
||||
return false;
|
||||
|
||||
// The start sequence is 00 00 03
|
||||
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x03);
|
||||
}
|
||||
|
||||
// This assumes that the buffer passed in is already a special sequence
|
||||
public static boolean isAvcFrameStart(AvBufferDescriptor specialSeq)
|
||||
{
|
||||
if (specialSeq.length != 4)
|
||||
return false;
|
||||
|
||||
// The frame start sequence is 00 00 00 01
|
||||
return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01);
|
||||
}
|
||||
|
||||
// Returns a buffer descriptor describing the start sequence
|
||||
public static AvBufferDescriptor getSpecialSequenceDescriptor(AvBufferDescriptor buffer)
|
||||
{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user