From c19ff71c9a22bbac37710d16e68bcd9f3511e9be Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 11 Aug 2015 21:12:34 -0700 Subject: [PATCH] Add experimental reference frame invalidation support --- .../limelight/nvstream/ConnectionContext.java | 3 + .../com/limelight/nvstream/NvConnection.java | 5 +- .../av/video/VideoDecoderRenderer.java | 4 + .../nvstream/av/video/VideoDepacketizer.java | 26 +++++- .../nvstream/av/video/VideoStream.java | 9 +-- .../nvstream/control/ControlStream.java | 80 ++++++++++++++----- 6 files changed, 95 insertions(+), 32 deletions(-) diff --git a/moonlight-common/src/com/limelight/nvstream/ConnectionContext.java b/moonlight-common/src/com/limelight/nvstream/ConnectionContext.java index 02611efb..9ae6ad41 100644 --- a/moonlight-common/src/com/limelight/nvstream/ConnectionContext.java +++ b/moonlight-common/src/com/limelight/nvstream/ConnectionContext.java @@ -4,6 +4,8 @@ import java.net.InetAddress; import javax.crypto.SecretKey; +import com.limelight.nvstream.av.video.VideoDecoderRenderer; + public class ConnectionContext { // Gen 3 servers are 2.1.1 - 2.2.1 public static final int SERVER_GENERATION_3 = 3; @@ -13,6 +15,7 @@ public class ConnectionContext { public InetAddress serverAddress; public StreamConfiguration streamConfig; + public VideoDecoderRenderer videoDecoderRenderer; public NvConnectionListener connListener; public SecretKey riKey; public int riKeyId; diff --git a/moonlight-common/src/com/limelight/nvstream/NvConnection.java b/moonlight-common/src/com/limelight/nvstream/NvConnection.java index cee6c8c1..1241a4f9 100644 --- a/moonlight-common/src/com/limelight/nvstream/NvConnection.java +++ b/moonlight-common/src/com/limelight/nvstream/NvConnection.java @@ -41,7 +41,6 @@ public class NvConnection { // Start parameters private int drFlags; private Object videoRenderTarget; - private VideoDecoderRenderer videoDecoderRenderer; private AudioRenderer audioRenderer; public NvConnection(String host, String uniqueId, NvConnectionListener listener, StreamConfiguration config, LimelightCryptoProvider cryptoProvider) @@ -246,7 +245,7 @@ public class NvConnection { private boolean startVideoStream() throws IOException { videoStream = new VideoStream(context, controlStream); - return videoStream.startVideoStream(videoDecoderRenderer, videoRenderTarget, drFlags); + return videoStream.startVideoStream(videoRenderTarget, drFlags); } private boolean startAudioStream() throws IOException @@ -329,7 +328,7 @@ public class NvConnection { this.drFlags = drFlags; this.audioRenderer = audioRenderer; this.videoRenderTarget = videoRenderTarget; - this.videoDecoderRenderer = videoDecoderRenderer; + this.context.videoDecoderRenderer = videoDecoderRenderer; new Thread(new Runnable() { public void run() { diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java index ab2fc236..54c33087 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java @@ -14,6 +14,10 @@ public abstract class VideoDecoderRenderer { // Allows decode units to be submitted directly from the receive thread public static final int CAPABILITY_DIRECT_SUBMIT = 0x2; + // !!! EXPERIMENTAL !!! + // Allows reference frame invalidation to be use to recover from packet loss + public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION = 0x4; + public int getCapabilities() { return 0; } diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java index 876e781f..6ef6cac0 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java @@ -1,6 +1,7 @@ package com.limelight.nvstream.av.video; import com.limelight.LimeLog; +import com.limelight.nvstream.ConnectionContext; import com.limelight.nvstream.av.ByteBufferDescriptor; import com.limelight.nvstream.av.DecodeUnit; import com.limelight.nvstream.av.ConnectionStatusListener; @@ -26,6 +27,7 @@ public class VideoDepacketizer { private boolean waitingForIdrFrame = true; private long frameStartTime; private boolean decodingFrame; + private boolean strictIdrFrameWait; // Cached objects private ByteBufferDescriptor cachedReassemblyDesc = new ByteBufferDescriptor(null, 0, 0); @@ -40,11 +42,23 @@ public class VideoDepacketizer { private static final int DU_LIMIT = 15; private AbstractPopulatedBufferList decodedUnits; - public VideoDepacketizer(ConnectionStatusListener controlListener, int nominalPacketSize, boolean unsynchronized) + public VideoDepacketizer(ConnectionContext context, ConnectionStatusListener controlListener, int nominalPacketSize) { this.controlListener = controlListener; this.nominalPacketDataLength = nominalPacketSize - VideoPacket.HEADER_SIZE; + boolean unsynchronized; + if (context.videoDecoderRenderer != null) { + int videoCaps = context.videoDecoderRenderer.getCapabilities(); + this.strictIdrFrameWait = (videoCaps & VideoDecoderRenderer.CAPABILITY_REFERENCE_FRAME_INVALIDATION) == 0; + unsynchronized = (videoCaps & VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT) != 0; + } + else { + // If there's no renderer, it doesn't matter if we synchronize or wait for IDRs + this.strictIdrFrameWait = false; + unsynchronized = true; + } + AbstractPopulatedBufferList.BufferFactory factory = new AbstractPopulatedBufferList.BufferFactory() { public Object createFreeBuffer() { return new DecodeUnit(); @@ -71,7 +85,10 @@ public class VideoDepacketizer { private void dropAvcFrameState() { - waitingForIdrFrame = true; + // We'll need an IDR frame now if we're in strict mode + if (strictIdrFrameWait) { + waitingForIdrFrame = true; + } // Count the number of consecutive frames dropped consecutiveFrameDrops++; @@ -84,7 +101,7 @@ public class VideoDepacketizer { // Restart the count consecutiveFrameDrops = 0; - // Request an IDR frame + // Request an IDR frame (0 tuple always generates an IDR frame) controlListener.connectionDetectedFrameLoss(0, 0); } @@ -128,7 +145,9 @@ public class VideoDepacketizer { LimeLog.warning("Video decoder is too slow! Forced to drop decode units"); // Invalidate all frames from the start of the DU queue + // (0 tuple always generates an IDR frame) controlListener.connectionSinkTooSlow(0, 0); + waitingForIdrFrame = true; // Remove existing frames decodedUnits.clearPopulatedObjects(); @@ -330,7 +349,6 @@ public class VideoDepacketizer { // Unexpected start of next frame before terminating the last waitingForNextSuccessfulFrame = true; - waitingForIdrFrame = true; // Clear the old state and wait for an IDR dropAvcFrameState(); diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java index ac18344a..2aaeccdd 100644 --- a/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java @@ -118,8 +118,7 @@ public class VideoStream { public boolean setupDecoderRenderer(VideoDecoderRenderer decRend, Object renderTarget, int drFlags) { this.decRend = decRend; - depacketizer = new VideoDepacketizer(avConnListener, context.streamConfig.getMaxPacketSize(), - decRend != null && (decRend.getCapabilities() & VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT) != 0); + depacketizer = new VideoDepacketizer(context, avConnListener, context.streamConfig.getMaxPacketSize()); if (decRend != null) { try { @@ -143,10 +142,10 @@ public class VideoStream { return true; } - public boolean startVideoStream(VideoDecoderRenderer decRend, Object renderTarget, int drFlags) throws IOException + public boolean startVideoStream(Object renderTarget, int drFlags) throws IOException { // Setup the decoder and renderer - if (!setupDecoderRenderer(decRend, renderTarget, drFlags)) { + if (!setupDecoderRenderer(context.videoDecoderRenderer, renderTarget, drFlags)) { // Nothing to cleanup here throw new IOException("Video decoder failed to initialize. Please restart your device and try again."); } @@ -154,7 +153,7 @@ public class VideoStream { // Open RTP sockets and start session setupRtpSession(); - if (decRend != null) { + if (this.decRend != null) { // Start the receive thread early to avoid missing // early packets that are part of the IDR frame startReceiveThread(); diff --git a/moonlight-common/src/com/limelight/nvstream/control/ControlStream.java b/moonlight-common/src/com/limelight/nvstream/control/ControlStream.java index 7a0615ec..59c01825 100644 --- a/moonlight-common/src/com/limelight/nvstream/control/ControlStream.java +++ b/moonlight-common/src/com/limelight/nvstream/control/ControlStream.java @@ -12,6 +12,7 @@ import java.util.concurrent.LinkedBlockingQueue; import com.limelight.LimeLog; import com.limelight.nvstream.ConnectionContext; import com.limelight.nvstream.av.ConnectionStatusListener; +import com.limelight.nvstream.av.video.VideoDecoderRenderer; public class ControlStream implements ConnectionStatusListener { @@ -101,6 +102,7 @@ public class ControlStream implements ConnectionStatusListener { private Thread resyncThread; private LinkedBlockingQueue invalidReferenceFrameTuples = new LinkedBlockingQueue(); private boolean aborting = false; + private boolean forceIdrRequest; private final short[] packetTypes; private final short[] payloadLengths; @@ -124,6 +126,11 @@ public class ControlStream implements ConnectionStatusListener { preconstructedPayloads = precontructedPayloadsGen4; break; } + + if (context.videoDecoderRenderer != null) { + forceIdrRequest = (context.videoDecoderRenderer.getCapabilities() & + VideoDecoderRenderer.CAPABILITY_REFERENCE_FRAME_INVALIDATION) == 0; + } } public void initialize() throws IOException @@ -239,6 +246,7 @@ public class ControlStream implements ConnectionStatusListener { while (!isInterrupted()) { int[] tuple; + boolean idrFrameRequired = false; // Wait for a tuple try { @@ -248,29 +256,46 @@ public class ControlStream implements ConnectionStatusListener { return; } - // Aggregate all lost frames into one range + // Check for the magic IDR frame tuple int[] lastTuple = null; - for (;;) { - int[] nextTuple = lastTuple = invalidReferenceFrameTuples.poll(); - if (nextTuple == null) { - break; + if (tuple[0] != 0 || tuple[1] != 0) { + // Aggregate all lost frames into one range + for (;;) { + int[] nextTuple = lastTuple = invalidReferenceFrameTuples.poll(); + if (nextTuple == null) { + break; + } + + // Check if this tuple has IDR frame magic values + if (nextTuple[0] == 0 && nextTuple[1] == 0) { + // We will need an IDR frame now, but we won't break out + // of the loop because we want to dequeue all pending requests + idrFrameRequired = true; + } + + lastTuple = nextTuple; } - - lastTuple = nextTuple; } - - // The server expects this to be the firstLostFrame + 1 - tuple[0]++; - - // Update the end of the range to the latest tuple - if (lastTuple != null) { - tuple[1] = lastTuple[1]; + else { + // We must require an IDR frame + idrFrameRequired = true; } try { - LimeLog.warning("Invalidating reference frames from "+tuple[0]+" to "+tuple[1]); - ControlStream.this.invalidateReferenceFrames(tuple[0], tuple[1]); - LimeLog.warning("Frames invalidated"); + if (forceIdrRequest || idrFrameRequired) { + requestIdrFrame(); + } + else { + // The server expects this to be the firstLostFrame + 1 + tuple[0]++; + + // Update the end of the range to the latest tuple + if (lastTuple != null) { + tuple[1] = lastTuple[1]; + } + + invalidateReferenceFrames(tuple[0], tuple[1]); + } } catch (IOException e) { context.connListener.connectionTerminated(e); return; @@ -310,9 +335,7 @@ public class ControlStream implements ConnectionStatusListener { } } - private void invalidateReferenceFrames(int firstLostFrame, int nextSuccessfulFrame) throws IOException - { - // We can't handle a real reference frame invalidation yet. + private void requestIdrFrame() throws IOException { // On Gen 3, we use the invalidate reference frames trick which works for about 5 hours of streaming at 60 FPS // On Gen 4+, we use the known IDR frame request packet @@ -333,6 +356,23 @@ public class ControlStream implements ConnectionStatusListener { (short) preconstructedPayloads[IDX_REQUEST_IDR_FRAME].length, preconstructedPayloads[IDX_REQUEST_IDR_FRAME])); } + + LimeLog.warning("IDR frame request sent"); + } + + private void invalidateReferenceFrames(int firstLostFrame, int nextSuccessfulFrame) throws IOException { + LimeLog.warning("Invalidating reference frames from "+firstLostFrame+" to "+nextSuccessfulFrame); + + ByteBuffer conf = ByteBuffer.wrap(new byte[payloadLengths[IDX_INVALIDATE_REF_FRAMES]]).order(ByteOrder.LITTLE_ENDIAN); + + conf.putLong(firstLostFrame); + conf.putLong(nextSuccessfulFrame); + conf.putLong(0); + + sendAndGetReply(new NvCtlPacket(packetTypes[IDX_INVALIDATE_REF_FRAMES], + payloadLengths[IDX_INVALIDATE_REF_FRAMES], conf.array())); + + LimeLog.warning("Reference frame invalidation sent"); } static class NvCtlPacket {