Guard reference frame invalidation behind a video renderer capability flag and do some refactoring and bugfixes to the new code

limelight-common/ControlStream.c

@@ -10,20 +10,20 @@ typedef struct _NVCTL_PACKET_HEADER {
 	unsigned short payloadLength;
 } NVCTL_PACKET_HEADER, *PNVCTL_PACKET_HEADER;
 
-typedef struct _QUEUED_LOSS_STAT {
+typedef struct _QUEUED_FRAME_INVALIDATION_TUPLE {
 	int startFrame;
 	int endFrame;
 	LINKED_BLOCKING_QUEUE_ENTRY entry;
-} QUEUED_LOSS_STAT, *PQUEUED_LOSS_STAT;
+} QUEUED_FRAME_INVALIDATION_TUPLE, *PQUEUED_FRAME_INVALIDATION_TUPLE;
 
 static SOCKET ctlSock = INVALID_SOCKET;
 static PLT_THREAD lossStatsThread;
 static PLT_THREAD invalidateRefFramesThread;
 static PLT_EVENT invalidateRefFramesEvent;
-static int lossCountSinceLastReport = 0;
+static int lossCountSinceLastReport;
-static long currentFrame = 0;
+static long currentFrame;
 
-static int requestIdr;
+static int idrFrameRequired;
 static LINKED_BLOCKING_QUEUE invalidReferenceFrameTuples;
 
 #define IDX_START_A 0
@@ -99,60 +99,73 @@ int initializeControlStream(void) {
         preconstructedPayloads = (char**)preconstructedPayloadsGen4;
     }
 
+    idrFrameRequired = 0;
+    currentFrame = 0;
+    lossCountSinceLastReport = 0;
+
 	return 0;
 }
 
-void freeLossStatList(PLINKED_BLOCKING_QUEUE_ENTRY entry) {
+void freeFrameInvalidationList(PLINKED_BLOCKING_QUEUE_ENTRY entry) {
 	PLINKED_BLOCKING_QUEUE_ENTRY nextEntry;
 
 	while (entry != NULL) {
 		nextEntry = entry->flink;
-
 		free(entry->data);
-
 		entry = nextEntry;
 	}
 }
 
 /* Cleans up control stream */
 void destroyControlStream(void) {
-	PltCloseEvent(&invalidateRefFramesEvent);
+    PltCloseEvent(&invalidateRefFramesEvent);
-	freeLossStatList(LbqDestroyLinkedBlockingQueue(&invalidReferenceFrameTuples));
+    freeFrameInvalidationList(LbqDestroyLinkedBlockingQueue(&invalidReferenceFrameTuples));
 }
 
-int getNextLossStat(PQUEUED_LOSS_STAT *qls) {
+int getNextFrameInvalidationTuple(PQUEUED_FRAME_INVALIDATION_TUPLE *qfit) {
-	int err = LbqPollQueueElement(&invalidReferenceFrameTuples, (void**) qls);
+	int err = LbqPollQueueElement(&invalidReferenceFrameTuples, (void**) qfit);
 	return (err == LBQ_SUCCESS);
 }
 
-void queueLossStat(int startFrame, int endFrame) {
+void queueFrameInvalidationTuple(int startFrame, int endFrame) {
-	PQUEUED_LOSS_STAT qls;
+
-	qls = malloc(sizeof(QUEUED_LOSS_STAT));
+    if (VideoCallbacks.capabilities & CAPABILITY_REFERENCE_FRAME_INVALIDATION) {
-	if (qls != NULL) {
+        PQUEUED_FRAME_INVALIDATION_TUPLE qfit;
-		qls->startFrame = startFrame;
+        qfit = malloc(sizeof(*qfit));
-		qls->endFrame = endFrame;
+        if (qfit != NULL) {
-		if (LbqOfferQueueItem(&invalidReferenceFrameTuples, qls, &qls->entry) == LBQ_BOUND_EXCEEDED) {
+            qfit->startFrame = startFrame;
-			free(qls);
+            qfit->endFrame = endFrame;
-			requestIdr = 1;
+            if (LbqOfferQueueItem(&invalidReferenceFrameTuples, qfit, &qfit->entry) == LBQ_BOUND_EXCEEDED) {
-                }
+                // Too many invalidation tuples, so we need an IDR frame now
-		PltSetEvent(&invalidateRefFramesEvent);
+                free(qfit);
-	}
+                idrFrameRequired = 1;
+            }
+        }
+        else {
+            idrFrameRequired = 1;
+        }
+    }
+    else {
+        idrFrameRequired = 1;
+    }
+
+    PltSetEvent(&invalidateRefFramesEvent);
 }
 
 /* Request an IDR frame on demand by the decoder */
 void requestIdrOnDemand(void) {
-    requestIdr = 1;
+    idrFrameRequired = 1;
     PltSetEvent(&invalidateRefFramesEvent);
 }
 
 /* Invalidate reference frames if the decoder is too slow */
 void connectionSinkTooSlow(int startFrame, int endFrame) {
-	queueLossStat(startFrame, endFrame);
+    queueFrameInvalidationTuple(startFrame, endFrame);
 }
 
 /* Invalidate reference frames lost by the network */
 void connectionDetectedFrameLoss(int startFrame, int endFrame) {
-	queueLossStat(startFrame, endFrame);
+    queueFrameInvalidationTuple(startFrame, endFrame);
 }
 
 /* When we receive a frame, update the number of our current frame */
@@ -308,33 +321,39 @@ static void requestIdrFrame(void) {
     Limelog("IDR frame request sent\n");
 }
 
-static void requestInvalidateRefFrames(void) {
+static void requestInvalidateReferenceFrames(void) {
-	long long payload[3];
+    long long payload[3];
-	PQUEUED_LOSS_STAT qls;
+    PQUEUED_FRAME_INVALIDATION_TUPLE qfit;
-	if (!getNextLossStat(&qls)) {
-		return;
-	}
 
-	payload[0] = qls->startFrame + 1;
+    LC_ASSERT(VideoCallbacks.capabilities & CAPABILITY_REFERENCE_FRAME_INVALIDATION);
 
-	// Aggregate all lost frames into one range
+    if (!getNextFrameInvalidationTuple(&qfit)) {
-	while (getNextLossStat(&qls));
+        return;
+    }
 
-	// The server expects this to be the firstLostFrame + 1
+    LC_ASSERT(qfit->startFrame <= qfit->endFrame);
-	payload[1] = qls->endFrame;
-	payload[2] = 0;
 
-	free(qls);
+    // The server expects this to be the firstLostFrame + 1
+    payload[0] = qfit->startFrame + 1;
+    payload[1] = qfit->endFrame;
+    payload[2] = 0;
 
-	// Send the reference frame invalidation request and read the response
+    // Aggregate all lost frames into one range
-	if (!sendMessageAndDiscardReply(packetTypes[IDX_INVALIDATE_REF_FRAMES],
+    do {
-		payloadLengths[IDX_INVALIDATE_REF_FRAMES], payload)) {
+        LC_ASSERT(qfit->endFrame >= payload[1]);
-		Limelog("Request Invaldiate Reference Frames: Transaction failed: %d\n", (int) LastSocketError());
+        payload[1] = qfit->endFrame;
-		ListenerCallbacks.connectionTerminated(LastSocketError());
+        free(qfit);
-		return;
+    } while (getNextFrameInvalidationTuple(&qfit));
-        }
 
-        Limelog("Invalidate Reference Frame request sent\n");
+    // Send the reference frame invalidation request and read the response
+    if (!sendMessageAndDiscardReply(packetTypes[IDX_INVALIDATE_REF_FRAMES],
+        payloadLengths[IDX_INVALIDATE_REF_FRAMES], payload)) {
+        Limelog("Request Invaldiate Reference Frames: Transaction failed: %d\n", (int) LastSocketError());
+        ListenerCallbacks.connectionTerminated(LastSocketError());
+        return;
+    }
+
+    Limelog("Invalidate reference frame request sent\n");
 }
 
 static void invalidateRefFramesFunc(void* context) {
@@ -343,17 +362,20 @@ static void invalidateRefFramesFunc(void* context) {
         PltWaitForEvent(&invalidateRefFramesEvent);
         PltClearEvent(&invalidateRefFramesEvent);
 
-        if (requestIdr) {
+        // Sometimes we absolutely need an IDR frame
-            // Send an IDR frame request
+        if (idrFrameRequired) {
-            requestIdrFrame();
+            // Empty invalidate reference frames tuples
-            requestIdr = 0;
+            PQUEUED_FRAME_INVALIDATION_TUPLE qfit;
+            while (getNextFrameInvalidationTuple(&qfit)) {
+                free(qfit);
+            }
 
-            //Empty invalidate reference frames tuples
+            // Send an IDR frame request
-            PQUEUED_LOSS_STAT qls;
+            idrFrameRequired = 0;
-            while (getNextLossStat(&qls));
+            requestIdrFrame();
-            free(qls);
         } else {
-            requestInvalidateRefFrames();
+            // Otherwise invalidate reference frames
+            requestInvalidateReferenceFrames();
         }
     }
 }

limelight-common/Limelight.h

@@ -52,8 +52,16 @@ typedef struct _DECODE_UNIT {
 	PLENTRY bufferList;
 } DECODE_UNIT, *PDECODE_UNIT;
 
+// If set in the renderer capabilities field, this flag will cause audio/video data to
+// be submitted directly from the receive thread. This should only be specified if the
+// renderer is non-blocking. This flag is valid on both audio and video renderers.
 #define CAPABILITY_DIRECT_SUBMIT 0x1
 
+// !!! HIGHLY EXPERIMENTAL - DO NOT SET IN PRODUCTION CODE !!!
+// If set in the video renderer capabilities field, this flag specifies that the renderer
+// supports reference frame invalidation. This flag is only valid on video renderers.
+#define CAPABILITY_REFERENCE_FRAME_INVALIDATION 0x2
+
 // This callback is invoked to provide details about the video stream and allow configuration of the decoder
 typedef void(*DecoderRendererSetup)(int width, int height, int redrawRate, void* context, int drFlags);
 

limelight-common/VideoDepacketizer.c

@@ -14,6 +14,7 @@ static int waitingForIdrFrame;
 static int gotNextFrameStart;
 static int lastPacketInStream;
 static int decodingFrame;
+static int strictIdrFrameWait;
 
 #define CONSECUTIVE_DROP_LIMIT 120
 static int consecutiveFrameDrops;
@@ -42,6 +43,7 @@ void initializeVideoDepacketizer(int pktSize) {
 	gotNextFrameStart = 0;
 	lastPacketInStream = -1;
 	decodingFrame = 0;
+    strictIdrFrameWait = !(VideoCallbacks.capabilities & CAPABILITY_REFERENCE_FRAME_INVALIDATION);
 }
 
 /* Free malloced memory in AvcFrameState*/
@@ -59,6 +61,11 @@ static void cleanupAvcFrameState(void) {
 
 /* Cleanup AVC frame state and set that we're waiting for an IDR Frame*/
 static void dropAvcFrameState(void) {
+    // We'll need an IDR frame now if we're in strict mode
+    if (strictIdrFrameWait) {
+        waitingForIdrFrame = 1;
+    }
+
     // Count the number of consecutive frames dropped
     consecutiveFrameDrops++;