Implement support for multi-FEC frames

src/RtpFecQueue.c

@@ -14,6 +14,8 @@ void RtpfInitializeQueue(PRTP_FEC_QUEUE queue) {
     memset(queue, 0, sizeof(*queue));
     
     queue->currentFrameNumber = UINT16_MAX;
+
+    queue->multiFecCapable = APP_VERSION_AT_LEAST(7, 1, 431);
 }
 
 static void purgeListEntries(PRTPFEC_QUEUE_LIST list) {
@@ -254,6 +256,9 @@ cleanup_packets:
 
                 PNV_VIDEO_PACKET nvPacket = (PNV_VIDEO_PACKET)(((char*)rtpPacket) + dataOffset);
                 nvPacket->frameIndex = queue->currentFrameNumber;
+                nvPacket->multiFecBlocks =
+                        ((queue->multiFecLastBlockNumber << 2) | queue->multiFecCurrentBlockNumber) << 4;
+                // TODO: nvPacket->multiFecFlags?
 
 #ifdef FEC_VALIDATION_MODE
                 if (i == dropIndex && droppedRtpPacket != NULL) {
@@ -272,6 +277,7 @@ cleanup_packets:
                     LC_ASSERT(nvPacket->frameIndex == droppedNvPacket->frameIndex);
                     LC_ASSERT(nvPacket->streamPacketIndex == droppedNvPacket->streamPacketIndex);
                     LC_ASSERT(nvPacket->reserved == droppedNvPacket->reserved);
+                    LC_ASSERT(!queue->multiFecCapable || nvPacket->multiFecBlocks == droppedNvPacket->multiFecBlocks);
 
                     // Check the data itself - use memcmp() and only loop if an error is detected
                     if (memcmp(nvPacket + 1, droppedNvPacket + 1, droppedDataLength)) {
@@ -441,6 +447,14 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_
     nvPacket->streamPacketIndex = LE32(nvPacket->streamPacketIndex);
     nvPacket->frameIndex = LE32(nvPacket->frameIndex);
     nvPacket->fecInfo = LE32(nvPacket->fecInfo);
+
+    // For legacy servers, we'll fixup the reserved data so that it looks like
+    // it's a single FEC frame from a multi-FEC capable server. This allows us
+    // to make our parsing logic simpler.
+    if (!queue->multiFecCapable) {
+        nvPacket->multiFecFlags = 0x10;
+        nvPacket->multiFecBlocks = 0x00;
+    }
     
     if (isBefore16(nvPacket->frameIndex, queue->currentFrameNumber)) {
         // Reject frames behind our current frame number
@@ -448,18 +462,68 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_
     }
 
     uint32_t fecIndex = (nvPacket->fecInfo & 0x3FF000) >> 12;
-    
+    uint8_t fecCurrentBlockNumber = (nvPacket->multiFecBlocks >> 4) & 0x3;
+
+    if (nvPacket->frameIndex == queue->currentFrameNumber && fecCurrentBlockNumber < queue->multiFecCurrentBlockNumber) {
+        // Reject FEC blocks behind our current block number
+        return RTPF_RET_REJECTED;
+    }
+
     // Reinitialize the queue if it's empty after a frame delivery or
     // if we can't finish a frame before receiving the next one.
-    if (queue->pendingFecBlockList.count == 0 || queue->currentFrameNumber != nvPacket->frameIndex) {
+    if (queue->pendingFecBlockList.count == 0 || queue->currentFrameNumber != nvPacket->frameIndex ||
-        if (queue->currentFrameNumber != nvPacket->frameIndex && queue->pendingFecBlockList.count != 0) {
+            queue->multiFecCurrentBlockNumber != fecCurrentBlockNumber) {
-            Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n",
+        if (queue->pendingFecBlockList.count != 0) {
-                    queue->currentFrameNumber, queue->receivedBufferDataPackets,
+            if (queue->multiFecLastBlockNumber != 0) {
-                    queue->pendingFecBlockList.count - queue->receivedBufferDataPackets,
+                Limelog("Unrecoverable frame %d (block %d of %d): %d+%d=%d received < %d needed\n",
-                    queue->pendingFecBlockList.count,
+                        queue->currentFrameNumber, queue->multiFecCurrentBlockNumber+1,
-                    queue->bufferDataPackets);
+                        queue->multiFecLastBlockNumber+1,
+                        queue->receivedBufferDataPackets,
+                        queue->pendingFecBlockList.count - queue->receivedBufferDataPackets,
+                        queue->pendingFecBlockList.count,
+                        queue->bufferDataPackets);
+
+                // If we just missed a block of this frame rather than the whole thing,
+                // we must manually advance the queue to the next frame. Parsing this
+                // frame further is not possible.
+                if (queue->currentFrameNumber == nvPacket->frameIndex) {
+                    // Discard any unsubmitted buffers from the previous frame
+                    purgeListEntries(&queue->pendingFecBlockList);
+                    purgeListEntries(&queue->completedFecBlockList);
+
+                    queue->currentFrameNumber++;
+                    queue->multiFecCurrentBlockNumber = 0;
+                    return RTPF_RET_REJECTED;
+                }
+            }
+            else {
+                Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n",
+                        queue->currentFrameNumber, queue->receivedBufferDataPackets,
+                        queue->pendingFecBlockList.count - queue->receivedBufferDataPackets,
+                        queue->pendingFecBlockList.count,
+                        queue->bufferDataPackets);
+            }
         }
         
+        // We must either start on the current FEC block number for the current frame,
+        // or block 0 of a new frame.
+        uint8_t expectedFecBlockNumber = (queue->currentFrameNumber == nvPacket->frameIndex ? queue->multiFecCurrentBlockNumber : 0);
+        if (fecCurrentBlockNumber != expectedFecBlockNumber) {
+            Limelog("Unrecoverable frame %d: lost FEC blocks %d to %d\n",
+                    nvPacket->frameIndex,
+                    expectedFecBlockNumber + 1,
+                    fecCurrentBlockNumber);
+
+            // Discard any unsubmitted buffers from the previous frame
+            purgeListEntries(&queue->pendingFecBlockList);
+            purgeListEntries(&queue->completedFecBlockList);
+
+            // We dropped a block of this frame, so we must skip to the next one.
+            queue->currentFrameNumber = nvPacket->frameIndex + 1;
+            queue->multiFecCurrentBlockNumber = 0;
+            return RTPF_RET_REJECTED;
+        }
+
         // Discard any pending buffers from the previous FEC block
         purgeListEntries(&queue->pendingFecBlockList);
 
@@ -483,6 +547,8 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_
         queue->bufferParityPackets = (queue->bufferDataPackets * queue->fecPercentage + 99) / 100;
         queue->bufferFirstParitySequenceNumber = U16(queue->bufferLowestSequenceNumber + queue->bufferDataPackets);
         queue->bufferHighestSequenceNumber = U16(queue->bufferFirstParitySequenceNumber + queue->bufferParityPackets - 1);
+        queue->multiFecCurrentBlockNumber = fecCurrentBlockNumber;
+        queue->multiFecLastBlockNumber = (nvPacket->multiFecBlocks >> 6) & 0x3;
     } else if (isBefore16(queue->bufferHighestSequenceNumber, packet->sequenceNumber)) {
         // In rare cases, we get extra parity packets. It's rare enough that it's probably
         // not worth handling, so we'll just drop them.
@@ -493,6 +559,14 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_
     LC_ASSERT((nvPacket->fecInfo & 0xFF0) >> 4 == queue->fecPercentage);
     LC_ASSERT((nvPacket->fecInfo & 0xFFC00000) >> 22 == queue->bufferDataPackets);
 
+    // Verify that the legacy non-multi-FEC compatibility code works
+    LC_ASSERT(queue->multiFecCapable || fecCurrentBlockNumber == 0);
+    LC_ASSERT(queue->multiFecCapable || queue->multiFecLastBlockNumber == 0);
+
+    // Multi-block FEC details must remain the same within a single frame
+    LC_ASSERT(fecCurrentBlockNumber == queue->multiFecCurrentBlockNumber);
+    LC_ASSERT(((nvPacket->multiFecBlocks >> 6) & 0x3) == queue->multiFecLastBlockNumber);
+
     LC_ASSERT((nvPacket->flags & FLAG_EOF) || length - dataOffset == StreamConfig.packetSize);
     if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber))) {
         return RTPF_RET_REJECTED;
@@ -512,17 +586,26 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_
             LC_ASSERT(queue->pendingFecBlockList.head == NULL);
             LC_ASSERT(queue->pendingFecBlockList.tail == NULL);
             LC_ASSERT(queue->pendingFecBlockList.count == 0);
+            
+            // If we're not yet at the last FEC block for this frame, move on to the next block.
+            // Otherwise, the frame is complete and we can move on to the next frame.
+            if (queue->multiFecCurrentBlockNumber < queue->multiFecLastBlockNumber) {
+                // Move on to the next FEC block for this frame
+                queue->multiFecCurrentBlockNumber++;
+            }
+            else {
+                // Submit all FEC blocks to the depacketizer
+                submitCompletedFrame(queue);
 
-            // Submit the completed frame
+                // submitCompletedFrame() should have consumed all completed FEC data
-            submitCompletedFrame(queue);
+                LC_ASSERT(queue->completedFecBlockList.head == NULL);
+                LC_ASSERT(queue->completedFecBlockList.tail == NULL);
+                LC_ASSERT(queue->completedFecBlockList.count == 0);
 
-            // submitCompletedFrame() should have consumed all completed FEC data
+                // Continue to the next frame
-            LC_ASSERT(queue->completedFecBlockList.head == NULL);
+                queue->currentFrameNumber++;
-            LC_ASSERT(queue->completedFecBlockList.tail == NULL);
+                queue->multiFecCurrentBlockNumber = 0;
-            LC_ASSERT(queue->completedFecBlockList.count == 0);
+            }
-
-            // Continue to the next frame
-            queue->currentFrameNumber++;
         }
 
         return RTPF_RET_QUEUED;

src/RtpFecQueue.h

@@ -34,6 +34,10 @@ typedef struct _RTP_FEC_QUEUE {
     uint32_t nextContiguousSequenceNumber;
 
     uint32_t currentFrameNumber;
+
+    bool multiFecCapable;
+    uint8_t multiFecCurrentBlockNumber;
+    uint8_t multiFecLastBlockNumber;
 } RTP_FEC_QUEUE, *PRTP_FEC_QUEUE;
 
 #define RTPF_RET_QUEUED    0

src/SdpGenerator.c

@@ -168,18 +168,6 @@ static int addGen5Options(PSDP_OPTION* head) {
         // Ask for the encrypted control protocol to ensure remote input will be encrypted.
         // This used to be done via separate RI encryption, but now it is all or nothing.
         err |= addAttributeString(head, "x-nv-general.useReliableUdp", "13");
-
-        if (StreamConfig.bitrate >= 30000 || StreamConfig.width * StreamConfig.height >= 3840 * 2160) {
-            // HACK: GFE 3.22 will split frames into 2 FEC blocks (sharing a frame number)
-            // if the number of packets exceeds ~120. We can't correctly handle those, so
-            // we'll turn off FEC at bitrates above 30 Mbps as an interim hack.
-            err |= addAttributeString(head, "x-nv-vqos[0].fec.repairPercent", "0");
-            err |= addAttributeString(head, "x-nv-vqos[0].fec.numSrcPackets", "511");
-        }
-        else {
-            err |= addAttributeString(head, "x-nv-vqos[0].fec.repairPercent", "20");
-            err |= addAttributeString(head, "x-nv-vqos[0].fec.numSrcPackets", "125");
-        }
     }
     else {
         // We want to use the new ENet connections for control and input

src/VideoDepacketizer.c

@@ -555,13 +555,12 @@ void requestDecoderRefresh(void) {
 }
 
 // Return 1 if packet is the first one in the frame
-static int isFirstPacket(uint8_t flags) {
+static int isFirstPacket(uint8_t flags, uint8_t fecBlockNumber) {
     // Clear the picture data flag
     flags &= ~FLAG_CONTAINS_PIC_DATA;
 
     // Check if it's just the start or both start and end of a frame
-    return (flags == (FLAG_SOF | FLAG_EOF) ||
+    return (flags == (FLAG_SOF | FLAG_EOF) || flags == FLAG_SOF) && fecBlockNumber == 0;
-        flags == FLAG_SOF);
 }
 
 // Process an RTP Payload
@@ -574,6 +573,8 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
     uint8_t flags;
     uint32_t firstPacket;
     uint32_t streamPacketIndex;
+    uint8_t fecCurrentBlockNumber;
+    uint8_t fecLastBlockNumber;
 
     // Mask the top 8 bits from the SPI
     videoPacket->streamPacketIndex >>= 8;
@@ -583,9 +584,11 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
     currentPos.offset = 0;
     currentPos.length = length - sizeof(*videoPacket);
 
+    fecCurrentBlockNumber = (videoPacket->multiFecBlocks >> 4) & 0x3;
+    fecLastBlockNumber = (videoPacket->multiFecBlocks >> 6) & 0x3;
     frameIndex = videoPacket->frameIndex;
     flags = videoPacket->flags;
-    firstPacket = isFirstPacket(flags);
+    firstPacket = isFirstPacket(flags, fecCurrentBlockNumber);
 
     LC_ASSERT((flags & ~(FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA)) == 0);
 
@@ -600,7 +603,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
     // It almost always detects them before they get to us, but in case it doesn't
     // the streamPacketIndex not matching correctly should find nearly all of the rest.
     if (isBefore24(streamPacketIndex, U24(lastPacketInStream + 1)) ||
-            (!firstPacket && streamPacketIndex != U24(lastPacketInStream + 1))) {
+            (!(flags & FLAG_SOF) && streamPacketIndex != U24(lastPacketInStream + 1))) {
         Limelog("Depacketizer detected corrupt frame: %d", frameIndex);
         decodingFrame = false;
         nextFrameNumber = frameIndex + 1;
@@ -698,7 +701,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
         queueFragment(existingEntry, currentPos.data, currentPos.offset, currentPos.length);
     }
 
-    if (flags & FLAG_EOF) {
+    if ((flags & FLAG_EOF) && fecCurrentBlockNumber == fecLastBlockNumber) {
         // Move on to the next frame
         decodingFrame = false;
         nextFrameNumber = frameIndex + 1;