From 7174caf5f1edafc34341860e233d5079f9dc3c5e Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 26 Apr 2021 20:36:07 -0500 Subject: [PATCH] Implement support for multi-FEC frames --- src/RtpFecQueue.c | 117 ++++++++++++++++++++++++++++++++++------ src/RtpFecQueue.h | 4 ++ src/SdpGenerator.c | 12 ----- src/VideoDepacketizer.c | 15 +++--- 4 files changed, 113 insertions(+), 35 deletions(-) diff --git a/src/RtpFecQueue.c b/src/RtpFecQueue.c index d6a2255..3c90529 100644 --- a/src/RtpFecQueue.c +++ b/src/RtpFecQueue.c @@ -14,6 +14,8 @@ void RtpfInitializeQueue(PRTP_FEC_QUEUE queue) { memset(queue, 0, sizeof(*queue)); queue->currentFrameNumber = UINT16_MAX; + + queue->multiFecCapable = APP_VERSION_AT_LEAST(7, 1, 431); } static void purgeListEntries(PRTPFEC_QUEUE_LIST list) { @@ -254,6 +256,9 @@ cleanup_packets: PNV_VIDEO_PACKET nvPacket = (PNV_VIDEO_PACKET)(((char*)rtpPacket) + dataOffset); nvPacket->frameIndex = queue->currentFrameNumber; + nvPacket->multiFecBlocks = + ((queue->multiFecLastBlockNumber << 2) | queue->multiFecCurrentBlockNumber) << 4; + // TODO: nvPacket->multiFecFlags? #ifdef FEC_VALIDATION_MODE if (i == dropIndex && droppedRtpPacket != NULL) { @@ -272,6 +277,7 @@ cleanup_packets: LC_ASSERT(nvPacket->frameIndex == droppedNvPacket->frameIndex); LC_ASSERT(nvPacket->streamPacketIndex == droppedNvPacket->streamPacketIndex); LC_ASSERT(nvPacket->reserved == droppedNvPacket->reserved); + LC_ASSERT(!queue->multiFecCapable || nvPacket->multiFecBlocks == droppedNvPacket->multiFecBlocks); // Check the data itself - use memcmp() and only loop if an error is detected if (memcmp(nvPacket + 1, droppedNvPacket + 1, droppedDataLength)) { @@ -441,6 +447,14 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_ nvPacket->streamPacketIndex = LE32(nvPacket->streamPacketIndex); nvPacket->frameIndex = LE32(nvPacket->frameIndex); nvPacket->fecInfo = LE32(nvPacket->fecInfo); + + // For legacy servers, we'll fixup the reserved data so that it looks like + // it's a single FEC frame from a multi-FEC capable server. This allows us + // to make our parsing logic simpler. + if (!queue->multiFecCapable) { + nvPacket->multiFecFlags = 0x10; + nvPacket->multiFecBlocks = 0x00; + } if (isBefore16(nvPacket->frameIndex, queue->currentFrameNumber)) { // Reject frames behind our current frame number @@ -448,18 +462,68 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_ } uint32_t fecIndex = (nvPacket->fecInfo & 0x3FF000) >> 12; - + uint8_t fecCurrentBlockNumber = (nvPacket->multiFecBlocks >> 4) & 0x3; + + if (nvPacket->frameIndex == queue->currentFrameNumber && fecCurrentBlockNumber < queue->multiFecCurrentBlockNumber) { + // Reject FEC blocks behind our current block number + return RTPF_RET_REJECTED; + } + // Reinitialize the queue if it's empty after a frame delivery or // if we can't finish a frame before receiving the next one. - if (queue->pendingFecBlockList.count == 0 || queue->currentFrameNumber != nvPacket->frameIndex) { - if (queue->currentFrameNumber != nvPacket->frameIndex && queue->pendingFecBlockList.count != 0) { - Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n", - queue->currentFrameNumber, queue->receivedBufferDataPackets, - queue->pendingFecBlockList.count - queue->receivedBufferDataPackets, - queue->pendingFecBlockList.count, - queue->bufferDataPackets); + if (queue->pendingFecBlockList.count == 0 || queue->currentFrameNumber != nvPacket->frameIndex || + queue->multiFecCurrentBlockNumber != fecCurrentBlockNumber) { + if (queue->pendingFecBlockList.count != 0) { + if (queue->multiFecLastBlockNumber != 0) { + Limelog("Unrecoverable frame %d (block %d of %d): %d+%d=%d received < %d needed\n", + queue->currentFrameNumber, queue->multiFecCurrentBlockNumber+1, + queue->multiFecLastBlockNumber+1, + queue->receivedBufferDataPackets, + queue->pendingFecBlockList.count - queue->receivedBufferDataPackets, + queue->pendingFecBlockList.count, + queue->bufferDataPackets); + + // If we just missed a block of this frame rather than the whole thing, + // we must manually advance the queue to the next frame. Parsing this + // frame further is not possible. + if (queue->currentFrameNumber == nvPacket->frameIndex) { + // Discard any unsubmitted buffers from the previous frame + purgeListEntries(&queue->pendingFecBlockList); + purgeListEntries(&queue->completedFecBlockList); + + queue->currentFrameNumber++; + queue->multiFecCurrentBlockNumber = 0; + return RTPF_RET_REJECTED; + } + } + else { + Limelog("Unrecoverable frame %d: %d+%d=%d received < %d needed\n", + queue->currentFrameNumber, queue->receivedBufferDataPackets, + queue->pendingFecBlockList.count - queue->receivedBufferDataPackets, + queue->pendingFecBlockList.count, + queue->bufferDataPackets); + } } + // We must either start on the current FEC block number for the current frame, + // or block 0 of a new frame. + uint8_t expectedFecBlockNumber = (queue->currentFrameNumber == nvPacket->frameIndex ? queue->multiFecCurrentBlockNumber : 0); + if (fecCurrentBlockNumber != expectedFecBlockNumber) { + Limelog("Unrecoverable frame %d: lost FEC blocks %d to %d\n", + nvPacket->frameIndex, + expectedFecBlockNumber + 1, + fecCurrentBlockNumber); + + // Discard any unsubmitted buffers from the previous frame + purgeListEntries(&queue->pendingFecBlockList); + purgeListEntries(&queue->completedFecBlockList); + + // We dropped a block of this frame, so we must skip to the next one. + queue->currentFrameNumber = nvPacket->frameIndex + 1; + queue->multiFecCurrentBlockNumber = 0; + return RTPF_RET_REJECTED; + } + // Discard any pending buffers from the previous FEC block purgeListEntries(&queue->pendingFecBlockList); @@ -483,6 +547,8 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_ queue->bufferParityPackets = (queue->bufferDataPackets * queue->fecPercentage + 99) / 100; queue->bufferFirstParitySequenceNumber = U16(queue->bufferLowestSequenceNumber + queue->bufferDataPackets); queue->bufferHighestSequenceNumber = U16(queue->bufferFirstParitySequenceNumber + queue->bufferParityPackets - 1); + queue->multiFecCurrentBlockNumber = fecCurrentBlockNumber; + queue->multiFecLastBlockNumber = (nvPacket->multiFecBlocks >> 6) & 0x3; } else if (isBefore16(queue->bufferHighestSequenceNumber, packet->sequenceNumber)) { // In rare cases, we get extra parity packets. It's rare enough that it's probably // not worth handling, so we'll just drop them. @@ -493,6 +559,14 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_ LC_ASSERT((nvPacket->fecInfo & 0xFF0) >> 4 == queue->fecPercentage); LC_ASSERT((nvPacket->fecInfo & 0xFFC00000) >> 22 == queue->bufferDataPackets); + // Verify that the legacy non-multi-FEC compatibility code works + LC_ASSERT(queue->multiFecCapable || fecCurrentBlockNumber == 0); + LC_ASSERT(queue->multiFecCapable || queue->multiFecLastBlockNumber == 0); + + // Multi-block FEC details must remain the same within a single frame + LC_ASSERT(fecCurrentBlockNumber == queue->multiFecCurrentBlockNumber); + LC_ASSERT(((nvPacket->multiFecBlocks >> 6) & 0x3) == queue->multiFecLastBlockNumber); + LC_ASSERT((nvPacket->flags & FLAG_EOF) || length - dataOffset == StreamConfig.packetSize); if (!queuePacket(queue, packetEntry, packet, length, !isBefore16(packet->sequenceNumber, queue->bufferFirstParitySequenceNumber))) { return RTPF_RET_REJECTED; @@ -512,17 +586,26 @@ int RtpfAddPacket(PRTP_FEC_QUEUE queue, PRTP_PACKET packet, int length, PRTPFEC_ LC_ASSERT(queue->pendingFecBlockList.head == NULL); LC_ASSERT(queue->pendingFecBlockList.tail == NULL); LC_ASSERT(queue->pendingFecBlockList.count == 0); + + // If we're not yet at the last FEC block for this frame, move on to the next block. + // Otherwise, the frame is complete and we can move on to the next frame. + if (queue->multiFecCurrentBlockNumber < queue->multiFecLastBlockNumber) { + // Move on to the next FEC block for this frame + queue->multiFecCurrentBlockNumber++; + } + else { + // Submit all FEC blocks to the depacketizer + submitCompletedFrame(queue); - // Submit the completed frame - submitCompletedFrame(queue); + // submitCompletedFrame() should have consumed all completed FEC data + LC_ASSERT(queue->completedFecBlockList.head == NULL); + LC_ASSERT(queue->completedFecBlockList.tail == NULL); + LC_ASSERT(queue->completedFecBlockList.count == 0); - // submitCompletedFrame() should have consumed all completed FEC data - LC_ASSERT(queue->completedFecBlockList.head == NULL); - LC_ASSERT(queue->completedFecBlockList.tail == NULL); - LC_ASSERT(queue->completedFecBlockList.count == 0); - - // Continue to the next frame - queue->currentFrameNumber++; + // Continue to the next frame + queue->currentFrameNumber++; + queue->multiFecCurrentBlockNumber = 0; + } } return RTPF_RET_QUEUED; diff --git a/src/RtpFecQueue.h b/src/RtpFecQueue.h index 84eb006..0856392 100644 --- a/src/RtpFecQueue.h +++ b/src/RtpFecQueue.h @@ -34,6 +34,10 @@ typedef struct _RTP_FEC_QUEUE { uint32_t nextContiguousSequenceNumber; uint32_t currentFrameNumber; + + bool multiFecCapable; + uint8_t multiFecCurrentBlockNumber; + uint8_t multiFecLastBlockNumber; } RTP_FEC_QUEUE, *PRTP_FEC_QUEUE; #define RTPF_RET_QUEUED 0 diff --git a/src/SdpGenerator.c b/src/SdpGenerator.c index 7d28af8..c32ab92 100644 --- a/src/SdpGenerator.c +++ b/src/SdpGenerator.c @@ -168,18 +168,6 @@ static int addGen5Options(PSDP_OPTION* head) { // Ask for the encrypted control protocol to ensure remote input will be encrypted. // This used to be done via separate RI encryption, but now it is all or nothing. err |= addAttributeString(head, "x-nv-general.useReliableUdp", "13"); - - if (StreamConfig.bitrate >= 30000 || StreamConfig.width * StreamConfig.height >= 3840 * 2160) { - // HACK: GFE 3.22 will split frames into 2 FEC blocks (sharing a frame number) - // if the number of packets exceeds ~120. We can't correctly handle those, so - // we'll turn off FEC at bitrates above 30 Mbps as an interim hack. - err |= addAttributeString(head, "x-nv-vqos[0].fec.repairPercent", "0"); - err |= addAttributeString(head, "x-nv-vqos[0].fec.numSrcPackets", "511"); - } - else { - err |= addAttributeString(head, "x-nv-vqos[0].fec.repairPercent", "20"); - err |= addAttributeString(head, "x-nv-vqos[0].fec.numSrcPackets", "125"); - } } else { // We want to use the new ENet connections for control and input diff --git a/src/VideoDepacketizer.c b/src/VideoDepacketizer.c index de54d5b..0a723d8 100644 --- a/src/VideoDepacketizer.c +++ b/src/VideoDepacketizer.c @@ -555,13 +555,12 @@ void requestDecoderRefresh(void) { } // Return 1 if packet is the first one in the frame -static int isFirstPacket(uint8_t flags) { +static int isFirstPacket(uint8_t flags, uint8_t fecBlockNumber) { // Clear the picture data flag flags &= ~FLAG_CONTAINS_PIC_DATA; // Check if it's just the start or both start and end of a frame - return (flags == (FLAG_SOF | FLAG_EOF) || - flags == FLAG_SOF); + return (flags == (FLAG_SOF | FLAG_EOF) || flags == FLAG_SOF) && fecBlockNumber == 0; } // Process an RTP Payload @@ -574,6 +573,8 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, uint8_t flags; uint32_t firstPacket; uint32_t streamPacketIndex; + uint8_t fecCurrentBlockNumber; + uint8_t fecLastBlockNumber; // Mask the top 8 bits from the SPI videoPacket->streamPacketIndex >>= 8; @@ -583,9 +584,11 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, currentPos.offset = 0; currentPos.length = length - sizeof(*videoPacket); + fecCurrentBlockNumber = (videoPacket->multiFecBlocks >> 4) & 0x3; + fecLastBlockNumber = (videoPacket->multiFecBlocks >> 6) & 0x3; frameIndex = videoPacket->frameIndex; flags = videoPacket->flags; - firstPacket = isFirstPacket(flags); + firstPacket = isFirstPacket(flags, fecCurrentBlockNumber); LC_ASSERT((flags & ~(FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA)) == 0); @@ -600,7 +603,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, // It almost always detects them before they get to us, but in case it doesn't // the streamPacketIndex not matching correctly should find nearly all of the rest. if (isBefore24(streamPacketIndex, U24(lastPacketInStream + 1)) || - (!firstPacket && streamPacketIndex != U24(lastPacketInStream + 1))) { + (!(flags & FLAG_SOF) && streamPacketIndex != U24(lastPacketInStream + 1))) { Limelog("Depacketizer detected corrupt frame: %d", frameIndex); decodingFrame = false; nextFrameNumber = frameIndex + 1; @@ -698,7 +701,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length, queueFragment(existingEntry, currentPos.data, currentPos.offset, currentPos.length); } - if (flags & FLAG_EOF) { + if ((flags & FLAG_EOF) && fecCurrentBlockNumber == fecLastBlockNumber) { // Move on to the next frame decodingFrame = false; nextFrameNumber = frameIndex + 1;