More fixes and work on video

limelight-common/ByteBuffer.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "Platform.h"
 
 #define BYTE_ORDER_LITTLE 1

limelight-common/ControlStream.cpp

@@ -2,19 +2,17 @@
 #include "PlatformSockets.h"
 #include "PlatformThreads.h"
 
-typedef struct _CONTROL_STREAM {
-	SOCKET s;
-	STREAM_CONFIGURATION streamConfig;
-	PLT_THREAD heartbeatThread;
-	PLT_THREAD jitterThread;
-	PLT_THREAD resyncThread;
-} CONTROL_STREAM, *PCONTROL_STREAM;
-
 typedef struct _NVCTL_PACKET_HEADER {
 	unsigned short type;
 	unsigned short payloadLength;
 } NVCTL_PACKET_HEADER, *PNVCTL_PACKET_HEADER;
 
+SOCKET ctlSock;
+STREAM_CONFIGURATION streamConfig;
+PLT_THREAD heartbeatThread;
+PLT_THREAD jitterThread;
+PLT_THREAD resyncThread;
+
 const short PTYPE_KEEPALIVE = 0x13ff;
 const short PPAYLEN_KEEPALIVE = 0x0000;
 
@@ -30,33 +28,29 @@ const short PPAYLEN_RESYNC = 16;
 const short PTYPE_JITTER = 0x140c;
 const short PPAYLEN_JITTER = 0x10;
 
-void* initializeControlStream(IP_ADDRESS host, PSTREAM_CONFIGURATION streamConfig) {
-	PCONTROL_STREAM ctx;
-
-	ctx = (PCONTROL_STREAM) malloc(sizeof(*ctx));
-	if (ctx == NULL) {
-		return NULL;
+int initializeControlStream(IP_ADDRESS host, PSTREAM_CONFIGURATION streamConfigPtr) {
+	ctlSock = connectTcpSocket(host, 47995);
+	if (ctlSock == INVALID_SOCKET) {
+		return LastSocketError();
 	}
 
-	ctx->s = connectTcpSocket(host, 47995);
-	if (ctx->s == INVALID_SOCKET) {
-		free(ctx);
-		return NULL;
-	}
+	enableNoDelay(ctlSock);
 
-	enableNoDelay(ctx->s);
+	memcpy(&streamConfig, streamConfigPtr, sizeof(*streamConfigPtr));
 
-	memcpy(&ctx->streamConfig, streamConfig, sizeof(*streamConfig));
-
-	return ctx;
+	return 0;
 }
 
-static PNVCTL_PACKET_HEADER readNvctlPacket(PCONTROL_STREAM stream) {
+void requestIdrFrame(void) {
+
+}
+
+static PNVCTL_PACKET_HEADER readNvctlPacket(void) {
 	NVCTL_PACKET_HEADER staticHeader;
 	PNVCTL_PACKET_HEADER fullPacket;
 	int err;
 
-	err = recv(stream->s, (char*) &staticHeader, sizeof(staticHeader), 0);
+	err = recv(ctlSock, (char*) &staticHeader, sizeof(staticHeader), 0);
 	if (err != sizeof(staticHeader)) {
 		return NULL;
 	}
@@ -67,7 +61,7 @@ static PNVCTL_PACKET_HEADER readNvctlPacket(PCONTROL_STREAM stream) {
 	}
 
 	memcpy(fullPacket, &staticHeader, sizeof(staticHeader));
-	err = recv(stream->s, (char*) (fullPacket + 1), staticHeader.payloadLength, 0);
+	err = recv(ctlSock, (char*) (fullPacket + 1), staticHeader.payloadLength, 0);
 	if (err != staticHeader.payloadLength) {
 		free(fullPacket);
 		return NULL;
@@ -76,29 +70,28 @@ static PNVCTL_PACKET_HEADER readNvctlPacket(PCONTROL_STREAM stream) {
 	return fullPacket;
 }
 
-static PNVCTL_PACKET_HEADER sendNoPayloadAndReceive(PCONTROL_STREAM stream, short ptype, short paylen) {
+static PNVCTL_PACKET_HEADER sendNoPayloadAndReceive(short ptype, short paylen) {
 	NVCTL_PACKET_HEADER header;
 	int err;
 
 	header.type = ptype;
 	header.payloadLength = paylen;
-	err = send(stream->s, (char*) &header, sizeof(header), 0);
+	err = send(ctlSock, (char*) &header, sizeof(header), 0);
 	if (err != sizeof(header)) {
 		return NULL;
 	}
 
-	return readNvctlPacket(stream);
+	return readNvctlPacket();
 }
 
 static void heartbeatThreadFunc(void* context) {
-	PCONTROL_STREAM stream = (PCONTROL_STREAM) context;
 	int err;
 	NVCTL_PACKET_HEADER header;
 
 	for (;;) {
 		header.type = PTYPE_HEARTBEAT;
 		header.payloadLength = PPAYLEN_HEARTBEAT;
-		err = send(stream->s, (char*) &header, sizeof(header), 0);
+		err = send(ctlSock, (char*) &header, sizeof(header), 0);
 		if (err != sizeof(header)) {
 			Limelog("Heartbeat thread terminating\n");
 			return;
@@ -109,7 +102,6 @@ static void heartbeatThreadFunc(void* context) {
 }
 
 static void jitterThreadFunc(void* context) {
-	PCONTROL_STREAM stream = (PCONTROL_STREAM) context;
 	int payload[4];
 	NVCTL_PACKET_HEADER header;
 	int err;
@@ -117,7 +109,7 @@ static void jitterThreadFunc(void* context) {
 	header.type = PTYPE_JITTER;
 	header.payloadLength = PPAYLEN_JITTER;
 	for (;;) {
-		err = send(stream->s, (char*) &header, sizeof(header), 0);
+		err = send(ctlSock, (char*) &header, sizeof(header), 0);
 		if (err != sizeof(header)) {
 			Limelog("Jitter thread terminating #1\n");
 			return;
@@ -128,7 +120,7 @@ static void jitterThreadFunc(void* context) {
 		payload[2] = 888;
 		payload[3] = 0; // FIXME: Sequence number?
 
-		err = send(stream->s, (char*) payload, sizeof(payload), 0);
+		err = send(ctlSock, (char*) payload, sizeof(payload), 0);
 		if (err != sizeof(payload)) {
 			Limelog("Jitter thread terminating #2\n");
 			return;
@@ -139,70 +131,66 @@ static void jitterThreadFunc(void* context) {
 }
 
 static void resyncThreadFunc(void* context) {
-	PCONTROL_STREAM stream = (PCONTROL_STREAM) context;
 }
 
-int stopControlStream(void* context) {
-	PCONTROL_STREAM stream = (PCONTROL_STREAM) context;
+int stopControlStream(void) {
+	closesocket(ctlSock);
 
-	closesocket(stream->s);
+	PltJoinThread(heartbeatThread);
+	PltJoinThread(jitterThread);
+	PltJoinThread(resyncThread);
 
-	PltJoinThread(stream->heartbeatThread);
-	PltJoinThread(stream->jitterThread);
-	PltJoinThread(stream->resyncThread);
-
-	PltCloseThread(stream->heartbeatThread);
-	PltCloseThread(stream->jitterThread);
-	PltCloseThread(stream->resyncThread);
+	PltCloseThread(heartbeatThread);
+	PltCloseThread(jitterThread);
+	PltCloseThread(resyncThread);
 
 	return 0;
 }
 
-int startControlStream(void* context) {
-	PCONTROL_STREAM stream = (PCONTROL_STREAM) context;
+int startControlStream(void) {
 	int err;
 	char* config;
 	int configSize;
 	PNVCTL_PACKET_HEADER response;
 
-	configSize = getConfigDataSize(&stream->streamConfig);
-	config = allocateConfigDataForStreamConfig(&stream->streamConfig);
+	configSize = getConfigDataSize(&streamConfig);
+	config = allocateConfigDataForStreamConfig(&streamConfig);
 	if (config == NULL) {
 		return NULL;
 	}
 
 	// Send config
-	err = send(stream->s, config, configSize, 0);
+	err = send(ctlSock, config, configSize, 0);
 	free(config);
 	if (err != configSize) {
 		return NULL;
 	}
 
 	// Ping pong
-	response = sendNoPayloadAndReceive(stream, PTYPE_HEARTBEAT, PPAYLEN_HEARTBEAT);
+	response = sendNoPayloadAndReceive(PTYPE_HEARTBEAT, PPAYLEN_HEARTBEAT);
 	if (response == NULL) {
 		return NULL;
 	}
 	free(response);
 
 	// 1405
-	response = sendNoPayloadAndReceive(stream, PTYPE_1405, PPAYLEN_1405);
+	response = sendNoPayloadAndReceive(PTYPE_1405, PPAYLEN_1405);
 	if (response == NULL) {
 		return NULL;
 	}
 	free(response);
 
-	err = PltCreateThread(heartbeatThreadFunc, context, &stream->heartbeatThread);
+	err = PltCreateThread(heartbeatThreadFunc, NULL, &heartbeatThread);
 	if (err != 0) {
 		return err;
 	}
 
-	err = PltCreateThread(jitterThreadFunc, context, &stream->jitterThread);
+	err = PltCreateThread(jitterThreadFunc, NULL, &jitterThread);
 	if (err != 0) {
 		return err;
 	}
 
-	err = PltCreateThread(resyncThreadFunc, context, &stream->resyncThread);
+	err = PltCreateThread(resyncThreadFunc, NULL, &resyncThread);
 	if (err != 0) {
 		return err;
 	}

limelight-common/Limelight.h

@@ -1,5 +1,8 @@
+#pragma once
+
 #include "Platform.h"
 #include "PlatformSockets.h"
+#include "Video.h"
 
 typedef struct _STREAM_CONFIGURATION {
 	int width;
@@ -7,9 +10,19 @@ typedef struct _STREAM_CONFIGURATION {
 	int fps;
 } STREAM_CONFIGURATION, *PSTREAM_CONFIGURATION;
 
-typedef struct _LENTRY {
-	struct _LENTRY *next;
-} LENTRY, *PLENTRY;
+typedef void (*DecoderRendererSetup)(int width, int height, int redrawRate, void* context, int drFlags);
+typedef void (*DecoderRendererStart)(void);
+typedef void (*DecoderRendererStop)(void);
+typedef void (*DecoderRendererRelease)(void);
+typedef void (*DecoderRendererSubmitDecodeUnit)(PDECODE_UNIT decodeUnit);
+
+typedef struct _DECODER_RENDERER_CALLBACKS {
+	DecoderRendererSetup setup;
+	DecoderRendererStart start;
+	DecoderRendererStop stop;
+	DecoderRendererRelease release;
+	DecoderRendererSubmitDecodeUnit submitDecodeUnit;
+} DECODER_RENDERER_CALLBACKS, *PDECODER_RENDERER_CALLBACKS;
 
 #include <stdio.h>
 #define Limelog printf
@@ -17,6 +30,14 @@ typedef struct _LENTRY {
 char* allocateConfigDataForStreamConfig(PSTREAM_CONFIGURATION streamConfig);
 int getConfigDataSize(PSTREAM_CONFIGURATION streamConfig);
 
-void* initializeControlStream(IP_ADDRESS host, PSTREAM_CONFIGURATION streamConfig);
-int startControlStream(void* context);
-int stopControlStream(void* context);
+int initializeControlStream(IP_ADDRESS host, PSTREAM_CONFIGURATION streamConfig);
+int startControlStream(void);
+int stopControlStream(void);
+
+int performHandshake(IP_ADDRESS host);
+
+void initializeVideoDepacketizer(void);
+void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length);
+PDECODE_UNIT getNextDecodeUnit(void);
+void freeDecodeUnit(PDECODE_UNIT decodeUnit);
+void queueRtpPacket(PRTP_PACKET rtpPacket, int length);

limelight-common/LinkedBlockingQueue.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "platform.h"
 #include "PlatformThreads.h"
 

limelight-common/Platform.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifdef _WIN32
 #include <Windows.h>
 #else

limelight-common/PlatformSockets.h

@@ -1,3 +1,4 @@
+#pragma once
 
 #ifdef _WIN32
 #include <Windows.h>

limelight-common/PlatformThreads.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "Platform.h"
 
 typedef void (*ThreadEntry)(void *context);

limelight-common/Video.h

@@ -0,0 +1,34 @@
+#pragma once
+
+typedef void(*RequestIdrFrame)(void);
+
+typedef struct _CONTROL_STATUS_LISTENER {
+	RequestIdrFrame requestIdrFrame;
+} CONTROL_STATUS_LISTENER, *PCONTROL_STATUS_LISTENER;
+
+typedef struct _LENTRY {
+	struct _LENTRY *next;
+	char* data;
+	int length;
+} LENTRY, *PLENTRY;
+
+typedef struct _NV_VIDEO_PACKET {
+	int frameIndex;
+	int packetIndex;
+	int totalPackets;
+	int reserved1;
+	int payloadLength;
+	char reserved2[36];
+} NV_VIDEO_PACKET, *PNV_VIDEO_PACKET;
+
+typedef struct _RTP_PACKET {
+	char flags;
+	char packetType;
+	unsigned short sequenceNumber;
+	char reserved[8];
+} RTP_PACKET, *PRTP_PACKET;
+
+typedef struct _DECODE_UNIT {
+	int fullLength;
+	PLENTRY bufferList;
+} DECODE_UNIT, *PDECODE_UNIT;

limelight-common/VideoDepacketizer.cpp

@@ -1,9 +1,223 @@
 #include "Platform.h"
 #include "Limelight.h"
 #include "LinkedBlockingQueue.h"
+#include "Video.h"
 
-LENTRY *nalChainHead;
+PLENTRY nalChainHead;
 int nalChainDataLength;
 int decodingAvc;
 
-LINKED_BLOCKING_QUEUE decodeUnitQueue;
+LINKED_BLOCKING_QUEUE decodeUnitQueue;
+
+unsigned short lastSequenceNumber;
+
+typedef struct _BUFFER_DESC {
+	char* data;
+	int offset;
+	int length;
+} BUFFER_DESC, *PBUFFER_DESC;
+
+void initializeVideoDepacketizer(void) {
+	initializeLinkedBlockingQueue(&decodeUnitQueue, 15);
+}
+
+static void clearAvcNalState(void) {
+	PLENTRY lastEntry;
+
+	while (nalChainHead != NULL) {
+		lastEntry = nalChainHead;
+		nalChainHead = lastEntry->next;
+		free(lastEntry->data);
+		free(lastEntry);
+	}
+
+	nalChainDataLength = 0;
+}
+
+static int isSeqFrameStart(PBUFFER_DESC candidate) {
+	return (candidate->length == 4 && candidate->data[candidate->offset + candidate->length - 1] == 1);
+}
+
+static int isSeqAvcStart(PBUFFER_DESC candidate) {
+	return (candidate->data[candidate->offset + candidate->length - 1] == 1);
+}
+
+static int isSeqPadding(PBUFFER_DESC candidate) {
+	return (candidate->data[candidate->offset + candidate->length - 1] == 0);
+}
+
+static int getSpecialSeq(PBUFFER_DESC current, PBUFFER_DESC candidate) {
+	if (current->length < 3) {
+		return 0;
+	}
+
+	if (current->data[current->offset] == 0 &&
+		current->data[current->offset + 1] == 0) {
+		// Padding or frame start
+		if (current->data[current->offset + 2] == 0) {
+			if (current->length >= 4 && current->data[current->offset + 3] == 1) {
+				// Frame start
+				candidate->data = current->data;
+				candidate->offset = current->offset;
+				candidate->length = 4;
+				return 1;
+			}
+			else {
+				// Padding
+				candidate->data = current->data;
+				candidate->offset = current->offset;
+				candidate->length = 3;
+				return 1;
+			}
+		}
+		else if (current->data[current->offset + 2] == 1) {
+			// NAL start
+			candidate->data = current->data;
+			candidate->offset = current->offset;
+			candidate->length = 3;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static void reassembleFrame(void) {
+	if (nalChainHead != NULL) {
+		PDECODE_UNIT du = (PDECODE_UNIT) malloc(sizeof(*du));
+		if (du != NULL) {
+			du->bufferList = nalChainHead;
+			du->fullLength = nalChainDataLength;
+
+			nalChainHead = NULL;
+			nalChainDataLength = 0;
+
+			if (!offerQueueItem(&decodeUnitQueue, du)) {
+				nalChainHead = du->bufferList;
+				nalChainDataLength = du->fullLength;
+				free(du);
+
+				clearAvcNalState();
+
+				// FIXME: IDR frame!!!
+			}
+		}
+	}
+}
+
+PDECODE_UNIT getNextDecodeUnit(void) {
+	return (PDECODE_UNIT) waitForQueueElement(&decodeUnitQueue);
+}
+
+void freeDecodeUnit(PDECODE_UNIT decodeUnit) {
+	PLENTRY lastEntry;
+
+	while (decodeUnit->bufferList != NULL) {
+		lastEntry = decodeUnit->bufferList;
+		decodeUnit->bufferList = lastEntry->next;
+		free(lastEntry->data);
+		free(lastEntry);
+	}
+
+	free(decodeUnit);
+}
+
+void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length) {
+	BUFFER_DESC currentPos, specialSeq;
+	
+	currentPos.data = (char*) (videoPacket + 1);
+	currentPos.offset = 0;
+	currentPos.length = length;
+
+	if (currentPos.length == 968) {
+		if (videoPacket->packetIndex < videoPacket->totalPackets) {
+			currentPos.length = videoPacket->payloadLength;
+		}
+		else {
+			return;
+		}
+	}
+
+	while (currentPos.length != 0) {
+		int start = currentPos.offset;
+
+		if (getSpecialSeq(&currentPos, &specialSeq)) {
+			if (isSeqAvcStart(&specialSeq)) {
+				if (isSeqFrameStart(&specialSeq)) {
+					decodingAvc = 1;
+
+					reassembleFrame();
+				}
+
+				currentPos.length -= specialSeq.length;
+				currentPos.offset += specialSeq.length;
+			}
+			else {
+				if (decodingAvc && isSeqPadding(&currentPos)) {
+					reassembleFrame();
+				}
+
+				decodingAvc = 0;
+
+				currentPos.length--;
+				currentPos.offset++;
+			}
+		}
+
+		while (currentPos.length != 0) {
+			if (getSpecialSeq(&currentPos, &specialSeq)) {
+				if (decodingAvc || isSeqPadding(&specialSeq)) {
+					break;
+				}
+			}
+
+			currentPos.offset++;
+			currentPos.length--;
+		}
+
+		if (decodingAvc) {
+			PLENTRY entry = (PLENTRY) malloc(sizeof(*entry));
+			if (entry != NULL) {
+				entry->length = currentPos.offset - start;
+				entry->data = (char*) malloc(entry->length);
+				if (entry->data == NULL) {
+					free(entry);
+					return;
+				}
+
+				memcpy(entry->data, &currentPos.data[start], entry->length);
+				
+				nalChainDataLength += entry->length;
+
+				if (nalChainHead == NULL) {
+					nalChainHead = entry;
+				}
+				else {
+					PLENTRY currentEntry = nalChainHead;
+
+					while (currentEntry->next != NULL) {
+						currentEntry = currentEntry->next;
+					}
+
+					currentEntry->next = entry;
+				}
+			}
+		}
+	}
+}
+
+void queueRtpPacket(PRTP_PACKET rtpPacket, int length) {
+	if (lastSequenceNumber != 0 &&
+		(unsigned short) (lastSequenceNumber + 1) != rtpPacket->sequenceNumber) {
+		Limelog("Received OOS video data (expected %d, but got %d)\n", lastSequenceNumber + 1, rtpPacket->sequenceNumber);
+
+		clearAvcNalState();
+
+		// FIXME: IDR frame here!!!
+	}
+
+	lastSequenceNumber = rtpPacket->sequenceNumber;
+
+	processRtpPayload((PNV_VIDEO_PACKET) (rtpPacket + 1), length - sizeof(*rtpPacket));
+}
+

limelight-common/VideoStream.cpp

@@ -0,0 +1,5 @@
+#include "Limelight.h"
+
+void* initializeVideoStream(IP_ADDRESS host, PSTREAM_CONFIGURATION streamConfig) {
+	return NULL;
+}

limelight-common/limelight-common.vcxproj

@@ -82,6 +82,7 @@
     <ClCompile Include="PlatformSockets.cpp" />
     <ClCompile Include="PlatformThreads.cpp" />
     <ClCompile Include="VideoDepacketizer.cpp" />
+    <ClCompile Include="VideoStream.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="ByteBuffer.h" />
@@ -90,6 +91,7 @@
     <ClInclude Include="Platform.h" />
     <ClInclude Include="PlatformSockets.h" />
     <ClInclude Include="PlatformThreads.h" />
+    <ClInclude Include="Video.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

limelight-common/limelight-common.vcxproj.filters

@@ -42,6 +42,9 @@
     <ClCompile Include="LinkedBlockingQueue.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="VideoStream.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="PlatformSockets.h">
@@ -62,5 +65,8 @@
     <ClInclude Include="LinkedBlockingQueue.h">
       <Filter>Source Files</Filter>
     </ClInclude>
+    <ClInclude Include="Video.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>