Fix state cleanup if LiStartConnection() fails

src/AudioStream.c

@@ -281,27 +281,47 @@ void stopAudioStream(void) {
 int startAudioStream(void) {
     int err;
 
-    AudioCallbacks.init(StreamConfig.audioConfiguration,
+    err = AudioCallbacks.init(StreamConfig.audioConfiguration,
         opusConfigArray[StreamConfig.audioConfiguration]);
+    if (err != 0) {
+        return err;
+    }
 
     rtpSocket = bindUdpSocket(RemoteAddr.ss_family, RTP_RECV_BUFFER);
     if (rtpSocket == INVALID_SOCKET) {
-        return LastSocketFail();
+        err = LastSocketFail();
+        AudioCallbacks.cleanup();
+        return err;
     }
 
     err = PltCreateThread(UdpPingThreadProc, NULL, &udpPingThread);
     if (err != 0) {
+        AudioCallbacks.cleanup();
+        closeSocket(rtpSocket);
         return err;
     }
 
     err = PltCreateThread(ReceiveThreadProc, NULL, &receiveThread);
     if (err != 0) {
+        PltInterruptThread(&udpPingThread);
+        PltJoinThread(&udpPingThread);
+        PltCloseThread(&udpPingThread);
+        closeSocket(rtpSocket);
+        AudioCallbacks.cleanup();
         return err;
     }
 
     if ((AudioCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
         err = PltCreateThread(DecoderThreadProc, NULL, &decoderThread);
         if (err != 0) {
+            PltInterruptThread(&udpPingThread);
+            PltInterruptThread(&receiveThread);
+            PltJoinThread(&udpPingThread);
+            PltJoinThread(&receiveThread);
+            PltCloseThread(&udpPingThread);
+            PltCloseThread(&receiveThread);
+            closeSocket(rtpSocket);
+            AudioCallbacks.cleanup();
             return err;
         }
     }

src/Connection.c

@@ -374,5 +374,9 @@ int LiStartConnection(PSERVER_INFORMATION serverInfo, PSTREAM_CONFIGURATION stre
     ListenerCallbacks.connectionStarted();
 
 Cleanup:
+    if (err != 0) {
+        // Undo any work we've done here before failing
+        LiStopConnection();
+    }
     return err;
 }

src/ControlStream.c

@@ -680,7 +680,19 @@ int startControlStream(void) {
         payloadLengths[IDX_START_A],
         preconstructedPayloads[IDX_START_A])) {
         Limelog("Start A failed: %d\n", (int)LastSocketError());
-        return LastSocketFail();
+        err = LastSocketFail();
+        stopping = 1;
+        if (ctlSock != INVALID_SOCKET) {
+            closeSocket(ctlSock);
+            ctlSock = INVALID_SOCKET;
+        }
+        else {
+            enet_peer_disconnect_now(peer, 0);
+            peer = NULL;
+            enet_host_destroy(client);
+            client = NULL;
+        }
+        return err;
     }
 
     // Send START B
@@ -688,16 +700,63 @@ int startControlStream(void) {
         payloadLengths[IDX_START_B],
         preconstructedPayloads[IDX_START_B])) {
         Limelog("Start B failed: %d\n", (int)LastSocketError());
-        return LastSocketFail();
+        err = LastSocketFail();
+        stopping = 1;
+        if (ctlSock != INVALID_SOCKET) {
+            closeSocket(ctlSock);
+            ctlSock = INVALID_SOCKET;
+        }
+        else {
+            enet_peer_disconnect_now(peer, 0);
+            peer = NULL;
+            enet_host_destroy(client);
+            client = NULL;
+        }
+        return err;
     }
 
     err = PltCreateThread(lossStatsThreadFunc, NULL, &lossStatsThread);
     if (err != 0) {
+        stopping = 1;
+        if (ctlSock != INVALID_SOCKET) {
+            closeSocket(ctlSock);
+            ctlSock = INVALID_SOCKET;
+        }
+        else {
+            enet_peer_disconnect_now(peer, 0);
+            peer = NULL;
+            enet_host_destroy(client);
+            client = NULL;
+        }
         return err;
     }
 
     err = PltCreateThread(invalidateRefFramesFunc, NULL, &invalidateRefFramesThread);
     if (err != 0) {
+        stopping = 1;
+
+        if (ctlSock != INVALID_SOCKET) {
+            shutdownTcpSocket(ctlSock);
+        }
+
+        if (peer != NULL) {
+            enet_peer_disconnect_now(peer, 0);
+            peer = NULL;
+        }
+
+        PltInterruptThread(&lossStatsThread);
+        PltJoinThread(&lossStatsThread);
+        PltCloseThread(&lossStatsThread);
+
+        if (ctlSock != INVALID_SOCKET) {
+            closeSocket(ctlSock);
+            ctlSock = INVALID_SOCKET;
+        }
+        else {
+            enet_host_destroy(client);
+            client = NULL;
+        }
+
         return err;
     }
 

src/FakeCallbacks.c

@@ -1,6 +1,6 @@
 #include "Limelight-internal.h"
 
-static void fakeDrSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {}
+static int fakeDrSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { return 0; }
 static void fakeDrCleanup(void) {}
 static int fakeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) { return DR_OK; }
 
@@ -10,7 +10,7 @@ static DECODER_RENDERER_CALLBACKS fakeDrCallbacks = {
     .submitDecodeUnit = fakeDrSubmitDecodeUnit,
 };
 
-static void fakeArInit(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig) {}
+static int fakeArInit(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig) { return 0; }
 static void fakeArCleanup(void) {}
 static void fakeArDecodeAndPlaySample(char* sampleData, int sampleLength) {}
 

src/InputStream.c

@@ -394,6 +394,10 @@ int startInputStream(void) {
 
     err = PltCreateThread(inputSendThreadProc, NULL, &inputSendThread);
     if (err != 0) {
+        if (inputSock != INVALID_SOCKET) {
+            closeSocket(inputSock);
+            inputSock = INVALID_SOCKET;
+        }
         return err;
     }
 

src/Limelight.h

@@ -101,8 +101,9 @@ typedef struct _DECODE_UNIT {
 // number of slices per frame. This capability is only valid on video renderers.
 #define CAPABILITY_SLICES_PER_FRAME(x) (((unsigned char)(x)) << 24)
 
-// This callback is invoked to provide details about the video stream and allow configuration of the decoder
+// This callback is invoked to provide details about the video stream and allow configuration of the decoder.
-typedef void(*DecoderRendererSetup)(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags);
+// Returns 0 on success, non-zero on failure.
+typedef int(*DecoderRendererSetup)(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags);
 
 // This callback performs the teardown of the video decoder
 typedef void(*DecoderRendererCleanup)(void);
@@ -148,8 +149,8 @@ typedef struct _OPUS_MULTISTREAM_CONFIGURATION {
 
 // This callback initializes the audio renderer. The audio configuration parameter
 // provides the negotiated audio configuration. This may differ from the one
-// specified in the stream configuration.
+// specified in the stream configuration. Returns 0 on success, non-zero on failure.
-typedef void(*AudioRendererInit)(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig);
+typedef int(*AudioRendererInit)(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig);
 
 // This callback performs the final teardown of the audio decoder
 typedef void(*AudioRendererCleanup)(void);

src/VideoStream.c

@@ -192,25 +192,38 @@ void stopVideoStream(void) {
 int startVideoStream(void* rendererContext, int drFlags) {
     int err;
 
+    firstFrameSocket = INVALID_SOCKET;
+
     // This must be called before the decoder thread starts submitting
     // decode units
     LC_ASSERT(NegotiatedVideoFormat != 0);
-    VideoCallbacks.setup(NegotiatedVideoFormat, StreamConfig.width,
+    err = VideoCallbacks.setup(NegotiatedVideoFormat, StreamConfig.width,
         StreamConfig.height, StreamConfig.fps, rendererContext, drFlags);
+    if (err != 0) {
+        return err;
+    }
 
     rtpSocket = bindUdpSocket(RemoteAddr.ss_family, RTP_RECV_BUFFER);
     if (rtpSocket == INVALID_SOCKET) {
+        VideoCallbacks.cleanup();
         return LastSocketError();
     }
 
     err = PltCreateThread(ReceiveThreadProc, NULL, &receiveThread);
     if (err != 0) {
+        closeSocket(rtpSocket);
+        VideoCallbacks.cleanup();
         return err;
     }
 
     if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
         err = PltCreateThread(DecoderThreadProc, NULL, &decoderThread);
         if (err != 0) {
+            PltInterruptThread(&receiveThread);
+            PltJoinThread(&receiveThread);
+            PltCloseThread(&receiveThread);
+            closeSocket(rtpSocket);
+            VideoCallbacks.cleanup();
             return err;
         }
     }
@@ -220,6 +233,21 @@ int startVideoStream(void* rendererContext, int drFlags) {
         firstFrameSocket = connectTcpSocket(&RemoteAddr, RemoteAddrLen,
                                             FIRST_FRAME_PORT, FIRST_FRAME_TIMEOUT_SEC);
         if (firstFrameSocket == INVALID_SOCKET) {
+            stopVideoDepacketizer();
+            PltInterruptThread(&receiveThread);
+            if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+                PltInterruptThread(&decoderThread);
+            }
+            PltJoinThread(&receiveThread);
+            if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+                PltJoinThread(&decoderThread);
+            }
+            PltCloseThread(&receiveThread);
+            if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+                PltCloseThread(&decoderThread);
+            }
+            closeSocket(rtpSocket);
+            VideoCallbacks.cleanup();
             return LastSocketError();
         }
     }
@@ -228,6 +256,25 @@ int startVideoStream(void* rendererContext, int drFlags) {
     // to send UDP data
     err = PltCreateThread(UdpPingThreadProc, NULL, &udpPingThread);
     if (err != 0) {
+        stopVideoDepacketizer();
+        PltInterruptThread(&receiveThread);
+        if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+            PltInterruptThread(&decoderThread);
+        }
+        PltJoinThread(&receiveThread);
+        if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+            PltJoinThread(&decoderThread);
+        }
+        PltCloseThread(&receiveThread);
+        if ((VideoCallbacks.capabilities & CAPABILITY_DIRECT_SUBMIT) == 0) {
+            PltCloseThread(&decoderThread);
+        }
+        closeSocket(rtpSocket);
+        if (firstFrameSocket != INVALID_SOCKET) {
+            closeSocket(firstFrameSocket);
+            firstFrameSocket = INVALID_SOCKET;
+        }
+        VideoCallbacks.cleanup();
         return err;
     }
 
@@ -235,6 +282,7 @@ int startVideoStream(void* rendererContext, int drFlags) {
         // Read the first frame to start the flow of video
         err = readFirstFrame();
         if (err != 0) {
+            stopVideoStream();
             return err;
         }
     }