diff --git a/Limelight/Connection.m b/Limelight/Connection.m index 897b6e4..fa89453 100644 --- a/Limelight/Connection.m +++ b/Limelight/Connection.m @@ -11,8 +11,6 @@ #import #import -#include - #include "Limelight.h" #include "opus.h" @@ -42,7 +40,6 @@ struct AUDIO_BUFFER_QUEUE_ENTRY { static short decodedPcmBuffer[512]; static NSLock *audioLock; static struct AUDIO_BUFFER_QUEUE_ENTRY *audioBufferQueue; -static dispatch_semaphore_t audioQueueSemaphore; static int audioBufferQueueLength; static AudioComponentInstance audioUnit; static VideoDecoderRenderer* renderer; @@ -65,9 +62,8 @@ void DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) entry = entry->next; } + // This function will take our buffer [renderer submitDecodeBuffer:data length:decodeUnit->fullLength]; - - free(data); } } @@ -95,8 +91,6 @@ void ArInit(void) opusDecoder = opus_decoder_create(48000, 2, &err); audioLock = [[NSLock alloc] init]; - - audioQueueSemaphore = dispatch_semaphore_create(0); } void ArRelease(void) @@ -112,7 +106,6 @@ void ArRelease(void) void ArStart(void) { printf("Start audio\n"); - AudioOutputUnitStart(audioUnit); } void ArStop(void) @@ -141,15 +134,9 @@ void ArDecodeAndPlaySample(char* sampleData, int sampleLength) // Clear all values from the buffer queue struct AUDIO_BUFFER_QUEUE_ENTRY *entry; while (audioBufferQueue != NULL) { - // Unlink the current entry entry = audioBufferQueue; audioBufferQueue = entry->next; - - // Decrease the semaphore count and queue length audioBufferQueueLength--; - dispatch_semaphore_wait(audioQueueSemaphore, DISPATCH_TIME_NOW); - - // Free the entry free(entry); } } @@ -164,13 +151,9 @@ void ArDecodeAndPlaySample(char* sampleData, int sampleLength) } lastEntry->next = newEntry; } - - // Increment the queue size and unlock audioBufferQueueLength++; - [audioLock unlock]; - // Signal the other thread - dispatch_semaphore_signal(audioQueueSemaphore); + [audioLock unlock]; } } } @@ -312,6 +295,13 @@ void ClDisplayTransientMessage(char* message) NSLog(@"Unable to initialize audioUnit: %d", (int32_t)status); } + // We start here because it seems to need some warmup time + // before it starts accepting samples + status = AudioOutputUnitStart(audioUnit); + if (status) { + NSLog(@"Unable to start audioUnit: %d", (int32_t)status); + } + return self; } @@ -325,9 +315,15 @@ static OSStatus playbackCallback(void *inRefCon, // Fill them up as much as you can. Remember to set the size value in each buffer to match how // much data is in the buffer. + bool ranOutOfData = false; for (int i = 0; i < ioData->mNumberBuffers; i++) { ioData->mBuffers[i].mNumberChannels = 2; + if (ranOutOfData) { + ioData->mBuffers[i].mDataByteSize = 0; + continue; + } + if (ioData->mBuffers[i].mDataByteSize != 0) { int thisBufferOffset = 0; @@ -337,22 +333,22 @@ static OSStatus playbackCallback(void *inRefCon, continue; } - // Wait for a buffer to be available struct AUDIO_BUFFER_QUEUE_ENTRY *audioEntry = NULL; - while (audioEntry == NULL) - { - // Wait for an entry to be present in the queue - dispatch_semaphore_wait(audioQueueSemaphore, DISPATCH_TIME_FOREVER); - - // If there's an entry there, dequeue it - [audioLock lock]; - if (audioBufferQueue != NULL) { - // Dequeue this entry temporarily - audioEntry = audioBufferQueue; - audioBufferQueue = audioBufferQueue->next; - audioBufferQueueLength--; - } - [audioLock unlock]; + + [audioLock lock]; + if (audioBufferQueue != NULL) { + // Dequeue this entry temporarily + audioEntry = audioBufferQueue; + audioBufferQueue = audioBufferQueue->next; + audioBufferQueueLength--; + } + [audioLock unlock]; + + if (audioEntry == NULL) { + // No data left + ranOutOfData = true; + ioData->mBuffers[i].mDataByteSize = thisBufferOffset; + continue; } // Figure out how much data we can write @@ -373,7 +369,6 @@ static OSStatus playbackCallback(void *inRefCon, audioBufferQueue = audioEntry; audioBufferQueueLength++; [audioLock unlock]; - dispatch_semaphore_signal(audioQueueSemaphore); } else { // This entry is fully depleted so free it diff --git a/Limelight/VideoDecoderRenderer.m b/Limelight/VideoDecoderRenderer.m index 813e257..50f602c 100644 --- a/Limelight/VideoDecoderRenderer.m +++ b/Limelight/VideoDecoderRenderer.m @@ -23,7 +23,7 @@ displayLayer = [[AVSampleBufferDisplayLayer alloc] init]; displayLayer.bounds = view.bounds; - displayLayer.backgroundColor = [UIColor greenColor].CGColor; + displayLayer.backgroundColor = [UIColor blackColor].CGColor; displayLayer.position = CGPointMake(CGRectGetMidX(view.bounds), CGRectGetMidY(view.bounds)); displayLayer.videoGravity = AVLayerVideoGravityResizeAspect; [view.layer addSublayer:displayLayer]; @@ -46,38 +46,69 @@ OSStatus status; size_t oldOffset = CMBlockBufferGetDataLength(existingBuffer); - // Append a 4 byte buffer to this block for the length prefix - status = CMBlockBufferAppendMemoryBlock(existingBuffer, NULL, - NAL_LENGTH_PREFIX_SIZE, - kCFAllocatorDefault, NULL, 0, - NAL_LENGTH_PREFIX_SIZE, 0); - if (status != noErr) { - NSLog(@"CMBlockBufferAppendMemoryBlock failed: %d", (int)status); - return; + // If we're at index 1 (first NALU in frame), enqueue this buffer to the memory block + // so it can handle freeing it when the block buffer is destroyed + if (offset == 1) { + int dataLength = nalLength - NALU_START_PREFIX_SIZE; + + // Pass the real buffer pointer directly (no offset) + // This will give it to the block buffer to free when it's released. + // All further calls to CMBlockBufferAppendMemoryBlock will do so + // at an offset and will not be asking the buffer to be freed. + status = CMBlockBufferAppendMemoryBlock(existingBuffer, data, + nalLength + 1, // Add 1 for the offset we decremented + kCFAllocatorDefault, + NULL, 0, nalLength + 1, 0); + if (status != noErr) { + NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); + return; + } + + // Write the length prefix to existing buffer + const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16), + (uint8_t)(dataLength >> 8), (uint8_t)dataLength}; + status = CMBlockBufferReplaceDataBytes(lengthBytes, existingBuffer, + oldOffset, NAL_LENGTH_PREFIX_SIZE); + if (status != noErr) { + NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); + return; + } } - - // Write the length prefix to the new buffer - int dataLength = nalLength - NALU_START_PREFIX_SIZE; - const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16), - (uint8_t)(dataLength >> 8), (uint8_t)dataLength}; - status = CMBlockBufferReplaceDataBytes(lengthBytes, existingBuffer, - oldOffset, NAL_LENGTH_PREFIX_SIZE); - if (status != noErr) { - NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); - return; - } - - // Append the rest of the data by simply attaching a reference to the buffer - status = CMBlockBufferAppendMemoryBlock(existingBuffer, &data[offset+NALU_START_PREFIX_SIZE], - dataLength, - kCFAllocatorNull, // Don't deallocate data on free - NULL, 0, dataLength, 0); - if (status != noErr) { - NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); - return; + else { + // Append a 4 byte buffer to this block for the length prefix + status = CMBlockBufferAppendMemoryBlock(existingBuffer, NULL, + NAL_LENGTH_PREFIX_SIZE, + kCFAllocatorDefault, NULL, 0, + NAL_LENGTH_PREFIX_SIZE, 0); + if (status != noErr) { + NSLog(@"CMBlockBufferAppendMemoryBlock failed: %d", (int)status); + return; + } + + // Write the length prefix to the new buffer + int dataLength = nalLength - NALU_START_PREFIX_SIZE; + const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16), + (uint8_t)(dataLength >> 8), (uint8_t)dataLength}; + status = CMBlockBufferReplaceDataBytes(lengthBytes, existingBuffer, + oldOffset, NAL_LENGTH_PREFIX_SIZE); + if (status != noErr) { + NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); + return; + } + + // Attach the buffer by reference to the block buffer + status = CMBlockBufferAppendMemoryBlock(existingBuffer, &data[offset+NALU_START_PREFIX_SIZE], + dataLength, + kCFAllocatorNull, // Don't deallocate data on free + NULL, 0, dataLength, 0); + if (status != noErr) { + NSLog(@"CMBlockBufferReplaceDataBytes failed: %d", (int)status); + return; + } } } +// This function must free data - (void)submitDecodeBuffer:(unsigned char *)data length:(int)length { unsigned char nalType = data[FRAME_START_PREFIX_SIZE] & 0x1F; @@ -120,21 +151,25 @@ if (status != noErr) { NSLog(@"Failed to create format description: %d", (int)status); formatDesc = NULL; - return; } } + // Free the data buffer + free(data); + // No frame data to submit for these NALUs return; } if (formatDesc == NULL) { // Can't decode if we haven't gotten our parameter sets yet + free(data); return; } if (nalType != 0x1 && nalType != 0x5) { // Don't submit parameter set data + free(data); return; } @@ -144,9 +179,10 @@ status = CMBlockBufferCreateEmpty(NULL, 0, 0, &blockBuffer); if (status != noErr) { NSLog(@"CMBlockBufferCreateEmpty failed: %d", (int)status); + free(data); return; } - + int lastOffset = -1; for (int i = 0; i < length - FRAME_START_PREFIX_SIZE; i++) { // Search for a NALU @@ -166,6 +202,8 @@ [self updateBufferForRange:blockBuffer data:data offset:lastOffset length:length - lastOffset]; } + // From now on, CMBlockBuffer owns the data pointer and will free it when it's dereferenced + CMSampleBufferRef sampleBuffer; status = CMSampleBufferCreate(kCFAllocatorDefault, @@ -176,6 +214,7 @@ &sampleBuffer); if (status != noErr) { NSLog(@"CMSampleBufferCreate failed: %d", (int)status); + CFRelease(blockBuffer); return; } @@ -197,6 +236,10 @@ } [displayLayer enqueueSampleBuffer:sampleBuffer]; + + // Dereference the buffers + CFRelease(blockBuffer); + CFRelease(sampleBuffer); } @end