From bd582aa6c0c0e4db283c166395ef3d362201fcc6 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Thu, 13 Jan 2022 22:37:18 -0600 Subject: [PATCH] Switch to a pull-based renderer and render from CADisplayLink callback --- Limelight/Stream/Connection.m | 17 ++++-- Limelight/Stream/VideoDecoderRenderer.h | 6 +- Limelight/Stream/VideoDecoderRenderer.m | 73 +++++++++++++------------ 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 3a3ca4e..2129ca2 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -47,7 +47,7 @@ static VideoDecoderRenderer* renderer; int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { - [renderer setupWithVideoFormat:videoFormat refreshRate:redrawRate]; + [renderer setupWithVideoFormat:videoFormat frameRate:redrawRate]; lastFrameNumber = 0; activeVideoFormat = videoFormat; memset(¤tVideoStats, 0, sizeof(currentVideoStats)); @@ -55,9 +55,14 @@ int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* return 0; } -void DrCleanup(void) +void DrStart(void) { - [renderer cleanup]; + [renderer start]; +} + +void DrStop(void) +{ + [renderer stop]; } -(BOOL) getVideoStats:(video_stats_t*)stats @@ -420,8 +425,8 @@ void ClConnectionStatusUpdate(int status) LiInitializeVideoCallbacks(&_drCallbacks); _drCallbacks.setup = DrDecoderSetup; - _drCallbacks.cleanup = DrCleanup; - _drCallbacks.submitDecodeUnit = DrSubmitDecodeUnit; + _drCallbacks.start = DrStart; + _drCallbacks.stop = DrStop; // RFI doesn't work properly with HEVC on iOS 11 with an iPhone SE (at least) // It doesnt work on macOS either, tested with Network Link Conditioner. @@ -431,7 +436,7 @@ void ClConnectionStatusUpdate(int status) #if !TARGET_OS_TV CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC | #endif - CAPABILITY_DIRECT_SUBMIT; + CAPABILITY_PULL_RENDERER; LiInitializeAudioCallbacks(&_arCallbacks); _arCallbacks.init = ArInit; diff --git a/Limelight/Stream/VideoDecoderRenderer.h b/Limelight/Stream/VideoDecoderRenderer.h index c0a57bb..5a6fb4b 100644 --- a/Limelight/Stream/VideoDecoderRenderer.h +++ b/Limelight/Stream/VideoDecoderRenderer.h @@ -14,9 +14,9 @@ - (id)initWithView:(UIView*)view callbacks:(id)callbacks; -- (void)setupWithVideoFormat:(int)videoFormat refreshRate:(int)refreshRate; - -- (void)cleanup; +- (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate; +- (void)start; +- (void)stop; - (void)updateBufferForRange:(CMBlockBufferRef)existingBuffer data:(unsigned char *)data offset:(int)offset length:(int)nalLength; diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index 24149c8..bc553ac 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -18,6 +18,7 @@ AVSampleBufferDisplayLayer* displayLayer; Boolean waitingForSps, waitingForPps, waitingForVps; int videoFormat; + int frameRate; NSData *spsData, *ppsData, *vpsData; CMVideoFormatDescriptionRef formatDesc; @@ -74,32 +75,39 @@ return self; } -- (void)setupWithVideoFormat:(int)videoFormat refreshRate:(int)refreshRate +- (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate { self->videoFormat = videoFormat; - - if (refreshRate > 60) { - // HACK: We seem to just get 60 Hz screen updates even with a 120 FPS stream if - // we don't set preferredFramesPerSecond somewhere. Since we're a UIKit view, we - // have to use CADisplayLink for that. See https://github.com/moonlight-stream/moonlight-ios/issues/372 - _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; - if (@available(iOS 10.0, tvOS 10.0, *)) { - _displayLink.preferredFramesPerSecond = refreshRate; - } - [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; - } -} - -- (void)displayLinkCallback:(CADisplayLink *)sender -{ - // No-op - rendering done in submitDecodeBuffer + self->frameRate = frameRate; } -- (void)cleanup +- (void)start +{ + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; + if (@available(iOS 10.0, tvOS 10.0, *)) { + _displayLink.preferredFramesPerSecond = self->frameRate; + } + [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; +} + +- (void)stop { [_displayLink invalidate]; } +// TODO: Refactor this +int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); + +- (void)displayLinkCallback:(CADisplayLink *)sender +{ + VIDEO_FRAME_HANDLE handle; + PDECODE_UNIT du; + + while (LiPollNextVideoFrame(&handle, &du)) { + LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du)); + } +} + #define FRAME_START_PREFIX_SIZE 4 #define NALU_START_PREFIX_SIZE 3 #define NAL_LENGTH_PREFIX_SIZE 4 @@ -343,23 +351,20 @@ CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); } - // Enqueue video samples on the main thread - dispatch_async(dispatch_get_main_queue(), ^{ - // Enqueue the next frame - [self->displayLayer enqueueSampleBuffer:sampleBuffer]; + // Enqueue the next frame + [self->displayLayer enqueueSampleBuffer:sampleBuffer]; + + if (frameType == FRAME_TYPE_IDR) { + // Ensure the layer is visible now + self->displayLayer.hidden = NO; - if (frameType == FRAME_TYPE_IDR) { - // Ensure the layer is visible now - self->displayLayer.hidden = NO; - - // Tell our parent VC to hide the progress indicator - [self->_callbacks videoContentShown]; - } - - // Dereference the buffers - CFRelease(blockBuffer); - CFRelease(sampleBuffer); - }); + // Tell our parent VC to hide the progress indicator + [self->_callbacks videoContentShown]; + } + + // Dereference the buffers + CFRelease(blockBuffer); + CFRelease(sampleBuffer); return DR_OK; }