Switch to a pull-based renderer and render from CADisplayLink callback

Limelight/Stream/Connection.m

@@ -47,7 +47,7 @@ static VideoDecoderRenderer* renderer;
 
 int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags)
 {
-    [renderer setupWithVideoFormat:videoFormat refreshRate:redrawRate];
+    [renderer setupWithVideoFormat:videoFormat frameRate:redrawRate];
     lastFrameNumber = 0;
     activeVideoFormat = videoFormat;
     memset(&currentVideoStats, 0, sizeof(currentVideoStats));
@@ -55,9 +55,14 @@ int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void*
     return 0;
 }
 
-void DrCleanup(void)
+void DrStart(void)
 {
-    [renderer cleanup];
+    [renderer start];
+}
+
+void DrStop(void)
+{
+    [renderer stop];
 }
 
 -(BOOL) getVideoStats:(video_stats_t*)stats
@@ -420,8 +425,8 @@ void ClConnectionStatusUpdate(int status)
 
     LiInitializeVideoCallbacks(&_drCallbacks);
     _drCallbacks.setup = DrDecoderSetup;
-    _drCallbacks.cleanup = DrCleanup;
-    _drCallbacks.submitDecodeUnit = DrSubmitDecodeUnit;
+    _drCallbacks.start = DrStart;
+    _drCallbacks.stop = DrStop;
 
     // RFI doesn't work properly with HEVC on iOS 11 with an iPhone SE (at least)
     // It doesnt work on macOS either, tested with Network Link Conditioner.
@@ -431,7 +436,7 @@ void ClConnectionStatusUpdate(int status)
 #if !TARGET_OS_TV
                                 CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC |
 #endif
-                                CAPABILITY_DIRECT_SUBMIT;
+                                CAPABILITY_PULL_RENDERER;
 
     LiInitializeAudioCallbacks(&_arCallbacks);
     _arCallbacks.init = ArInit;

Limelight/Stream/VideoDecoderRenderer.h

@@ -14,9 +14,9 @@
 
 - (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks;
 
-- (void)setupWithVideoFormat:(int)videoFormat refreshRate:(int)refreshRate;
-
-- (void)cleanup;
+- (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate;
+- (void)start;
+- (void)stop;
 
 - (void)updateBufferForRange:(CMBlockBufferRef)existingBuffer data:(unsigned char *)data offset:(int)offset length:(int)nalLength;
 

Limelight/Stream/VideoDecoderRenderer.m

@@ -18,6 +18,7 @@
     AVSampleBufferDisplayLayer* displayLayer;
     Boolean waitingForSps, waitingForPps, waitingForVps;
     int videoFormat;
+    int frameRate;
     
     NSData *spsData, *ppsData, *vpsData;
     CMVideoFormatDescriptionRef formatDesc;
@@ -74,32 +75,39 @@
     return self;
 }
 
-- (void)setupWithVideoFormat:(int)videoFormat refreshRate:(int)refreshRate
+- (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate
 {
     self->videoFormat = videoFormat;
-    
-    if (refreshRate > 60) {
-        // HACK: We seem to just get 60 Hz screen updates even with a 120 FPS stream if
-        // we don't set preferredFramesPerSecond somewhere. Since we're a UIKit view, we
-        // have to use CADisplayLink for that. See https://github.com/moonlight-stream/moonlight-ios/issues/372
-        _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)];
-        if (@available(iOS 10.0, tvOS 10.0, *)) {
-            _displayLink.preferredFramesPerSecond = refreshRate;
-        }
-        [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode];
-    }
-}
-                     
-- (void)displayLinkCallback:(CADisplayLink *)sender
-{
-    // No-op - rendering done in submitDecodeBuffer
+    self->frameRate = frameRate;
 }
 
-- (void)cleanup
+- (void)start
+{
+    _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)];
+    if (@available(iOS 10.0, tvOS 10.0, *)) {
+        _displayLink.preferredFramesPerSecond = self->frameRate;
+    }
+    [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode];
+}
+
+- (void)stop
 {
     [_displayLink invalidate];
 }
 
+// TODO: Refactor this
+int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
+
+- (void)displayLinkCallback:(CADisplayLink *)sender
+{
+    VIDEO_FRAME_HANDLE handle;
+    PDECODE_UNIT du;
+    
+    while (LiPollNextVideoFrame(&handle, &du)) {
+        LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du));
+    }
+}
+
 #define FRAME_START_PREFIX_SIZE 4
 #define NALU_START_PREFIX_SIZE 3
 #define NAL_LENGTH_PREFIX_SIZE 4
@@ -343,23 +351,20 @@
         CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse);
     }
 
-    // Enqueue video samples on the main thread
-    dispatch_async(dispatch_get_main_queue(), ^{
-        // Enqueue the next frame
-        [self->displayLayer enqueueSampleBuffer:sampleBuffer];
+    // Enqueue the next frame
+    [self->displayLayer enqueueSampleBuffer:sampleBuffer];
+    
+    if (frameType == FRAME_TYPE_IDR) {
+        // Ensure the layer is visible now
+        self->displayLayer.hidden = NO;
         
-        if (frameType == FRAME_TYPE_IDR) {
-            // Ensure the layer is visible now
-            self->displayLayer.hidden = NO;
-            
-            // Tell our parent VC to hide the progress indicator
-            [self->_callbacks videoContentShown];
-        }
-        
-        // Dereference the buffers
-        CFRelease(blockBuffer);
-        CFRelease(sampleBuffer);
-    });
+        // Tell our parent VC to hide the progress indicator
+        [self->_callbacks videoContentShown];
+    }
+    
+    // Dereference the buffers
+    CFRelease(blockBuffer);
+    CFRelease(sampleBuffer);
     
     return DR_OK;
 }