From 7d6cb247b82fe39ca844af173da282e12b8def76 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Wed, 2 Feb 2022 13:37:07 -0300 Subject: [PATCH] Add Frame Pacing feature --- Limelight/Database/DataManager.h | 1 + Limelight/Database/DataManager.m | 2 + Limelight/Database/TemporarySettings.h | 1 + Limelight/Database/TemporarySettings.m | 2 + .../Limelight.xcdatamodeld/.xccurrentversion | 2 +- .../Moonlight v1.7.xcdatamodel/contents | 45 +++++++++++++++++++ Limelight/Stream/Connection.m | 1 - Limelight/Stream/StreamConfiguration.h | 1 + Limelight/Stream/StreamManager.m | 2 +- Limelight/Stream/VideoDecoderRenderer.h | 2 +- Limelight/Stream/VideoDecoderRenderer.m | 29 +++++++----- .../ViewControllers/MainFrameViewController.m | 3 +- .../ViewControllers/SettingsViewController.h | 1 + .../ViewControllers/SettingsViewController.m | 3 ++ Moonlight TV/Settings.bundle/Root.plist | 10 +++++ Moonlight.xcodeproj/project.pbxproj | 4 +- iPad.storyboard | 30 ++++++++++--- iPhone.storyboard | 28 +++++++++--- moonlight-common/moonlight-common-c | 2 +- 19 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 Limelight/Limelight.xcdatamodeld/Moonlight v1.7.xcdatamodel/contents diff --git a/Limelight/Database/DataManager.h b/Limelight/Database/DataManager.h index d155885..d630165 100644 --- a/Limelight/Database/DataManager.h +++ b/Limelight/Database/DataManager.h @@ -22,6 +22,7 @@ multiController:(BOOL)multiController audioOnPC:(BOOL)audioOnPC useHevc:(BOOL)useHevc + useFramePacing:(BOOL)useFramePacing enableHdr:(BOOL)enableHdr btMouseSupport:(BOOL)btMouseSupport absoluteTouchMode:(BOOL)absoluteTouchMode diff --git a/Limelight/Database/DataManager.m b/Limelight/Database/DataManager.m index a4049a9..4402a87 100644 --- a/Limelight/Database/DataManager.m +++ b/Limelight/Database/DataManager.m @@ -61,6 +61,7 @@ multiController:(BOOL)multiController audioOnPC:(BOOL)audioOnPC useHevc:(BOOL)useHevc + useFramePacing:(BOOL)useFramePacing enableHdr:(BOOL)enableHdr btMouseSupport:(BOOL)btMouseSupport absoluteTouchMode:(BOOL)absoluteTouchMode @@ -77,6 +78,7 @@ settingsToSave.multiController = multiController; settingsToSave.playAudioOnPC = audioOnPC; settingsToSave.useHevc = useHevc; + settingsToSave.useFramePacing = useFramePacing; settingsToSave.enableHdr = enableHdr; settingsToSave.btMouseSupport = btMouseSupport; settingsToSave.absoluteTouchMode = absoluteTouchMode; diff --git a/Limelight/Database/TemporarySettings.h b/Limelight/Database/TemporarySettings.h index cb9f2cb..822f30f 100644 --- a/Limelight/Database/TemporarySettings.h +++ b/Limelight/Database/TemporarySettings.h @@ -19,6 +19,7 @@ @property (nonatomic, retain) NSNumber * onscreenControls; @property (nonatomic, retain) NSString * uniqueId; @property (nonatomic) BOOL useHevc; +@property (nonatomic) BOOL useFramePacing; @property (nonatomic) BOOL multiController; @property (nonatomic) BOOL playAudioOnPC; @property (nonatomic) BOOL optimizeGames; diff --git a/Limelight/Database/TemporarySettings.m b/Limelight/Database/TemporarySettings.m index 0711971..42b2928 100644 --- a/Limelight/Database/TemporarySettings.m +++ b/Limelight/Database/TemporarySettings.m @@ -35,6 +35,7 @@ self.framerate = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"framerate"]]; assert([self.framerate intValue] != 0); self.useHevc = [[NSUserDefaults standardUserDefaults] boolForKey:@"useHevc"]; + self.useFramePacing = [[NSUserDefaults standardUserDefaults] boolForKey:@"useFramePacing"]; self.playAudioOnPC = [[NSUserDefaults standardUserDefaults] boolForKey:@"audioOnPC"]; self.enableHdr = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableHdr"]; self.optimizeGames = [[NSUserDefaults standardUserDefaults] boolForKey:@"optimizeGames"]; @@ -66,6 +67,7 @@ self.height = settings.height; self.width = settings.width; self.useHevc = settings.useHevc; + self.useFramePacing = settings.useFramePacing; self.playAudioOnPC = settings.playAudioOnPC; self.enableHdr = settings.enableHdr; self.optimizeGames = settings.optimizeGames; diff --git a/Limelight/Limelight.xcdatamodeld/.xccurrentversion b/Limelight/Limelight.xcdatamodeld/.xccurrentversion index c2a8b51..980be35 100644 --- a/Limelight/Limelight.xcdatamodeld/.xccurrentversion +++ b/Limelight/Limelight.xcdatamodeld/.xccurrentversion @@ -3,6 +3,6 @@ _XCCurrentVersionName - Moonlight v1.6.xcdatamodel + Moonlight v1.7.xcdatamodel diff --git a/Limelight/Limelight.xcdatamodeld/Moonlight v1.7.xcdatamodel/contents b/Limelight/Limelight.xcdatamodeld/Moonlight v1.7.xcdatamodel/contents new file mode 100644 index 0000000..4ab0e55 --- /dev/null +++ b/Limelight/Limelight.xcdatamodeld/Moonlight v1.7.xcdatamodel/contents @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 2129ca2..2c9d002 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -437,7 +437,6 @@ void ClConnectionStatusUpdate(int status) CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC | #endif CAPABILITY_PULL_RENDERER; - LiInitializeAudioCallbacks(&_arCallbacks); _arCallbacks.init = ArInit; _arCallbacks.cleanup = ArCleanup; diff --git a/Limelight/Stream/StreamConfiguration.h b/Limelight/Stream/StreamConfiguration.h index 1113a97..7a20759 100644 --- a/Limelight/Stream/StreamConfiguration.h +++ b/Limelight/Stream/StreamConfiguration.h @@ -27,6 +27,7 @@ @property BOOL enableHdr; @property BOOL multiController; @property BOOL allowHevc; +@property BOOL useFramePacing; @property NSData* serverCert; @end diff --git a/Limelight/Stream/StreamManager.m b/Limelight/Stream/StreamManager.m index 6437862..152e459 100644 --- a/Limelight/Stream/StreamManager.m +++ b/Limelight/Stream/StreamManager.m @@ -100,7 +100,7 @@ // Initializing the renderer must be done on the main thread dispatch_async(dispatch_get_main_queue(), ^{ - VideoDecoderRenderer* renderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks]; + VideoDecoderRenderer* renderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks useFramePacing:self->_config.useFramePacing]; self->_connection = [[Connection alloc] initWithConfig:self->_config renderer:renderer connectionCallbacks:self->_callbacks]; NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; [opQueue addOperation:self->_connection]; diff --git a/Limelight/Stream/VideoDecoderRenderer.h b/Limelight/Stream/VideoDecoderRenderer.h index 5a6fb4b..e27c6bf 100644 --- a/Limelight/Stream/VideoDecoderRenderer.h +++ b/Limelight/Stream/VideoDecoderRenderer.h @@ -12,7 +12,7 @@ @interface VideoDecoderRenderer : NSObject -- (id)initWithView:(UIView*)view callbacks:(id)callbacks; +- (id)initWithView:(UIView*)view callbacks:(id)callbacks useFramePacing:(BOOL)useFramePacing; - (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate; - (void)start; diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index bc553ac..4da8ff6 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -24,6 +24,7 @@ CMVideoFormatDescriptionRef formatDesc; CADisplayLink* _displayLink; + BOOL framePacing; } - (void)reinitializeDisplayLayer @@ -63,12 +64,13 @@ } } -- (id)initWithView:(StreamView*)view callbacks:(id)callbacks +- (id)initWithView:(StreamView*)view callbacks:(id)callbacks useFramePacing:(BOOL)useFramePacing { self = [super init]; _view = view; _callbacks = callbacks; + framePacing = useFramePacing; [self reinitializeDisplayLayer]; @@ -90,12 +92,6 @@ [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; } -- (void)stop -{ - [_displayLink invalidate]; -} - -// TODO: Refactor this int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); - (void)displayLinkCallback:(CADisplayLink *)sender @@ -105,9 +101,22 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); while (LiPollNextVideoFrame(&handle, &du)) { LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du)); + + if (framePacing){ + // Always keep one pending frame smooth out gaps due to + // network jitter at the cost of 1 frame of latency + if (LiGetPendingVideoFrames() == 1) { + break; + } + } } } +- (void)stop +{ + [_displayLink invalidate]; +} + #define FRAME_START_PREFIX_SIZE 4 #define NALU_START_PREFIX_SIZE 3 #define NAL_LENGTH_PREFIX_SIZE 4 @@ -350,14 +359,14 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); } - + // Enqueue the next frame [self->displayLayer enqueueSampleBuffer:sampleBuffer]; if (frameType == FRAME_TYPE_IDR) { // Ensure the layer is visible now self->displayLayer.hidden = NO; - + // Tell our parent VC to hide the progress indicator [self->_callbacks videoContentShown]; } @@ -365,7 +374,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); // Dereference the buffers CFRelease(blockBuffer); CFRelease(sampleBuffer); - + return DR_OK; } diff --git a/Limelight/ViewControllers/MainFrameViewController.m b/Limelight/ViewControllers/MainFrameViewController.m index 0921f76..222e62e 100644 --- a/Limelight/ViewControllers/MainFrameViewController.m +++ b/Limelight/ViewControllers/MainFrameViewController.m @@ -626,6 +626,7 @@ static NSMutableSet* hostList; _streamConfig.optimizeGameSettings = streamSettings.optimizeGames; _streamConfig.playAudioOnPC = streamSettings.playAudioOnPC; _streamConfig.allowHevc = streamSettings.useHevc; + _streamConfig.useFramePacing = streamSettings.useFramePacing; // multiController must be set before calling getConnectedGamepadMask _streamConfig.multiController = streamSettings.multiController; @@ -650,7 +651,7 @@ static NSMutableSet* hostList; // // It should also be a user preference, since some games may require higher peak // brightness than the iOS device can support to look correct in HDR mode. - if (@available(iOS 11.3, *)) { + if (@available(iOS 11.3, tvOS 11.2, *)) { _streamConfig.enableHdr = app.hdrSupported && // App supported (app.host.serverCodecModeSupport & 0x200) != 0 && // HEVC Main10 encoding on host PC GPU diff --git a/Limelight/ViewControllers/SettingsViewController.h b/Limelight/ViewControllers/SettingsViewController.h index 5e6b7a4..ad7736f 100644 --- a/Limelight/ViewControllers/SettingsViewController.h +++ b/Limelight/ViewControllers/SettingsViewController.h @@ -20,6 +20,7 @@ @property (strong, nonatomic) IBOutlet UISegmentedControl *multiControllerSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *audioOnPCSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *hevcSelector; +@property (strong, nonatomic) IBOutlet UISegmentedControl *framePacingSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *btMouseSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *statsOverlaySelector; @property (strong, nonatomic) IBOutlet UIScrollView *scrollView; diff --git a/Limelight/ViewControllers/SettingsViewController.m b/Limelight/ViewControllers/SettingsViewController.m index c31fbc5..9dafe78 100644 --- a/Limelight/ViewControllers/SettingsViewController.m +++ b/Limelight/ViewControllers/SettingsViewController.m @@ -198,6 +198,7 @@ static const int bitrateTable[] = { [self.statsOverlaySelector setSelectedSegmentIndex:currentSettings.statsOverlay ? 1 : 0]; [self.btMouseSelector setSelectedSegmentIndex:currentSettings.btMouseSupport ? 1 : 0]; [self.optimizeSettingsSelector setSelectedSegmentIndex:currentSettings.optimizeGames ? 1 : 0]; + [self.framePacingSelector setSelectedSegmentIndex:currentSettings.useFramePacing ? 1 : 0]; [self.multiControllerSelector setSelectedSegmentIndex:currentSettings.multiController ? 1 : 0]; [self.audioOnPCSelector setSelectedSegmentIndex:currentSettings.playAudioOnPC ? 1 : 0]; NSInteger onscreenControls = [currentSettings.onscreenControls integerValue]; @@ -303,6 +304,7 @@ static const int bitrateTable[] = { BOOL audioOnPC = [self.audioOnPCSelector selectedSegmentIndex] == 1; BOOL useHevc = [self.hevcSelector selectedSegmentIndex] == 1; BOOL btMouseSupport = [self.btMouseSelector selectedSegmentIndex] == 1; + BOOL useFramePacing = [self.framePacingSelector selectedSegmentIndex] == 1; BOOL absoluteTouchMode = [self.touchModeSelector selectedSegmentIndex] == 1; BOOL statsOverlay = [self.statsOverlaySelector selectedSegmentIndex] == 1; [dataMan saveSettingsWithBitrate:_bitrate @@ -314,6 +316,7 @@ static const int bitrateTable[] = { multiController:multiController audioOnPC:audioOnPC useHevc:useHevc + useFramePacing:useFramePacing enableHdr:NO btMouseSupport:btMouseSupport absoluteTouchMode:absoluteTouchMode diff --git a/Moonlight TV/Settings.bundle/Root.plist b/Moonlight TV/Settings.bundle/Root.plist index 49e2e79..4da754d 100644 --- a/Moonlight TV/Settings.bundle/Root.plist +++ b/Moonlight TV/Settings.bundle/Root.plist @@ -158,6 +158,16 @@ DefaultValue + + Type + PSToggleSwitchSpecifier + Title + Use Frame Pacing + Key + useFramePacing + DefaultValue + + Type PSToggleSwitchSpecifier diff --git a/Moonlight.xcodeproj/project.pbxproj b/Moonlight.xcodeproj/project.pbxproj index 9c55968..03eb33e 100644 --- a/Moonlight.xcodeproj/project.pbxproj +++ b/Moonlight.xcodeproj/project.pbxproj @@ -163,6 +163,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + 566E9D2B2770B23A00EF7BFE /* Moonlight v1.7.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = "Moonlight v1.7.xcdatamodel"; sourceTree = ""; }; 693B3A9A218638CD00982F7B /* Settings.bundle */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.plug-in"; path = Settings.bundle; sourceTree = ""; }; 9803CCAB254F9EAF00EE185E /* ConnectionCallbacks.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ConnectionCallbacks.h; sourceTree = ""; }; 98132E8C20BC9A62007A053F /* Moonlight v1.1.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = "Moonlight v1.1.xcdatamodel"; sourceTree = ""; }; @@ -1428,6 +1429,7 @@ FB290D0519B2C406004C83CF /* Limelight.xcdatamodeld */ = { isa = XCVersionGroup; children = ( + 566E9D2B2770B23A00EF7BFE /* Moonlight v1.7.xcdatamodel */, 9819CC25254F2734008A7C8E /* Moonlight v1.6.xcdatamodel */, 98783FEA242EAC5D00F00EF4 /* Moonlight v1.5.xcdatamodel */, 98608BDD22DC0C2C000E5672 /* Moonlight v1.4.xcdatamodel */, @@ -1440,7 +1442,7 @@ FB4678F21A51BDCB00377732 /* Limelight 0.3.0.xcdatamodel */, FB290D0619B2C406004C83CF /* Limelight.xcdatamodel */, ); - currentVersion = 9819CC25254F2734008A7C8E /* Moonlight v1.6.xcdatamodel */; + currentVersion = 566E9D2B2770B23A00EF7BFE /* Moonlight v1.7.xcdatamodel */; path = Limelight.xcdatamodeld; sourceTree = ""; versionGroupType = wrapper.xcdatamodel; diff --git a/iPad.storyboard b/iPad.storyboard index ca9a510..6304c44 100644 --- a/iPad.storyboard +++ b/iPad.storyboard @@ -12,7 +12,7 @@ - + @@ -219,14 +219,14 @@ -