diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 1410bd2b..efeae15a 100644 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -389,58 +389,6 @@ public class Game extends Activity implements SurfaceHolder.Callback, float displayRefreshRate = prepareDisplayForRendering(); LimeLog.info("Display refresh rate: "+displayRefreshRate); - // HACK: Despite many efforts to ensure low latency consistent frame - // delivery, the best non-lossy mechanism is to buffer 1 extra frame - // in the output pipeline. Android does some buffering on its end - // in SurfaceFlinger and it's difficult (impossible?) to inspect - // the precise state of the buffer queue to the screen after we - // release a frame for rendering. - // - // Since buffering a frame adds latency and we are primarily a - // latency-optimized client, rather than one designed for picture-perfect - // accuracy, we will synthetically induce a negative pressure on the display - // output pipeline by driving the decoder input pipeline under the speed - // that the display can refresh. This ensures a constant negative pressure - // to keep latency down but does induce a periodic frame loss. However, this - // periodic frame loss is *way* less than what we'd already get in Marshmallow's - // display pipeline where frames are dropped outside of our control if they land - // on the same V-sync. - // - // Hopefully, we can get rid of this once someone comes up with a better way - // to track the state of the pipeline and time frames. - int roundedRefreshRate = Math.round(displayRefreshRate); - int chosenFrameRate = prefConfig.fps; - if (!prefConfig.disableFrameDrop || prefConfig.unlockFps) { - if (Build.DEVICE.equals("coral") || Build.DEVICE.equals("flame")) { - // HACK: Pixel 4 (XL) ignores the preferred display mode and lowers refresh rate, - // causing frame pacing issues. See https://issuetracker.google.com/issues/143401475 - // To work around this, use frame drop mode if we want to stream at >= 60 FPS. - if (prefConfig.fps >= 60) { - LimeLog.info("Using Pixel 4 rendering hack"); - decoderRenderer.enableLegacyFrameDropRendering(); - } - } - else if (prefConfig.fps >= roundedRefreshRate) { - if (prefConfig.unlockFps) { - // Use frame drops when rendering above the screen frame rate - decoderRenderer.enableLegacyFrameDropRendering(); - LimeLog.info("Using drop mode for FPS > Hz"); - } else if (roundedRefreshRate <= 49) { - // Let's avoid clearly bogus refresh rates and fall back to legacy rendering - decoderRenderer.enableLegacyFrameDropRendering(); - LimeLog.info("Bogus refresh rate: " + roundedRefreshRate); - } - // HACK: Avoid crashing on some MTK devices - else if (decoderRenderer.isBlacklistedForFrameRate(roundedRefreshRate - 1)) { - // Use the old rendering strategy on these broken devices - decoderRenderer.enableLegacyFrameDropRendering(); - } else { - chosenFrameRate = roundedRefreshRate - 1; - LimeLog.info("Adjusting FPS target for screen to " + chosenFrameRate); - } - } - } - boolean vpnActive = NetHelper.isActiveNetworkVpn(this); if (vpnActive) { LimeLog.info("Detected active network is a VPN"); @@ -449,7 +397,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, StreamConfiguration config = new StreamConfiguration.Builder() .setResolution(prefConfig.width, prefConfig.height) .setLaunchRefreshRate(prefConfig.fps) - .setRefreshRate(chosenFrameRate) + .setRefreshRate(prefConfig.fps) .setApp(new NvApp(appName != null ? appName : "app", appId, appSupportsHdr)) .setBitrate(prefConfig.bitrate) .setEnableSops(prefConfig.enableSops) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 1933978e..7db02978 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -2,6 +2,7 @@ package com.limelight.binding.video; import java.nio.ByteBuffer; import java.util.Locale; +import java.util.concurrent.LinkedBlockingQueue; import org.jcodec.codecs.h264.H264Utils; import org.jcodec.codecs.h264.io.model.SeqParameterSet; @@ -20,11 +21,14 @@ import android.media.MediaFormat; import android.media.MediaCodec.BufferInfo; import android.media.MediaCodec.CodecException; import android.os.Build; +import android.os.Handler; +import android.os.Looper; import android.os.SystemClock; import android.util.Range; +import android.view.Choreographer; import android.view.SurfaceHolder; -public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { +public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements Choreographer.FrameCallback { private static final boolean USE_FRAME_RENDER_TIME = false; private static final boolean FRAME_RENDER_TIME_ONLY = USE_FRAME_RENDER_TIME && false; @@ -58,7 +62,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private int consecutiveCrashCount; private String glRenderer; private boolean foreground = true; - private boolean legacyFrameDropRendering = false; private PerfOverlayListener perfListener; private MediaFormat inputFormat; @@ -81,6 +84,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private int refreshRate; private PreferenceConfiguration prefs; + private LinkedBlockingQueue outputBufferQueue = new LinkedBlockingQueue<>(); + private static final int OUTPUT_BUFFER_QUEUE_LIMIT = 2; + private int numSpsIn; private int numPpsIn; private int numVpsIn; @@ -198,15 +204,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { return avcDecoder != null; } - public boolean isBlacklistedForFrameRate(int frameRate) { - return avcDecoder != null && MediaCodecHelper.decoderBlacklistedForFrameRate(avcDecoder.getName(), frameRate); - } - - public void enableLegacyFrameDropRendering() { - LimeLog.info("Legacy frame drop rendering enabled"); - legacyFrameDropRendering = true; - } - public boolean isHevcMain10Hdr10Supported() { if (hevcDecoder == null) { return false; @@ -310,10 +307,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { // Avoid setting KEY_FRAME_RATE on Lollipop and earlier to reduce compatibility risk if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // We use prefs.fps instead of redrawRate here because the low latency hack in Game.java - // may leave us with an odd redrawRate value like 59 or 49 which might cause the decoder - // to puke. To be safe, we'll use the unmodified value. - videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, prefs.fps); + videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, redrawRate); } // Adaptive playback can also be enabled by the whitelist on pre-KitKat devices @@ -418,6 +412,34 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { } } + @Override + public void doFrame(long frameTimeNanos) { + // Do nothing if we're stopping + if (stopping) { + return; + } + + // Render up to one frame when in frame pacing mode. + // + // NB: Since the queue limit is 2, we won't starve the decoder of output buffers + // by holding onto them for too long. This also ensures we will have that 1 extra + // frame of buffer to smooth over network/rendering jitter. + Integer nextOutputBuffer = outputBufferQueue.poll(); + if (nextOutputBuffer != null) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + videoDecoder.releaseOutputBuffer(nextOutputBuffer, frameTimeNanos); + } + else { + videoDecoder.releaseOutputBuffer(nextOutputBuffer, true); + } + + activeWindowVideoStats.totalFramesRendered++; + } + + // Request another callback for next frame + Choreographer.getInstance().postFrameCallback(this); + } + private void startRendererThread() { rendererThread = new Thread() { @@ -434,34 +456,45 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { numFramesOut++; - // Get the last output buffer in the queue - while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) { - videoDecoder.releaseOutputBuffer(lastIndex, false); + // Render the latest frame now if frame pacing is off + if (prefs.framePacing == PreferenceConfiguration.FRAME_PACING_OFF) { + // Get the last output buffer in the queue + while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) { + videoDecoder.releaseOutputBuffer(lastIndex, false); - numFramesOut++; + numFramesOut++; - lastIndex = outIndex; - presentationTimeUs = info.presentationTimeUs; - } + lastIndex = outIndex; + presentationTimeUs = info.presentationTimeUs; + } - // Render the last buffer - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - if (legacyFrameDropRendering) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { // Use a PTS that will cause this frame to be dropped if another comes in within // the same V-sync period videoDecoder.releaseOutputBuffer(lastIndex, System.nanoTime()); } else { - // Use a PTS that will cause this frame to never be dropped if frame dropping - // is disabled - videoDecoder.releaseOutputBuffer(lastIndex, 0); + videoDecoder.releaseOutputBuffer(lastIndex, true); } + + activeWindowVideoStats.totalFramesRendered++; } else { - videoDecoder.releaseOutputBuffer(lastIndex, true); - } + // For the frame pacing case, the Choreographer callback will handle rendering. + // We just put all frames into the output buffer queue and let it handle things. - activeWindowVideoStats.totalFramesRendered++; + // Discard the oldest buffer if we've exceeded our limit. + // + // NB: We have to do this on the producer side because the consumer may not + // run for a while (if there is a huge mismatch between stream FPS and display + // refresh rate). + if (outputBufferQueue.size() == OUTPUT_BUFFER_QUEUE_LIMIT) { + videoDecoder.releaseOutputBuffer(outputBufferQueue.take(), false); + } + + // Add this buffer + outputBufferQueue.add(lastIndex); + } // Add delta time to the totals (excluding probable outliers) long delta = MediaCodecHelper.getMonotonicMillis() - (presentationTimeUs / 1000); @@ -536,6 +569,18 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { @Override public void start() { startRendererThread(); + + // Start Choreographer callbacks for rendering with frame pacing enabled + // NB: This must be done on a thread with a looper! + if (prefs.framePacing != PreferenceConfiguration.FRAME_PACING_OFF) { + Handler h = new Handler(Looper.getMainLooper()); + h.post(new Runnable() { + @Override + public void run() { + Choreographer.getInstance().postFrameCallback(MediaCodecDecoderRenderer.this); + } + }); + } } // !!! May be called even if setup()/start() fails !!! @@ -547,6 +592,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { if (rendererThread != null) { rendererThread.interrupt(); } + + // Halt further Choreographer callbacks + if (prefs.framePacing != PreferenceConfiguration.FRAME_PACING_OFF) { + Handler h = new Handler(Looper.getMainLooper()); + h.post(new Runnable() { + @Override + public void run() { + Choreographer.getInstance().removeFrameCallback(MediaCodecDecoderRenderer.this); + } + }); + } } @Override diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java index e4e3e053..b76e1147 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -38,8 +38,6 @@ public class MediaCodecHelper { private static final List whitelistedHevcDecoders; private static final List refFrameInvalidationAvcPrefixes; private static final List refFrameInvalidationHevcPrefixes; - private static final List blacklisted49FpsDecoderPrefixes; - private static final List blacklisted59FpsDecoderPrefixes; private static final List qualcommDecoderPrefixes; private static final List kirinDecoderPrefixes; private static final List exynosDecoderPrefixes; @@ -194,24 +192,6 @@ public class MediaCodecHelper { // Qualcomm is currently the only decoders in this group. } - static { - blacklisted49FpsDecoderPrefixes = new LinkedList<>(); - blacklisted59FpsDecoderPrefixes = new LinkedList<>(); - - // We see a bunch of crashes on MediaTek Android TVs running - // at 49 FPS (PAL 50 Hz - 1). Blacklist this frame rate for - // these devices and hope they fix it in Pie. - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { - blacklisted49FpsDecoderPrefixes.add("omx.mtk"); - - // 59 FPS also seems to crash on the Sony Bravia TV ATV3 model. - // Blacklist that frame rate on these devices too. - if (Build.DEVICE.startsWith("BRAVIA_ATV3")) { - blacklisted59FpsDecoderPrefixes.add("omx.mtk"); - } - } - } - static { qualcommDecoderPrefixes = new LinkedList<>(); @@ -486,18 +466,6 @@ public class MediaCodecHelper { return isDecoderInList(baselineProfileHackPrefixes, decoderName); } - public static boolean decoderBlacklistedForFrameRate(String decoderName, int fps) { - if (fps == 49) { - return isDecoderInList(blacklisted49FpsDecoderPrefixes, decoderName); - } - else if (fps == 59) { - return isDecoderInList(blacklisted59FpsDecoderPrefixes, decoderName); - } - else { - return false; - } - } - public static boolean decoderSupportsRefFrameInvalidationAvc(String decoderName, int videoHeight) { // Reference frame invalidation is broken on low-end Snapdragon SoCs at 1080p. if (videoHeight > 720 && isLowEndSnapdragon) { diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index 87b96ce0..93111e91 100644 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -30,7 +30,7 @@ public class PreferenceConfiguration { private static final String VIDEO_FORMAT_PREF_STRING = "video_format"; private static final String ONSCREEN_CONTROLLER_PREF_STRING = "checkbox_show_onscreen_controls"; private static final String ONLY_L3_R3_PREF_STRING = "checkbox_only_show_L3R3"; - private static final String DISABLE_FRAME_DROP_PREF_STRING = "checkbox_disable_frame_drop"; + private static final String LEGACY_DISABLE_FRAME_DROP_PREF_STRING = "checkbox_disable_frame_drop"; private static final String ENABLE_HDR_PREF_STRING = "checkbox_enable_hdr"; private static final String ENABLE_PIP_PREF_STRING = "checkbox_enable_pip"; private static final String ENABLE_PERF_OVERLAY_STRING = "checkbox_enable_perf_overlay"; @@ -43,6 +43,7 @@ public class PreferenceConfiguration { private static final String FLIP_FACE_BUTTONS_PREF_STRING = "checkbox_flip_face_buttons"; private static final String TOUCHSCREEN_TRACKPAD_PREF_STRING = "checkbox_touchscreen_trackpad"; private static final String LATENCY_TOAST_PREF_STRING = "checkbox_enable_post_stream_toast"; + private static final String FRAME_PACING_PREF_STRING = "frame_pacing"; static final String DEFAULT_RESOLUTION = "1280x720"; static final String DEFAULT_FPS = "60"; @@ -58,7 +59,6 @@ public class PreferenceConfiguration { private static final String DEFAULT_VIDEO_FORMAT = "auto"; private static final boolean ONSCREEN_CONTROLLER_DEFAULT = false; private static final boolean ONLY_L3_R3_DEFAULT = false; - private static final boolean DEFAULT_DISABLE_FRAME_DROP = false; private static final boolean DEFAULT_ENABLE_HDR = false; private static final boolean DEFAULT_ENABLE_PIP = false; private static final boolean DEFAULT_ENABLE_PERF_OVERLAY = false; @@ -72,11 +72,15 @@ public class PreferenceConfiguration { private static final boolean DEFAULT_TOUCHSCREEN_TRACKPAD = true; private static final String DEFAULT_AUDIO_CONFIG = "2"; // Stereo private static final boolean DEFAULT_LATENCY_TOAST = false; + private static final String DEFAULT_FRAME_PACING = "latency"; public static final int FORCE_H265_ON = -1; public static final int AUTOSELECT_H265 = 0; public static final int FORCE_H265_OFF = 1; + public static final int FRAME_PACING_OFF = 0; + public static final int FRAME_PACING_ON = 1; + public static final String RES_360P = "640x360"; public static final String RES_480P = "854x480"; public static final String RES_720P = "1280x720"; @@ -95,7 +99,6 @@ public class PreferenceConfiguration { public boolean smallIconMode, multiController, usbDriver, flipFaceButtons; public boolean onscreenController; public boolean onlyL3R3; - public boolean disableFrameDrop; public boolean enableHdr; public boolean enablePip; public boolean enablePerfOverlay; @@ -108,6 +111,7 @@ public class PreferenceConfiguration { public boolean vibrateFallbackToDevice; public boolean touchscreenTrackpad; public MoonBridge.AudioConfiguration audioConfiguration; + public int framePacing; public static boolean isNativeResolution(int width, int height) { // It's not a native resolution if it matches an existing resolution option @@ -264,6 +268,27 @@ public class PreferenceConfiguration { } } + private static int getFramePacingValue(Context context) { + SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + + // Migrate legacy never drop frames option to the new location + if (prefs.contains(LEGACY_DISABLE_FRAME_DROP_PREF_STRING)) { + boolean legacyNeverDropFrames = prefs.getBoolean(LEGACY_DISABLE_FRAME_DROP_PREF_STRING, false); + prefs.edit() + .remove(LEGACY_DISABLE_FRAME_DROP_PREF_STRING) + .putString(FRAME_PACING_PREF_STRING, legacyNeverDropFrames ? "smoothness" : "latency") + .apply(); + } + + String str = prefs.getString(FRAME_PACING_PREF_STRING, DEFAULT_FRAME_PACING); + if (str.equals("latency")) { + return FRAME_PACING_OFF; + } + else { + return FRAME_PACING_ON; + } + } + public static void resetStreamingSettings(Context context) { // We consider resolution, FPS, bitrate, HDR, and video format as "streaming settings" here SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); @@ -387,6 +412,7 @@ public class PreferenceConfiguration { } config.videoFormat = getVideoFormatValue(context); + config.framePacing = getFramePacingValue(context); config.deadzonePercentage = prefs.getInt(DEADZONE_PREF_STRING, DEFAULT_DEADZONE); @@ -404,7 +430,6 @@ public class PreferenceConfiguration { config.usbDriver = prefs.getBoolean(USB_DRIVER_PREF_SRING, DEFAULT_USB_DRIVER); config.onscreenController = prefs.getBoolean(ONSCREEN_CONTROLLER_PREF_STRING, ONSCREEN_CONTROLLER_DEFAULT); config.onlyL3R3 = prefs.getBoolean(ONLY_L3_R3_PREF_STRING, ONLY_L3_R3_DEFAULT); - config.disableFrameDrop = prefs.getBoolean(DISABLE_FRAME_DROP_PREF_STRING, DEFAULT_DISABLE_FRAME_DROP); config.enableHdr = prefs.getBoolean(ENABLE_HDR_PREF_STRING, DEFAULT_ENABLE_HDR); config.enablePip = prefs.getBoolean(ENABLE_PIP_PREF_STRING, DEFAULT_ENABLE_PIP); config.enablePerfOverlay = prefs.getBoolean(ENABLE_PERF_OVERLAY_STRING, DEFAULT_ENABLE_PERF_OVERLAY); diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index c07ea5f9..fe245807 100644 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -94,4 +94,13 @@ forceh265 neverh265 + + + @string/pacing_latency + @string/pacing_smoothness + + + latency + smoothness + diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index e84000ce..99c67d9f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -247,4 +247,9 @@ Use HEVC only if stable Always use HEVC (may crash) Never use HEVC + + Video frame pacing + Specify how to balance video latency and smoothness + Prefer Lowest Latency + Prefer Smoothest Video diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 48d36804..f79d91f2 100644 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -29,6 +29,13 @@ android:summary="@string/summary_seekbar_bitrate" android:text="@string/suffix_seekbar_bitrate_mbps" android:title="@string/title_seekbar_bitrate" /> + -