moonlight-stream/moonlight-android
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-android.git synced 2025-07-24 21:42:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace frame pacing hack with Choreographer-based rendering

Browse Source 
This mimics the frame pacing logic now present in the iOS client.
This commit is contained in:
Cameron Gutman 2022-02-17 23:48:02 -06:00
parent c0f8001627
commit c25faf6426
7 changed files with 138 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
app/src/main/java/com/limelight/Game.java
View File

@ -389,58 +389,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
float displayRefreshRate = prepareDisplayForRendering();
LimeLog.info("Display refresh rate: "+displayRefreshRate);
// HACK: Despite many efforts to ensure low latency consistent frame
// delivery, the best non-lossy mechanism is to buffer 1 extra frame
// in the output pipeline. Android does some buffering on its end
// in SurfaceFlinger and it's difficult (impossible?) to inspect
// the precise state of the buffer queue to the screen after we
// release a frame for rendering.
//
// Since buffering a frame adds latency and we are primarily a
// latency-optimized client, rather than one designed for picture-perfect
// accuracy, we will synthetically induce a negative pressure on the display
// output pipeline by driving the decoder input pipeline under the speed
// that the display can refresh. This ensures a constant negative pressure
// to keep latency down but does induce a periodic frame loss. However, this
// periodic frame loss is *way* less than what we'd already get in Marshmallow's
// display pipeline where frames are dropped outside of our control if they land
// on the same V-sync.
//
// Hopefully, we can get rid of this once someone comes up with a better way
// to track the state of the pipeline and time frames.
int roundedRefreshRate = Math.round(displayRefreshRate);
int chosenFrameRate = prefConfig.fps;
if (!prefConfig.disableFrameDrop || prefConfig.unlockFps) {
if (Build.DEVICE.equals("coral") || Build.DEVICE.equals("flame")) {
// HACK: Pixel 4 (XL) ignores the preferred display mode and lowers refresh rate,
// causing frame pacing issues. See https://issuetracker.google.com/issues/143401475
// To work around this, use frame drop mode if we want to stream at >= 60 FPS.
if (prefConfig.fps >= 60) {
LimeLog.info("Using Pixel 4 rendering hack");
decoderRenderer.enableLegacyFrameDropRendering();
}
}
else if (prefConfig.fps >= roundedRefreshRate) {
if (prefConfig.unlockFps) {
// Use frame drops when rendering above the screen frame rate
decoderRenderer.enableLegacyFrameDropRendering();
LimeLog.info("Using drop mode for FPS > Hz");
} else if (roundedRefreshRate <= 49) {
// Let's avoid clearly bogus refresh rates and fall back to legacy rendering
decoderRenderer.enableLegacyFrameDropRendering();
LimeLog.info("Bogus refresh rate: " + roundedRefreshRate);
}
// HACK: Avoid crashing on some MTK devices
else if (decoderRenderer.isBlacklistedForFrameRate(roundedRefreshRate - 1)) {
// Use the old rendering strategy on these broken devices
decoderRenderer.enableLegacyFrameDropRendering();
} else {
chosenFrameRate = roundedRefreshRate - 1;
LimeLog.info("Adjusting FPS target for screen to " + chosenFrameRate);
}
}
}
boolean vpnActive = NetHelper.isActiveNetworkVpn(this);
if (vpnActive) {
LimeLog.info("Detected active network is a VPN");
@ -449,7 +397,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
StreamConfiguration config = new StreamConfiguration.Builder()
.setResolution(prefConfig.width, prefConfig.height)
.setLaunchRefreshRate(prefConfig.fps)
.setRefreshRate(chosenFrameRate)
.setRefreshRate(prefConfig.fps)
.setApp(new NvApp(appName != null ? appName : "app", appId, appSupportsHdr))
.setBitrate(prefConfig.bitrate)
.setEnableSops(prefConfig.enableSops)

118
app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java
View File

@ -2,6 +2,7 @@ package com.limelight.binding.video;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.LinkedBlockingQueue;
import org.jcodec.codecs.h264.H264Utils;
import org.jcodec.codecs.h264.io.model.SeqParameterSet;
@ -20,11 +21,14 @@ import android.media.MediaFormat;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaCodec.CodecException;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Range;
import android.view.Choreographer;
import android.view.SurfaceHolder;
public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements Choreographer.FrameCallback {
private static final boolean USE_FRAME_RENDER_TIME = false;
private static final boolean FRAME_RENDER_TIME_ONLY = USE_FRAME_RENDER_TIME && false;
@ -58,7 +62,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
private int consecutiveCrashCount;
private String glRenderer;
private boolean foreground = true;
private boolean legacyFrameDropRendering = false;
private PerfOverlayListener perfListener;
private MediaFormat inputFormat;
@ -81,6 +84,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
private int refreshRate;
private PreferenceConfiguration prefs;
private LinkedBlockingQueue<Integer> outputBufferQueue = new LinkedBlockingQueue<>();
private static final int OUTPUT_BUFFER_QUEUE_LIMIT = 2;
private int numSpsIn;
private int numPpsIn;
private int numVpsIn;
@ -198,15 +204,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
return avcDecoder != null;
}
public boolean isBlacklistedForFrameRate(int frameRate) {
return avcDecoder != null && MediaCodecHelper.decoderBlacklistedForFrameRate(avcDecoder.getName(), frameRate);
}
public void enableLegacyFrameDropRendering() {
LimeLog.info("Legacy frame drop rendering enabled");
legacyFrameDropRendering = true;
}
public boolean isHevcMain10Hdr10Supported() {
if (hevcDecoder == null) {
return false;
@ -310,10 +307,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
// Avoid setting KEY_FRAME_RATE on Lollipop and earlier to reduce compatibility risk
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
// We use prefs.fps instead of redrawRate here because the low latency hack in Game.java
// may leave us with an odd redrawRate value like 59 or 49 which might cause the decoder
// to puke. To be safe, we'll use the unmodified value.
videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, prefs.fps);
videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, redrawRate);
}
// Adaptive playback can also be enabled by the whitelist on pre-KitKat devices
@ -418,6 +412,34 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
}
}
@Override
public void doFrame(long frameTimeNanos) {
// Do nothing if we're stopping
if (stopping) {
return;
}
// Render up to one frame when in frame pacing mode.
//
// NB: Since the queue limit is 2, we won't starve the decoder of output buffers
// by holding onto them for too long. This also ensures we will have that 1 extra
// frame of buffer to smooth over network/rendering jitter.
Integer nextOutputBuffer = outputBufferQueue.poll();
if (nextOutputBuffer != null) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
videoDecoder.releaseOutputBuffer(nextOutputBuffer, frameTimeNanos);
}
else {
videoDecoder.releaseOutputBuffer(nextOutputBuffer, true);
}
activeWindowVideoStats.totalFramesRendered++;
}
// Request another callback for next frame
Choreographer.getInstance().postFrameCallback(this);
}
private void startRendererThread()
{
rendererThread = new Thread() {
@ -434,34 +456,45 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
numFramesOut++;
// Get the last output buffer in the queue
while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) {
videoDecoder.releaseOutputBuffer(lastIndex, false);
// Render the latest frame now if frame pacing is off
if (prefs.framePacing == PreferenceConfiguration.FRAME_PACING_OFF) {
// Get the last output buffer in the queue
while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) {
videoDecoder.releaseOutputBuffer(lastIndex, false);
numFramesOut++;
numFramesOut++;
lastIndex = outIndex;
presentationTimeUs = info.presentationTimeUs;
}
lastIndex = outIndex;
presentationTimeUs = info.presentationTimeUs;
}
// Render the last buffer
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
if (legacyFrameDropRendering) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
// Use a PTS that will cause this frame to be dropped if another comes in within
// the same V-sync period
videoDecoder.releaseOutputBuffer(lastIndex, System.nanoTime());
}
else {
// Use a PTS that will cause this frame to never be dropped if frame dropping
// is disabled
videoDecoder.releaseOutputBuffer(lastIndex, 0);
videoDecoder.releaseOutputBuffer(lastIndex, true);
}
activeWindowVideoStats.totalFramesRendered++;
}
else {
videoDecoder.releaseOutputBuffer(lastIndex, true);
}
// For the frame pacing case, the Choreographer callback will handle rendering.
// We just put all frames into the output buffer queue and let it handle things.
activeWindowVideoStats.totalFramesRendered++;
// Discard the oldest buffer if we've exceeded our limit.
//
// NB: We have to do this on the producer side because the consumer may not
// run for a while (if there is a huge mismatch between stream FPS and display
// refresh rate).
if (outputBufferQueue.size() == OUTPUT_BUFFER_QUEUE_LIMIT) {
videoDecoder.releaseOutputBuffer(outputBufferQueue.take(), false);
}
// Add this buffer
outputBufferQueue.add(lastIndex);
}
// Add delta time to the totals (excluding probable outliers)
long delta = MediaCodecHelper.getMonotonicMillis() - (presentationTimeUs / 1000);
@ -536,6 +569,18 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
@Override
public void start() {
startRendererThread();
// Start Choreographer callbacks for rendering with frame pacing enabled
// NB: This must be done on a thread with a looper!
if (prefs.framePacing != PreferenceConfiguration.FRAME_PACING_OFF) {
Handler h = new Handler(Looper.getMainLooper());
h.post(new Runnable() {
@Override
public void run() {
Choreographer.getInstance().postFrameCallback(MediaCodecDecoderRenderer.this);
}
});
}
}
// !!! May be called even if setup()/start() fails !!!
@ -547,6 +592,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
if (rendererThread != null) {
rendererThread.interrupt();
}
// Halt further Choreographer callbacks
if (prefs.framePacing != PreferenceConfiguration.FRAME_PACING_OFF) {
Handler h = new Handler(Looper.getMainLooper());
h.post(new Runnable() {
@Override
public void run() {
Choreographer.getInstance().removeFrameCallback(MediaCodecDecoderRenderer.this);
}
});
}
}
@Override

32
app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java
View File

@ -38,8 +38,6 @@ public class MediaCodecHelper {
private static final List<String> whitelistedHevcDecoders;
private static final List<String> refFrameInvalidationAvcPrefixes;
private static final List<String> refFrameInvalidationHevcPrefixes;
private static final List<String> blacklisted49FpsDecoderPrefixes;
private static final List<String> blacklisted59FpsDecoderPrefixes;
private static final List<String> qualcommDecoderPrefixes;
private static final List<String> kirinDecoderPrefixes;
private static final List<String> exynosDecoderPrefixes;
@ -194,24 +192,6 @@ public class MediaCodecHelper {
// Qualcomm is currently the only decoders in this group.
}
static {
blacklisted49FpsDecoderPrefixes = new LinkedList<>();
blacklisted59FpsDecoderPrefixes = new LinkedList<>();
// We see a bunch of crashes on MediaTek Android TVs running
// at 49 FPS (PAL 50 Hz - 1). Blacklist this frame rate for
// these devices and hope they fix it in Pie.
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
blacklisted49FpsDecoderPrefixes.add("omx.mtk");
// 59 FPS also seems to crash on the Sony Bravia TV ATV3 model.
// Blacklist that frame rate on these devices too.
if (Build.DEVICE.startsWith("BRAVIA_ATV3")) {
blacklisted59FpsDecoderPrefixes.add("omx.mtk");
}
}
}
static {
qualcommDecoderPrefixes = new LinkedList<>();
@ -486,18 +466,6 @@ public class MediaCodecHelper {
return isDecoderInList(baselineProfileHackPrefixes, decoderName);
}
public static boolean decoderBlacklistedForFrameRate(String decoderName, int fps) {
if (fps == 49) {
return isDecoderInList(blacklisted49FpsDecoderPrefixes, decoderName);
}
else if (fps == 59) {
return isDecoderInList(blacklisted59FpsDecoderPrefixes, decoderName);
}
else {
return false;
}
}
public static boolean decoderSupportsRefFrameInvalidationAvc(String decoderName, int videoHeight) {
// Reference frame invalidation is broken on low-end Snapdragon SoCs at 1080p.
if (videoHeight > 720 && isLowEndSnapdragon) {

33
app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java
View File

@ -30,7 +30,7 @@ public class PreferenceConfiguration {
private static final String VIDEO_FORMAT_PREF_STRING = "video_format";
private static final String ONSCREEN_CONTROLLER_PREF_STRING = "checkbox_show_onscreen_controls";
private static final String ONLY_L3_R3_PREF_STRING = "checkbox_only_show_L3R3";
private static final String DISABLE_FRAME_DROP_PREF_STRING = "checkbox_disable_frame_drop";
private static final String LEGACY_DISABLE_FRAME_DROP_PREF_STRING = "checkbox_disable_frame_drop";
private static final String ENABLE_HDR_PREF_STRING = "checkbox_enable_hdr";
private static final String ENABLE_PIP_PREF_STRING = "checkbox_enable_pip";
private static final String ENABLE_PERF_OVERLAY_STRING = "checkbox_enable_perf_overlay";
@ -43,6 +43,7 @@ public class PreferenceConfiguration {
private static final String FLIP_FACE_BUTTONS_PREF_STRING = "checkbox_flip_face_buttons";
private static final String TOUCHSCREEN_TRACKPAD_PREF_STRING = "checkbox_touchscreen_trackpad";
private static final String LATENCY_TOAST_PREF_STRING = "checkbox_enable_post_stream_toast";
private static final String FRAME_PACING_PREF_STRING = "frame_pacing";
static final String DEFAULT_RESOLUTION = "1280x720";
static final String DEFAULT_FPS = "60";
@ -58,7 +59,6 @@ public class PreferenceConfiguration {
private static final String DEFAULT_VIDEO_FORMAT = "auto";
private static final boolean ONSCREEN_CONTROLLER_DEFAULT = false;
private static final boolean ONLY_L3_R3_DEFAULT = false;
private static final boolean DEFAULT_DISABLE_FRAME_DROP = false;
private static final boolean DEFAULT_ENABLE_HDR = false;
private static final boolean DEFAULT_ENABLE_PIP = false;
private static final boolean DEFAULT_ENABLE_PERF_OVERLAY = false;
@ -72,11 +72,15 @@ public class PreferenceConfiguration {
private static final boolean DEFAULT_TOUCHSCREEN_TRACKPAD = true;
private static final String DEFAULT_AUDIO_CONFIG = "2"; // Stereo
private static final boolean DEFAULT_LATENCY_TOAST = false;
private static final String DEFAULT_FRAME_PACING = "latency";
public static final int FORCE_H265_ON = -1;
public static final int AUTOSELECT_H265 = 0;
public static final int FORCE_H265_OFF = 1;
public static final int FRAME_PACING_OFF = 0;
public static final int FRAME_PACING_ON = 1;
public static final String RES_360P = "640x360";
public static final String RES_480P = "854x480";
public static final String RES_720P = "1280x720";
@ -95,7 +99,6 @@ public class PreferenceConfiguration {
public boolean smallIconMode, multiController, usbDriver, flipFaceButtons;
public boolean onscreenController;
public boolean onlyL3R3;
public boolean disableFrameDrop;
public boolean enableHdr;
public boolean enablePip;
public boolean enablePerfOverlay;
@ -108,6 +111,7 @@ public class PreferenceConfiguration {
public boolean vibrateFallbackToDevice;
public boolean touchscreenTrackpad;
public MoonBridge.AudioConfiguration audioConfiguration;
public int framePacing;
public static boolean isNativeResolution(int width, int height) {
// It's not a native resolution if it matches an existing resolution option
@ -264,6 +268,27 @@ public class PreferenceConfiguration {
}
}
private static int getFramePacingValue(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
// Migrate legacy never drop frames option to the new location
if (prefs.contains(LEGACY_DISABLE_FRAME_DROP_PREF_STRING)) {
boolean legacyNeverDropFrames = prefs.getBoolean(LEGACY_DISABLE_FRAME_DROP_PREF_STRING, false);
prefs.edit()
.remove(LEGACY_DISABLE_FRAME_DROP_PREF_STRING)
.putString(FRAME_PACING_PREF_STRING, legacyNeverDropFrames ? "smoothness" : "latency")
.apply();
}
String str = prefs.getString(FRAME_PACING_PREF_STRING, DEFAULT_FRAME_PACING);
if (str.equals("latency")) {
return FRAME_PACING_OFF;
}
else {
return FRAME_PACING_ON;
}
}
public static void resetStreamingSettings(Context context) {
// We consider resolution, FPS, bitrate, HDR, and video format as "streaming settings" here
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
@ -387,6 +412,7 @@ public class PreferenceConfiguration {
}
config.videoFormat = getVideoFormatValue(context);
config.framePacing = getFramePacingValue(context);
config.deadzonePercentage = prefs.getInt(DEADZONE_PREF_STRING, DEFAULT_DEADZONE);
@ -404,7 +430,6 @@ public class PreferenceConfiguration {
config.usbDriver = prefs.getBoolean(USB_DRIVER_PREF_SRING, DEFAULT_USB_DRIVER);
config.onscreenController = prefs.getBoolean(ONSCREEN_CONTROLLER_PREF_STRING, ONSCREEN_CONTROLLER_DEFAULT);
config.onlyL3R3 = prefs.getBoolean(ONLY_L3_R3_PREF_STRING, ONLY_L3_R3_DEFAULT);
config.disableFrameDrop = prefs.getBoolean(DISABLE_FRAME_DROP_PREF_STRING, DEFAULT_DISABLE_FRAME_DROP);
config.enableHdr = prefs.getBoolean(ENABLE_HDR_PREF_STRING, DEFAULT_ENABLE_HDR);
config.enablePip = prefs.getBoolean(ENABLE_PIP_PREF_STRING, DEFAULT_ENABLE_PIP);
config.enablePerfOverlay = prefs.getBoolean(ENABLE_PERF_OVERLAY_STRING, DEFAULT_ENABLE_PERF_OVERLAY);

9
app/src/main/res/values/arrays.xml
View File

@ -94,4 +94,13 @@
<item>forceh265</item>
<item>neverh265</item>
</string-array>
<string-array name="video_frame_pacing_names">
<item>@string/pacing_latency</item>
<item>@string/pacing_smoothness</item>
</string-array>
<string-array name="video_frame_pacing_values" translatable="false">
<item>latency</item>
<item>smoothness</item>
</string-array>
</resources>

5
app/src/main/res/values/strings.xml
View File

@ -247,4 +247,9 @@
<string name="videoformat_hevcauto">Use HEVC only if stable</string>
<string name="videoformat_hevcalways">Always use HEVC (may crash)</string>
<string name="videoformat_hevcnever">Never use HEVC</string>
<string name="title_frame_pacing">Video frame pacing</string>
<string name="summary_frame_pacing">Specify how to balance video latency and smoothness</string>
<string name="pacing_latency">Prefer Lowest Latency</string>
<string name="pacing_smoothness">Prefer Smoothest Video</string>
</resources>

12
app/src/main/res/xml/preferences.xml
View File

@ -29,6 +29,13 @@
android:summary="@string/summary_seekbar_bitrate"
android:text="@string/suffix_seekbar_bitrate_mbps"
android:title="@string/title_seekbar_bitrate" />
<ListPreference
android:key="frame_pacing"
android:title="@string/title_frame_pacing"
android:entries="@array/video_frame_pacing_names"
android:entryValues="@array/video_frame_pacing_values"
android:summary="@string/summary_frame_pacing"
android:defaultValue="latency" />
<CheckBoxPreference
android:key="checkbox_stretch_video"
android:title="@string/title_checkbox_stretch_video"
@ -183,11 +190,6 @@
android:entryValues="@array/video_format_values"
android:summary="@string/summary_video_format"
android:defaultValue="auto" />
<CheckBoxPreference
android:key="checkbox_disable_frame_drop"
android:title="@string/title_disable_frame_drop"
android:summary="@string/summary_disable_frame_drop"
android:defaultValue="false" />
<CheckBoxPreference
android:key="checkbox_enable_hdr"
android:title="@string/title_enable_hdr"