moonlight-stream/moonlight-android
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-android.git synced 2025-07-26 06:22:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add additional vendor-specific low latency options for Qualcomm and HiSilicon SoCs

Browse Source
This commit is contained in:
Cameron Gutman 2021-04-18 12:49:25 -05:00
parent fe704af62f
commit 699cc361a2
2 changed files with 62 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java
View File

@ -46,7 +46,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
private Thread rendererThread; private Thread rendererThread;
private boolean needsSpsBitstreamFixup, isExynos4; private boolean needsSpsBitstreamFixup, isExynos4;
private boolean adaptivePlayback, directSubmit; private boolean adaptivePlayback, directSubmit;
private boolean lowLatency;
private boolean constrainedHighProfile; private boolean constrainedHighProfile;
private boolean refFrameInvalidationAvc, refFrameInvalidationHevc; private boolean refFrameInvalidationAvc, refFrameInvalidationHevc;
private boolean refFrameInvalidationActive; private boolean refFrameInvalidationActive;
@ -241,11 +240,11 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
this.refreshRate = redrawRate; this.refreshRate = redrawRate;
String mimeType; String mimeType;
String selectedDecoderName; MediaCodecInfo selectedDecoderInfo;
if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H264) != 0) { if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H264) != 0) {
mimeType = "video/avc"; mimeType = "video/avc";
selectedDecoderName = avcDecoder.getName(); selectedDecoderInfo = avcDecoder;
if (avcDecoder == null) { if (avcDecoder == null) {
LimeLog.severe("No available AVC decoder!"); LimeLog.severe("No available AVC decoder!");
@ -258,31 +257,28 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
} }
// These fixups only apply to H264 decoders // These fixups only apply to H264 decoders
needsSpsBitstreamFixup = MediaCodecHelper.decoderNeedsSpsBitstreamRestrictions(selectedDecoderName); needsSpsBitstreamFixup = MediaCodecHelper.decoderNeedsSpsBitstreamRestrictions(selectedDecoderInfo.getName());
needsBaselineSpsHack = MediaCodecHelper.decoderNeedsBaselineSpsHack(selectedDecoderName); needsBaselineSpsHack = MediaCodecHelper.decoderNeedsBaselineSpsHack(selectedDecoderInfo.getName());
constrainedHighProfile = MediaCodecHelper.decoderNeedsConstrainedHighProfile(selectedDecoderName); constrainedHighProfile = MediaCodecHelper.decoderNeedsConstrainedHighProfile(selectedDecoderInfo.getName());
isExynos4 = MediaCodecHelper.isExynos4Device(); isExynos4 = MediaCodecHelper.isExynos4Device();
if (needsSpsBitstreamFixup) { if (needsSpsBitstreamFixup) {
LimeLog.info("Decoder "+selectedDecoderName+" needs SPS bitstream restrictions fixup"); LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs SPS bitstream restrictions fixup");
} }
if (needsBaselineSpsHack) { if (needsBaselineSpsHack) {
LimeLog.info("Decoder "+selectedDecoderName+" needs baseline SPS hack"); LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs baseline SPS hack");
} }
if (constrainedHighProfile) { if (constrainedHighProfile) {
LimeLog.info("Decoder "+selectedDecoderName+" needs constrained high profile"); LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs constrained high profile");
} }
if (isExynos4) { if (isExynos4) {
LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4"); LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" is on Exynos 4");
} }
refFrameInvalidationActive = refFrameInvalidationAvc; refFrameInvalidationActive = refFrameInvalidationAvc;
lowLatency = MediaCodecHelper.decoderSupportsLowLatency(avcDecoder, mimeType);
adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder, mimeType);
} }
else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) { else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) {
mimeType = "video/hevc"; mimeType = "video/hevc";
selectedDecoderName = hevcDecoder.getName(); selectedDecoderInfo = hevcDecoder;
if (hevcDecoder == null) { if (hevcDecoder == null) {
LimeLog.severe("No available HEVC decoder!"); LimeLog.severe("No available HEVC decoder!");
@ -290,9 +286,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
} }
refFrameInvalidationActive = refFrameInvalidationHevc; refFrameInvalidationActive = refFrameInvalidationHevc;
lowLatency = MediaCodecHelper.decoderSupportsLowLatency(hevcDecoder, mimeType);
adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(hevcDecoder, mimeType);
} }
else { else {
// Unknown format // Unknown format
@ -300,10 +293,12 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
return -3; return -3;
} }
adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(selectedDecoderInfo, mimeType);
// Codecs have been known to throw all sorts of crazy runtime exceptions // Codecs have been known to throw all sorts of crazy runtime exceptions
// due to implementation problems // due to implementation problems
try { try {
videoDecoder = MediaCodec.createByCodecName(selectedDecoderName); videoDecoder = MediaCodec.createByCodecName(selectedDecoderInfo.getName());
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return -4; return -4;
@ -326,26 +321,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
videoFormat.setInteger(MediaFormat.KEY_MAX_HEIGHT, height); videoFormat.setInteger(MediaFormat.KEY_MAX_HEIGHT, height);
} }
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && lowLatency) { MediaCodecHelper.setDecoderLowLatencyOptions(videoFormat, selectedDecoderInfo, mimeType);
videoFormat.setInteger(MediaFormat.KEY_LOW_LATENCY, 1);
}
else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
// Set the Qualcomm vendor low latency extension if the Android R option is unavailable
if (MediaCodecHelper.decoderSupportsQcomVendorLowLatency(selectedDecoderName)) {
// MediaCodec supports vendor-defined format keys using the "vendor.<extension name>.<parameter name>" syntax.
// These allow access to functionality that is not exposed through documented MediaFormat.KEY_* values.
// https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/common/inc/vidc_vendor_extensions.h;l=67
//
// Examples of Qualcomm's vendor extensions for Snapdragon 845:
// https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp
// https://cs.android.com/android/_/android/platform/hardware/qcom/sm8150/media/+/0621ceb1c1b19564999db8293574a0e12952ff6c
videoFormat.setInteger("vendor.qti-ext-dec-low-latency.enable", 1);
}
if (MediaCodecHelper.decoderSupportsMaxOperatingRate(selectedDecoderName)) {
videoFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE);
}
}
configuredFormat = videoFormat; configuredFormat = videoFormat;
LimeLog.info("Configuring with format: "+configuredFormat); LimeLog.info("Configuring with format: "+configuredFormat);
@ -375,7 +351,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
}, null); }, null);
} }
LimeLog.info("Using codec "+selectedDecoderName+" for hardware decoding "+mimeType); LimeLog.info("Using codec "+selectedDecoderInfo.getName()+" for hardware decoding "+mimeType);
// Start the decoder // Start the decoder
videoDecoder.start(); videoDecoder.start();
@ -1103,7 +1079,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
str += "Consecutive crashes: "+renderer.consecutiveCrashCount+"\n"; str += "Consecutive crashes: "+renderer.consecutiveCrashCount+"\n";
str += "RFI active: "+renderer.refFrameInvalidationActive+"\n"; str += "RFI active: "+renderer.refFrameInvalidationActive+"\n";
str += "Using modern SPS patching: "+(Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)+"\n"; str += "Using modern SPS patching: "+(Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)+"\n";
str += "Low latency mode: "+renderer.lowLatency+"\n";
str += "Video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n"; str += "Video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n";
str += "FPS target: "+renderer.refreshRate+"\n"; str += "FPS target: "+renderer.refreshRate+"\n";
str += "Bitrate: "+renderer.prefs.bitrate+" Kbps \n"; str += "Bitrate: "+renderer.prefs.bitrate+" Kbps \n";

56
app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java
View File

@ -18,6 +18,7 @@ import android.media.MediaCodecInfo;
import android.media.MediaCodecList; import android.media.MediaCodecList;
import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCodecInfo.CodecProfileLevel; import android.media.MediaCodecInfo.CodecProfileLevel;
import android.media.MediaFormat;
import android.os.Build; import android.os.Build;
import com.limelight.LimeLog; import com.limelight.LimeLog;
@ -39,6 +40,7 @@ public class MediaCodecHelper {
private static final List<String> blacklisted49FpsDecoderPrefixes; private static final List<String> blacklisted49FpsDecoderPrefixes;
private static final List<String> blacklisted59FpsDecoderPrefixes; private static final List<String> blacklisted59FpsDecoderPrefixes;
private static final List<String> qualcommDecoderPrefixes; private static final List<String> qualcommDecoderPrefixes;
private static final List<String> kirinDecoderPrefixes;
public static final boolean IS_EMULATOR = Build.HARDWARE.equals("ranchu") || Build.HARDWARE.equals("cheets"); public static final boolean IS_EMULATOR = Build.HARDWARE.equals("ranchu") || Build.HARDWARE.equals("cheets");
@ -206,6 +208,12 @@ public class MediaCodecHelper {
qualcommDecoderPrefixes.add("c2.qti"); qualcommDecoderPrefixes.add("c2.qti");
} }
static {
kirinDecoderPrefixes = new LinkedList<>();
kirinDecoderPrefixes.add("omx.hisi");
}
private static boolean isPowerVR(String glRenderer) { private static boolean isPowerVR(String glRenderer) {
return glRenderer.toLowerCase().contains("powervr"); return glRenderer.toLowerCase().contains("powervr");
} }
@ -352,7 +360,7 @@ public class MediaCodecHelper {
return System.nanoTime() / 1000000L; return System.nanoTime() / 1000000L;
} }
public static boolean decoderSupportsLowLatency(MediaCodecInfo decoderInfo, String mimeType) { private static boolean decoderSupportsAndroidRLowLatency(MediaCodecInfo decoderInfo, String mimeType) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try { try {
if (decoderInfo.getCapabilitiesForType(mimeType).isFeatureSupported(CodecCapabilities.FEATURE_LowLatency)) { if (decoderInfo.getCapabilitiesForType(mimeType).isFeatureSupported(CodecCapabilities.FEATURE_LowLatency)) {
@ -368,7 +376,7 @@ public class MediaCodecHelper {
return false; return false;
} }
public static boolean decoderSupportsMaxOperatingRate(String decoderName) { private static boolean decoderSupportsMaxOperatingRate(String decoderName) {
// Operate at maximum rate to lower latency as much as possible on // Operate at maximum rate to lower latency as much as possible on
// some Qualcomm platforms. We could also set KEY_PRIORITY to 0 (realtime) // some Qualcomm platforms. We could also set KEY_PRIORITY to 0 (realtime)
// but that will actually result in the decoder crashing if it can't satisfy // but that will actually result in the decoder crashing if it can't satisfy
@ -383,6 +391,43 @@ public class MediaCodecHelper {
!isAdreno620; !isAdreno620;
} }
public static void setDecoderLowLatencyOptions(MediaFormat videoFormat, MediaCodecInfo decoderInfo, String mimeType) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && decoderSupportsAndroidRLowLatency(decoderInfo, mimeType)) {
videoFormat.setInteger(MediaFormat.KEY_LOW_LATENCY, 1);
}
else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
// MediaCodec supports vendor-defined format keys using the "vendor.<extension name>.<parameter name>" syntax.
// These allow access to functionality that is not exposed through documented MediaFormat.KEY_* values.
// https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/common/inc/vidc_vendor_extensions.h;l=67
//
// MediaCodec vendor extension support was introduced in Android 8.0:
// https://cs.android.com/android/_/android/platform/frameworks/av/+/01c10f8cdcd58d1e7025f426a72e6e75ba5d7fc2
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
// Try vendor-specific low latency options
if (isDecoderInList(qualcommDecoderPrefixes, decoderInfo.getName())) {
// Examples of Qualcomm's vendor extensions for Snapdragon 845:
// https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp
// https://cs.android.com/android/_/android/platform/hardware/qcom/sm8150/media/+/0621ceb1c1b19564999db8293574a0e12952ff6c
videoFormat.setInteger("vendor.qti-ext-dec-low-latency.enable", 1);
}
else if (isDecoderInList(kirinDecoderPrefixes, decoderInfo.getName())) {
// Kirin low latency options
// https://developer.huawei.com/consumer/cn/forum/topic/0202325564295980115
videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1);
videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-rdy", -1);
}
}
else if (isDecoderInList(qualcommDecoderPrefixes, decoderInfo.getName())) {
// This is an older low latency option used on some Qualcomm devices
videoFormat.setInteger("vt-low-latency", 1);
}
if (MediaCodecHelper.decoderSupportsMaxOperatingRate(decoderInfo.getName())) {
videoFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE);
}
}
}
public static boolean decoderSupportsAdaptivePlayback(MediaCodecInfo decoderInfo, String mimeType) { public static boolean decoderSupportsAdaptivePlayback(MediaCodecInfo decoderInfo, String mimeType) {
// Possibly enable adaptive playback on KitKat and above // Possibly enable adaptive playback on KitKat and above
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
@ -408,13 +453,6 @@ public class MediaCodecHelper {
return false; return false;
} }
public static boolean decoderSupportsQcomVendorLowLatency(String decoderName) {
// MediaCodec vendor extension support was introduced in Android 8.0:
// https://cs.android.com/android/_/android/platform/frameworks/av/+/01c10f8cdcd58d1e7025f426a72e6e75ba5d7fc2
return Build.VERSION.SDK_INT >= Build.VERSION_CODES.O &&
isDecoderInList(qualcommDecoderPrefixes, decoderName);
}
public static boolean decoderNeedsConstrainedHighProfile(String decoderName) { public static boolean decoderNeedsConstrainedHighProfile(String decoderName) {
return isDecoderInList(constrainedHighProfilePrefixes, decoderName); return isDecoderInList(constrainedHighProfilePrefixes, decoderName);
} }