From 699cc361a265f613b63631eb7678e8ab49ac16b0 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sun, 18 Apr 2021 12:49:25 -0500 Subject: [PATCH] Add additional vendor-specific low latency options for Qualcomm and HiSilicon SoCs --- .../video/MediaCodecDecoderRenderer.java | 55 +++++------------- .../binding/video/MediaCodecHelper.java | 56 ++++++++++++++++--- 2 files changed, 62 insertions(+), 49 deletions(-) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 53a662b5..b778ceba 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -46,7 +46,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private Thread rendererThread; private boolean needsSpsBitstreamFixup, isExynos4; private boolean adaptivePlayback, directSubmit; - private boolean lowLatency; private boolean constrainedHighProfile; private boolean refFrameInvalidationAvc, refFrameInvalidationHevc; private boolean refFrameInvalidationActive; @@ -241,11 +240,11 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { this.refreshRate = redrawRate; String mimeType; - String selectedDecoderName; + MediaCodecInfo selectedDecoderInfo; if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H264) != 0) { mimeType = "video/avc"; - selectedDecoderName = avcDecoder.getName(); + selectedDecoderInfo = avcDecoder; if (avcDecoder == null) { LimeLog.severe("No available AVC decoder!"); @@ -258,31 +257,28 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { } // These fixups only apply to H264 decoders - needsSpsBitstreamFixup = MediaCodecHelper.decoderNeedsSpsBitstreamRestrictions(selectedDecoderName); - needsBaselineSpsHack = MediaCodecHelper.decoderNeedsBaselineSpsHack(selectedDecoderName); - constrainedHighProfile = MediaCodecHelper.decoderNeedsConstrainedHighProfile(selectedDecoderName); + needsSpsBitstreamFixup = MediaCodecHelper.decoderNeedsSpsBitstreamRestrictions(selectedDecoderInfo.getName()); + needsBaselineSpsHack = MediaCodecHelper.decoderNeedsBaselineSpsHack(selectedDecoderInfo.getName()); + constrainedHighProfile = MediaCodecHelper.decoderNeedsConstrainedHighProfile(selectedDecoderInfo.getName()); isExynos4 = MediaCodecHelper.isExynos4Device(); if (needsSpsBitstreamFixup) { - LimeLog.info("Decoder "+selectedDecoderName+" needs SPS bitstream restrictions fixup"); + LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs SPS bitstream restrictions fixup"); } if (needsBaselineSpsHack) { - LimeLog.info("Decoder "+selectedDecoderName+" needs baseline SPS hack"); + LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs baseline SPS hack"); } if (constrainedHighProfile) { - LimeLog.info("Decoder "+selectedDecoderName+" needs constrained high profile"); + LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" needs constrained high profile"); } if (isExynos4) { - LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4"); + LimeLog.info("Decoder "+selectedDecoderInfo.getName()+" is on Exynos 4"); } refFrameInvalidationActive = refFrameInvalidationAvc; - - lowLatency = MediaCodecHelper.decoderSupportsLowLatency(avcDecoder, mimeType); - adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder, mimeType); } else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) { mimeType = "video/hevc"; - selectedDecoderName = hevcDecoder.getName(); + selectedDecoderInfo = hevcDecoder; if (hevcDecoder == null) { LimeLog.severe("No available HEVC decoder!"); @@ -290,9 +286,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { } refFrameInvalidationActive = refFrameInvalidationHevc; - - lowLatency = MediaCodecHelper.decoderSupportsLowLatency(hevcDecoder, mimeType); - adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(hevcDecoder, mimeType); } else { // Unknown format @@ -300,10 +293,12 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { return -3; } + adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(selectedDecoderInfo, mimeType); + // Codecs have been known to throw all sorts of crazy runtime exceptions // due to implementation problems try { - videoDecoder = MediaCodec.createByCodecName(selectedDecoderName); + videoDecoder = MediaCodec.createByCodecName(selectedDecoderInfo.getName()); } catch (Exception e) { e.printStackTrace(); return -4; @@ -326,26 +321,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { videoFormat.setInteger(MediaFormat.KEY_MAX_HEIGHT, height); } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && lowLatency) { - videoFormat.setInteger(MediaFormat.KEY_LOW_LATENCY, 1); - } - else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // Set the Qualcomm vendor low latency extension if the Android R option is unavailable - if (MediaCodecHelper.decoderSupportsQcomVendorLowLatency(selectedDecoderName)) { - // MediaCodec supports vendor-defined format keys using the "vendor.." syntax. - // These allow access to functionality that is not exposed through documented MediaFormat.KEY_* values. - // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/common/inc/vidc_vendor_extensions.h;l=67 - // - // Examples of Qualcomm's vendor extensions for Snapdragon 845: - // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp - // https://cs.android.com/android/_/android/platform/hardware/qcom/sm8150/media/+/0621ceb1c1b19564999db8293574a0e12952ff6c - videoFormat.setInteger("vendor.qti-ext-dec-low-latency.enable", 1); - } - - if (MediaCodecHelper.decoderSupportsMaxOperatingRate(selectedDecoderName)) { - videoFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE); - } - } + MediaCodecHelper.setDecoderLowLatencyOptions(videoFormat, selectedDecoderInfo, mimeType); configuredFormat = videoFormat; LimeLog.info("Configuring with format: "+configuredFormat); @@ -375,7 +351,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { }, null); } - LimeLog.info("Using codec "+selectedDecoderName+" for hardware decoding "+mimeType); + LimeLog.info("Using codec "+selectedDecoderInfo.getName()+" for hardware decoding "+mimeType); // Start the decoder videoDecoder.start(); @@ -1103,7 +1079,6 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { str += "Consecutive crashes: "+renderer.consecutiveCrashCount+"\n"; str += "RFI active: "+renderer.refFrameInvalidationActive+"\n"; str += "Using modern SPS patching: "+(Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)+"\n"; - str += "Low latency mode: "+renderer.lowLatency+"\n"; str += "Video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n"; str += "FPS target: "+renderer.refreshRate+"\n"; str += "Bitrate: "+renderer.prefs.bitrate+" Kbps \n"; diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java index 317e51f1..f097f0d4 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -18,6 +18,7 @@ import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo.CodecProfileLevel; +import android.media.MediaFormat; import android.os.Build; import com.limelight.LimeLog; @@ -39,6 +40,7 @@ public class MediaCodecHelper { private static final List blacklisted49FpsDecoderPrefixes; private static final List blacklisted59FpsDecoderPrefixes; private static final List qualcommDecoderPrefixes; + private static final List kirinDecoderPrefixes; public static final boolean IS_EMULATOR = Build.HARDWARE.equals("ranchu") || Build.HARDWARE.equals("cheets"); @@ -206,6 +208,12 @@ public class MediaCodecHelper { qualcommDecoderPrefixes.add("c2.qti"); } + static { + kirinDecoderPrefixes = new LinkedList<>(); + + kirinDecoderPrefixes.add("omx.hisi"); + } + private static boolean isPowerVR(String glRenderer) { return glRenderer.toLowerCase().contains("powervr"); } @@ -352,7 +360,7 @@ public class MediaCodecHelper { return System.nanoTime() / 1000000L; } - public static boolean decoderSupportsLowLatency(MediaCodecInfo decoderInfo, String mimeType) { + private static boolean decoderSupportsAndroidRLowLatency(MediaCodecInfo decoderInfo, String mimeType) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { try { if (decoderInfo.getCapabilitiesForType(mimeType).isFeatureSupported(CodecCapabilities.FEATURE_LowLatency)) { @@ -368,7 +376,7 @@ public class MediaCodecHelper { return false; } - public static boolean decoderSupportsMaxOperatingRate(String decoderName) { + private static boolean decoderSupportsMaxOperatingRate(String decoderName) { // Operate at maximum rate to lower latency as much as possible on // some Qualcomm platforms. We could also set KEY_PRIORITY to 0 (realtime) // but that will actually result in the decoder crashing if it can't satisfy @@ -383,6 +391,43 @@ public class MediaCodecHelper { !isAdreno620; } + public static void setDecoderLowLatencyOptions(MediaFormat videoFormat, MediaCodecInfo decoderInfo, String mimeType) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && decoderSupportsAndroidRLowLatency(decoderInfo, mimeType)) { + videoFormat.setInteger(MediaFormat.KEY_LOW_LATENCY, 1); + } + else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // MediaCodec supports vendor-defined format keys using the "vendor.." syntax. + // These allow access to functionality that is not exposed through documented MediaFormat.KEY_* values. + // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/common/inc/vidc_vendor_extensions.h;l=67 + // + // MediaCodec vendor extension support was introduced in Android 8.0: + // https://cs.android.com/android/_/android/platform/frameworks/av/+/01c10f8cdcd58d1e7025f426a72e6e75ba5d7fc2 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // Try vendor-specific low latency options + if (isDecoderInList(qualcommDecoderPrefixes, decoderInfo.getName())) { + // Examples of Qualcomm's vendor extensions for Snapdragon 845: + // https://cs.android.com/android/platform/superproject/+/master:hardware/qcom/sdm845/media/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp + // https://cs.android.com/android/_/android/platform/hardware/qcom/sm8150/media/+/0621ceb1c1b19564999db8293574a0e12952ff6c + videoFormat.setInteger("vendor.qti-ext-dec-low-latency.enable", 1); + } + else if (isDecoderInList(kirinDecoderPrefixes, decoderInfo.getName())) { + // Kirin low latency options + // https://developer.huawei.com/consumer/cn/forum/topic/0202325564295980115 + videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1); + videoFormat.setInteger("vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-rdy", -1); + } + } + else if (isDecoderInList(qualcommDecoderPrefixes, decoderInfo.getName())) { + // This is an older low latency option used on some Qualcomm devices + videoFormat.setInteger("vt-low-latency", 1); + } + + if (MediaCodecHelper.decoderSupportsMaxOperatingRate(decoderInfo.getName())) { + videoFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE); + } + } + } + public static boolean decoderSupportsAdaptivePlayback(MediaCodecInfo decoderInfo, String mimeType) { // Possibly enable adaptive playback on KitKat and above if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { @@ -408,13 +453,6 @@ public class MediaCodecHelper { return false; } - public static boolean decoderSupportsQcomVendorLowLatency(String decoderName) { - // MediaCodec vendor extension support was introduced in Android 8.0: - // https://cs.android.com/android/_/android/platform/frameworks/av/+/01c10f8cdcd58d1e7025f426a72e6e75ba5d7fc2 - return Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && - isDecoderInList(qualcommDecoderPrefixes, decoderName); - } - public static boolean decoderNeedsConstrainedHighProfile(String decoderName) { return isDecoderInList(constrainedHighProfilePrefixes, decoderName); }