diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index f1b8c797..0d6d8ca4 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -35,6 +35,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private boolean needsSpsBitstreamFixup, isExynos4; private boolean adaptivePlayback, directSubmit; private boolean constrainedHighProfile; + private boolean refFrameInvalidationAvc, refFrameInvalidationHevc; + private boolean refFrameInvalidationActive; private int initialWidth, initialHeight; private int videoFormat; private Object renderTarget; @@ -118,10 +120,18 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { if (avcDecoder != null) { directSubmit = MediaCodecHelper.decoderCanDirectSubmit(avcDecoder.getName()); adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder.getName()); + refFrameInvalidationAvc = MediaCodecHelper.decoderSupportsRefFrameInvalidationAvc(avcDecoder.getName()); + refFrameInvalidationHevc = MediaCodecHelper.decoderSupportsRefFrameInvalidationHevc(avcDecoder.getName()); if (directSubmit) { LimeLog.info("Decoder "+avcDecoder.getName()+" will use direct submit"); } + if (refFrameInvalidationAvc) { + LimeLog.info("Decoder "+avcDecoder.getName()+" will use reference frame invalidation for AVC"); + } + if (refFrameInvalidationHevc) { + LimeLog.info("Decoder "+avcDecoder.getName()+" will use reference frame invalidation for HEVC"); + } } } @@ -172,6 +182,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { if (isExynos4) { LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4"); } + + refFrameInvalidationActive = refFrameInvalidationAvc; } else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) { mimeType = "video/hevc"; @@ -181,6 +193,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { LimeLog.severe("No available HEVC decoder!"); return false; } + + refFrameInvalidationActive = refFrameInvalidationHevc; } else { // Unknown format @@ -468,19 +482,22 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { // Some decoders rely on H264 level to decide how many buffers are needed // Since we only need one frame buffered, we'll set the level as low as we can - // for known resolution combinations - if (initialWidth == 1280 && initialHeight == 720) { - // Max 5 buffered frames at 1280x720x60 - LimeLog.info("Patching level_idc to 32"); - sps.level_idc = 32; - } - else if (initialWidth == 1920 && initialHeight == 1080) { - // Max 4 buffered frames at 1920x1080x64 - LimeLog.info("Patching level_idc to 42"); - sps.level_idc = 42; - } - else { - // Leave the profile alone (currently 5.0) + // for known resolution combinations. Reference frame invalidation may need + // these, so leave them be for those decoders. + if (!refFrameInvalidationActive) { + if (initialWidth == 1280 && initialHeight == 720) { + // Max 5 buffered frames at 1280x720x60 + LimeLog.info("Patching level_idc to 32"); + sps.level_idc = 32; + } + else if (initialWidth == 1920 && initialHeight == 1080) { + // Max 4 buffered frames at 1920x1080x64 + LimeLog.info("Patching level_idc to 42"); + sps.level_idc = 42; + } + else { + // Leave the profile alone (currently 5.0) + } } // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4 @@ -489,8 +506,13 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { // I'm doing this fixup for all devices because I haven't seen any devices that // this causes issues for. At worst, it seems to do nothing and at best it fixes // issues with video lag, hangs, and crashes. - LimeLog.info("Patching num_ref_frames in SPS"); - sps.num_ref_frames = 1; + // + // It does break reference frame invalidation, so we will not do that for decoders + // where we've enabled reference frame invalidation. + if (!refFrameInvalidationActive) { + LimeLog.info("Patching num_ref_frames in SPS"); + sps.num_ref_frames = 1; + } // GFE 2.5.11 changed the SPS to add additional extensions // Some devices don't like these so we remove them here. @@ -498,7 +520,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { sps.vuiParams.colour_description_present_flag = false; sps.vuiParams.chroma_loc_info_present_flag = false; - if (needsSpsBitstreamFixup || isExynos4) { + if ((needsSpsBitstreamFixup || isExynos4) && !refFrameInvalidationActive) { // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag // or max_dec_frame_buffering which increases decoding latency on Tegra. @@ -627,6 +649,24 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { LimeLog.info("SPS replay complete"); } + @Override + public int getCapabilities() { + int capabilities = 0; + + // We always request 4 slices per frame to speed up decoding on some hardware + capabilities |= MoonBridge.CAPABILITY_SLICES_PER_FRAME((byte) 4); + + // Enable reference frame invalidation on supported hardware + if (refFrameInvalidationAvc) { + capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC; + } + if (refFrameInvalidationHevc) { + capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; + } + + return capabilities; + } + public int getAverageEndToEndLatency() { if (totalFrames == 0) { return 0; diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java index 91c7e3a8..2bdb25de 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -32,6 +32,8 @@ public class MediaCodecHelper { private static final List directSubmitPrefixes; private static final List constrainedHighProfilePrefixes; private static final List whitelistedHevcDecoders; + private static final List refFrameInvalidationAvcPrefixes; + private static final List refFrameInvalidationHevcPrefixes; static { directSubmitPrefixes = new LinkedList<>(); @@ -47,6 +49,13 @@ public class MediaCodecHelper { directSubmitPrefixes.add("omx.arc"); } + static { + refFrameInvalidationAvcPrefixes = new LinkedList<>(); + refFrameInvalidationHevcPrefixes = new LinkedList<>(); + + // Qualcomm and NVIDIA may be added at runtime + } + static { preferredDecoders = new LinkedList<>(); } @@ -65,6 +74,8 @@ public class MediaCodecHelper { } static { + // If a decoder qualifies for reference frame invalidation, + // these entries will be ignored for those decoders. spsFixupBitstreamFixupDecoderPrefixes = new LinkedList<>(); spsFixupBitstreamFixupDecoderPrefixes.add("omx.nvidia"); spsFixupBitstreamFixupDecoderPrefixes.add("omx.qcom"); @@ -113,20 +124,33 @@ public class MediaCodecHelper { (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); ConfigurationInfo configInfo = activityManager.getDeviceConfigurationInfo(); if (configInfo.reqGlEsVersion != ConfigurationInfo.GL_ES_VERSION_UNDEFINED) { - // Qualcomm's early HEVC decoders break hard on our HEVC stream. The best check to - // tell the good from the bad decoders are the generation of Adreno GPU included: - // 3xx - bad - // 4xx - good - // - // Unfortunately, it's not that easy to get that information here, so I'll use an - // approximation by checking the GLES level (<= 3.0 is bad). LimeLog.info("OpenGL ES version: "+configInfo.reqGlEsVersion); - if (configInfo.reqGlEsVersion > 0x30000) { - LimeLog.info("Added omx.qcom to supported decoders based on GLES 3.1+ support"); - whitelistedHevcDecoders.add("omx.qcom"); + + // Tegra K1 and later can do reference frame invalidation properly + if (configInfo.reqGlEsVersion >= 0x30000) { + LimeLog.info("Added omx.nvidia to AVC reference frame invalidation support list"); + refFrameInvalidationAvcPrefixes.add("omx.nvidia"); + + LimeLog.info("Added omx.qcom to AVC reference frame invalidation support list"); + refFrameInvalidationAvcPrefixes.add("omx.qcom"); + + // Qualcomm's early HEVC decoders break hard on our HEVC stream. The best check to + // tell the good from the bad decoders are the generation of Adreno GPU included: + // 3xx - bad + // 4xx - good + // + // Unfortunately, it's not that easy to get that information here, so I'll use an + // approximation by checking the GLES level (<= 3.0 is bad). + if (configInfo.reqGlEsVersion > 0x30000) { + // FIXME: We prefer reference frame invalidation support (which is only doable on AVC on + // older Qualcomm chips) vs. enabling HEVC by default. The user can override using the settings + // to force HEVC on. + //LimeLog.info("Added omx.qcom to supported HEVC decoders based on GLES 3.1+ support"); + //whitelistedHevcDecoders.add("omx.qcom"); + } } } - } + } private static boolean isDecoderInList(List decoderList, String decoderName) { for (String badPrefix : decoderList) { @@ -190,6 +214,14 @@ public class MediaCodecHelper { return isDecoderInList(baselineProfileHackPrefixes, decoderName); } + public static boolean decoderSupportsRefFrameInvalidationAvc(String decoderName) { + return isDecoderInList(refFrameInvalidationAvcPrefixes, decoderName); + } + + public static boolean decoderSupportsRefFrameInvalidationHevc(String decoderName) { + return isDecoderInList(refFrameInvalidationHevcPrefixes, decoderName); + } + public static boolean decoderIsWhitelistedForHevc(String decoderName) { // TODO: Shield Tablet K1/LTE? // diff --git a/moonlight-common b/moonlight-common index bda8e3b8..7794ca7a 160000 --- a/moonlight-common +++ b/moonlight-common @@ -1 +1 @@ -Subproject commit bda8e3b84ddb633d47b00ffc7ec4d5f20c7eecae +Subproject commit 7794ca7a3ecb4398d977beca138d2e584e533002