Enable reference frame invalidation for recent Qualcomm and NVIDIA decoders

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -35,6 +35,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
     private boolean needsSpsBitstreamFixup, isExynos4;
     private boolean adaptivePlayback, directSubmit;
     private boolean constrainedHighProfile;
+    private boolean refFrameInvalidationAvc, refFrameInvalidationHevc;
+    private boolean refFrameInvalidationActive;
     private int initialWidth, initialHeight;
     private int videoFormat;
     private Object renderTarget;
@@ -118,10 +120,18 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
         if (avcDecoder != null) {
             directSubmit = MediaCodecHelper.decoderCanDirectSubmit(avcDecoder.getName());
             adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder.getName());
+            refFrameInvalidationAvc = MediaCodecHelper.decoderSupportsRefFrameInvalidationAvc(avcDecoder.getName());
+            refFrameInvalidationHevc = MediaCodecHelper.decoderSupportsRefFrameInvalidationHevc(avcDecoder.getName());
 
             if (directSubmit) {
                 LimeLog.info("Decoder "+avcDecoder.getName()+" will use direct submit");
             }
+            if (refFrameInvalidationAvc) {
+                LimeLog.info("Decoder "+avcDecoder.getName()+" will use reference frame invalidation for AVC");
+            }
+            if (refFrameInvalidationHevc) {
+                LimeLog.info("Decoder "+avcDecoder.getName()+" will use reference frame invalidation for HEVC");
+            }
         }
     }
 
@@ -172,6 +182,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
             if (isExynos4) {
                 LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4");
             }
+
+            refFrameInvalidationActive = refFrameInvalidationAvc;
         }
         else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) {
             mimeType = "video/hevc";
@@ -181,6 +193,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
                 LimeLog.severe("No available HEVC decoder!");
                 return false;
             }
+
+            refFrameInvalidationActive = refFrameInvalidationHevc;
         }
         else {
             // Unknown format
@@ -468,19 +482,22 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
 
             // Some decoders rely on H264 level to decide how many buffers are needed
             // Since we only need one frame buffered, we'll set the level as low as we can
-            // for known resolution combinations
+            // for known resolution combinations. Reference frame invalidation may need
-            if (initialWidth == 1280 && initialHeight == 720) {
+            // these, so leave them be for those decoders.
-                // Max 5 buffered frames at 1280x720x60
+            if (!refFrameInvalidationActive) {
-                LimeLog.info("Patching level_idc to 32");
+                if (initialWidth == 1280 && initialHeight == 720) {
-                sps.level_idc = 32;
+                    // Max 5 buffered frames at 1280x720x60
-            }
+                    LimeLog.info("Patching level_idc to 32");
-            else if (initialWidth == 1920 && initialHeight == 1080) {
+                    sps.level_idc = 32;
-                // Max 4 buffered frames at 1920x1080x64
+                }
-                LimeLog.info("Patching level_idc to 42");
+                else if (initialWidth == 1920 && initialHeight == 1080) {
-                sps.level_idc = 42;
+                    // Max 4 buffered frames at 1920x1080x64
-            }
+                    LimeLog.info("Patching level_idc to 42");
-            else {
+                    sps.level_idc = 42;
-                // Leave the profile alone (currently 5.0)
+                }
+                else {
+                    // Leave the profile alone (currently 5.0)
+                }
             }
 
             // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4
@@ -489,8 +506,13 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
             // I'm doing this fixup for all devices because I haven't seen any devices that
             // this causes issues for. At worst, it seems to do nothing and at best it fixes
             // issues with video lag, hangs, and crashes.
-            LimeLog.info("Patching num_ref_frames in SPS");
+            //
-            sps.num_ref_frames = 1;
+            // It does break reference frame invalidation, so we will not do that for decoders
+            // where we've enabled reference frame invalidation.
+            if (!refFrameInvalidationActive) {
+                LimeLog.info("Patching num_ref_frames in SPS");
+                sps.num_ref_frames = 1;
+            }
 
             // GFE 2.5.11 changed the SPS to add additional extensions
             // Some devices don't like these so we remove them here.
@@ -498,7 +520,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
             sps.vuiParams.colour_description_present_flag = false;
             sps.vuiParams.chroma_loc_info_present_flag = false;
 
-            if (needsSpsBitstreamFixup || isExynos4) {
+            if ((needsSpsBitstreamFixup || isExynos4) && !refFrameInvalidationActive) {
                 // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag
                 // or max_dec_frame_buffering which increases decoding latency on Tegra.
 
@@ -627,6 +649,24 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
         LimeLog.info("SPS replay complete");
     }
 
+    @Override
+    public int getCapabilities() {
+        int capabilities = 0;
+
+        // We always request 4 slices per frame to speed up decoding on some hardware
+        capabilities |= MoonBridge.CAPABILITY_SLICES_PER_FRAME((byte) 4);
+
+        // Enable reference frame invalidation on supported hardware
+        if (refFrameInvalidationAvc) {
+            capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC;
+        }
+        if (refFrameInvalidationHevc) {
+            capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC;
+        }
+
+        return capabilities;
+    }
+
     public int getAverageEndToEndLatency() {
         if (totalFrames == 0) {
             return 0;

app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java

@@ -32,6 +32,8 @@ public class MediaCodecHelper {
     private static final List<String> directSubmitPrefixes;
 	private static final List<String> constrainedHighProfilePrefixes;
 	private static final List<String> whitelistedHevcDecoders;
+	private static final List<String> refFrameInvalidationAvcPrefixes;
+    private static final List<String> refFrameInvalidationHevcPrefixes;
 
     static {
         directSubmitPrefixes = new LinkedList<>();
@@ -47,6 +49,13 @@ public class MediaCodecHelper {
         directSubmitPrefixes.add("omx.arc");
     }
 
+    static {
+        refFrameInvalidationAvcPrefixes = new LinkedList<>();
+        refFrameInvalidationHevcPrefixes = new LinkedList<>();
+
+		// Qualcomm and NVIDIA may be added at runtime
+	}
+
 	static {
 		preferredDecoders = new LinkedList<>();
 	}
@@ -65,6 +74,8 @@ public class MediaCodecHelper {
 	}
 	
 	static {
+		// If a decoder qualifies for reference frame invalidation,
+		// these entries will be ignored for those decoders.
 		spsFixupBitstreamFixupDecoderPrefixes = new LinkedList<>();
 		spsFixupBitstreamFixupDecoderPrefixes.add("omx.nvidia");
 		spsFixupBitstreamFixupDecoderPrefixes.add("omx.qcom");
@@ -113,20 +124,33 @@ public class MediaCodecHelper {
 				(ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE);
 		ConfigurationInfo configInfo = activityManager.getDeviceConfigurationInfo();
 		if (configInfo.reqGlEsVersion != ConfigurationInfo.GL_ES_VERSION_UNDEFINED) {
-			// Qualcomm's early HEVC decoders break hard on our HEVC stream. The best check to
-			// tell the good from the bad decoders are the generation of Adreno GPU included:
-			// 3xx - bad
-			// 4xx - good
-			//
-			// Unfortunately, it's not that easy to get that information here, so I'll use an
-			// approximation by checking the GLES level (<= 3.0 is bad).
 			LimeLog.info("OpenGL ES version: "+configInfo.reqGlEsVersion);
-			if (configInfo.reqGlEsVersion > 0x30000) {
+
-				LimeLog.info("Added omx.qcom to supported decoders based on GLES 3.1+ support");
+			// Tegra K1 and later can do reference frame invalidation properly
-				whitelistedHevcDecoders.add("omx.qcom");
+			if (configInfo.reqGlEsVersion >= 0x30000) {
+				LimeLog.info("Added omx.nvidia to AVC reference frame invalidation support list");
+				refFrameInvalidationAvcPrefixes.add("omx.nvidia");
+
+                LimeLog.info("Added omx.qcom to AVC reference frame invalidation support list");
+                refFrameInvalidationAvcPrefixes.add("omx.qcom");
+
+				// Qualcomm's early HEVC decoders break hard on our HEVC stream. The best check to
+				// tell the good from the bad decoders are the generation of Adreno GPU included:
+				// 3xx - bad
+				// 4xx - good
+				//
+				// Unfortunately, it's not that easy to get that information here, so I'll use an
+				// approximation by checking the GLES level (<= 3.0 is bad).
+				if (configInfo.reqGlEsVersion > 0x30000) {
+					// FIXME: We prefer reference frame invalidation support (which is only doable on AVC on
+					// older Qualcomm chips) vs. enabling HEVC by default. The user can override using the settings
+					// to force HEVC on.
+					//LimeLog.info("Added omx.qcom to supported HEVC decoders based on GLES 3.1+ support");
+					//whitelistedHevcDecoders.add("omx.qcom");
+				}
 			}
 		}
-	}
+    }
 
 	private static boolean isDecoderInList(List<String> decoderList, String decoderName) {
 		for (String badPrefix : decoderList) {
@@ -190,6 +214,14 @@ public class MediaCodecHelper {
         return isDecoderInList(baselineProfileHackPrefixes, decoderName);
     }
 
+    public static boolean decoderSupportsRefFrameInvalidationAvc(String decoderName) {
+		return isDecoderInList(refFrameInvalidationAvcPrefixes, decoderName);
+	}
+
+    public static boolean decoderSupportsRefFrameInvalidationHevc(String decoderName) {
+        return isDecoderInList(refFrameInvalidationHevcPrefixes, decoderName);
+    }
+
 	public static boolean decoderIsWhitelistedForHevc(String decoderName) {
 		// TODO: Shield Tablet K1/LTE?
 		//

moonlight-common

@@ -1 +1 @@
-Subproject commit bda8e3b84ddb633d47b00ffc7ec4d5f20c7eecae
+Subproject commit 7794ca7a3ecb4398d977beca138d2e584e533002