Reduce Nexus Player video latency by 10x

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -33,6 +33,9 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 	private boolean adaptivePlayback;
 	private int initialWidth, initialHeight;
 
+    private boolean needsBaselineSpsHack;
+    private SeqParameterSet savedSps;
+	
 	private long lastTimestampUs;
 	private long totalTimeMs;
 	private long decoderTimeMs;
@@ -63,10 +66,14 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 		// Set decoder-specific attributes
 		adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(decoderName, decoder);
 		needsSpsBitstreamFixup = MediaCodecHelper.decoderNeedsSpsBitstreamRestrictions(decoderName, decoder);
-		if (needsSpsBitstreamFixup) {
+        needsBaselineSpsHack = MediaCodecHelper.decoderNeedsBaselineSpsHack(decoderName, decoder);
+        isExynos4 = MediaCodecHelper.isExynos4Device();
+        if (needsSpsBitstreamFixup) {
 			LimeLog.info("Decoder "+decoderName+" needs SPS bitstream restrictions fixup");
 		}
-		isExynos4 = MediaCodecHelper.isExynos4Device();
+        if (needsBaselineSpsHack) {
+            LimeLog.info("Decoder "+decoderName+" needs baseline SPS hack");
+        }
 		if (isExynos4) {
 			LimeLog.info("Decoder "+decoderName+" is on Exynos 4");
 		}
@@ -390,6 +397,8 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 			numIframeIn++;
 		}
 
+        boolean needsSpsReplay = false;
+		
 		if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
 			ByteBufferDescriptor header = decodeUnit.getBufferList().get(0);
 			if (header.data[header.offset+4] == 0x67) {
@@ -426,6 +435,13 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 					sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = 1;
 				}
 
+                // If we need to hack this SPS to say we're baseline, do so now
+                if (needsBaselineSpsHack) {
+                    LimeLog.info("Hacking SPS to baseline");
+                    sps.profile_idc = 66;
+                    savedSps = sps;
+                }
+				
 				// Write the annex B header
 				buf.put(header.data, header.offset, 5);
 				
@@ -440,6 +456,14 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 				return;
 			} else if (header.data[header.offset+4] == 0x68) {
 				numPpsIn++;
+
+                if (needsBaselineSpsHack) {
+                    LimeLog.info("Saw PPS; disabling SPS hack");
+                    needsBaselineSpsHack = false;
+
+                    // Give the decoder the SPS again with the proper profile now
+                    needsSpsReplay = true;
+                }
 			}
 		}
 
@@ -454,8 +478,39 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 				timestampUs, codecFlags);
 		
 		depacketizer.freeDecodeUnit(decodeUnit);
+
+        if (needsSpsReplay) {
+            replaySps();
+        }
 	}
 
+    private void replaySps() {
+        int inputIndex = videoDecoder.dequeueInputBuffer(-1);
+        ByteBuffer inputBuffer = videoDecoderInputBuffers[inputIndex];
+
+        inputBuffer.clear();
+
+        // Write the Annex B header
+        inputBuffer.put(new byte[]{0x00, 0x00, 0x00, 0x01, 0x67});
+
+        // Switch the H264 profile back to high
+        savedSps.profile_idc = 100;
+
+        // Write the SPS data
+        savedSps.write(inputBuffer);
+
+        // No need for the SPS anymore
+        savedSps = null;
+
+        // Queue the new SPS
+        queueInputBuffer(inputIndex,
+                0, inputBuffer.position(),
+                System.currentTimeMillis() * 1000,
+                MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
+
+        LimeLog.info("SPS replay complete");
+    }
+
 	@Override
 	public int getCapabilities() {
 		return adaptivePlayback ?

app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java

@@ -25,6 +25,7 @@ public class MediaCodecHelper {
 	public static final List<String> blacklistedDecoderPrefixes;
 	public static final List<String> spsFixupBitstreamFixupDecoderPrefixes;
 	public static final List<String> whitelistedAdaptiveResolutionPrefixes;
+    public static final List<String> baselineProfileHackPrefixes;
 	
 	static {
 		preferredDecoders = new LinkedList<String>();
@@ -44,6 +45,9 @@ public class MediaCodecHelper {
 		spsFixupBitstreamFixupDecoderPrefixes.add("omx.qcom");
 		spsFixupBitstreamFixupDecoderPrefixes.add("omx.mtk");
 
+        baselineProfileHackPrefixes = new LinkedList<String>();
+        baselineProfileHackPrefixes.add("omx.intel");
+
 		whitelistedAdaptiveResolutionPrefixes = new LinkedList<String>();
 		whitelistedAdaptiveResolutionPrefixes.add("omx.nvidia");
 		whitelistedAdaptiveResolutionPrefixes.add("omx.qcom");
@@ -97,6 +101,10 @@ public class MediaCodecHelper {
 		return isDecoderInList(spsFixupBitstreamFixupDecoderPrefixes, decoderName);
 	}
 
+    public static boolean decoderNeedsBaselineSpsHack(String decoderName, MediaCodecInfo decoderInfo) {
+        return isDecoderInList(baselineProfileHackPrefixes, decoderName);
+    }
+	
 	@SuppressWarnings("deprecation")
 	@SuppressLint("NewApi")
 	private static LinkedList<MediaCodecInfo> getMediaCodecList() {