Frame latency and jitter improvements

src/com/limelight/binding/video/AndroidCpuDecoderRenderer.java

@@ -13,6 +13,7 @@ import com.limelight.LimeLog;
 import com.limelight.nvstream.av.ByteBufferDescriptor;
 import com.limelight.nvstream.av.DecodeUnit;
 import com.limelight.nvstream.av.video.VideoDecoderRenderer;
+import com.limelight.nvstream.av.video.VideoDepacketizer;
 import com.limelight.nvstream.av.video.cpu.AvcDecoder;
 
 public class AndroidCpuDecoderRenderer implements VideoDecoderRenderer {
@@ -141,22 +142,23 @@ public class AndroidCpuDecoderRenderer implements VideoDecoderRenderer {
 	}
 
 	@Override
-	public void start() {
+	public void start(final VideoDepacketizer depacketizer) {
 		rendererThread = new Thread() {
 			@Override
 			public void run() {
 				long nextFrameTime = System.currentTimeMillis();
-				
+				DecodeUnit du;
 				while (!isInterrupted())
 				{
+					du = depacketizer.pollNextDecodeUnit(); 
+					if (du != null) {
+						submitDecodeUnit(du);
+					}
+					
 					long diff = nextFrameTime - System.currentTimeMillis();
 
 					if (diff > WAIT_CEILING_MS) {
-						try {
+						continue;
-							Thread.sleep(diff);
-						} catch (InterruptedException e) {
-							return;
-						}
 					}
 
 					nextFrameTime = computePresentationTimeMs(targetFps);
@@ -165,6 +167,7 @@ public class AndroidCpuDecoderRenderer implements VideoDecoderRenderer {
 			}
 		};
 		rendererThread.setName("Video - Renderer (CPU)");
+		rendererThread.setPriority(Thread.MAX_PRIORITY);
 		rendererThread.start();
 	}
 	
@@ -186,8 +189,7 @@ public class AndroidCpuDecoderRenderer implements VideoDecoderRenderer {
 		AvcDecoder.destroy();
 	}
 
-	@Override
+	private boolean submitDecodeUnit(DecodeUnit decodeUnit) {
-	public boolean submitDecodeUnit(DecodeUnit decodeUnit) {
 		byte[] data;
 		
 		// Use the reserved decoder buffer if this decode unit will fit

src/com/limelight/binding/video/ConfigurableDecoderRenderer.java

@@ -1,7 +1,7 @@
 package com.limelight.binding.video;
 
-import com.limelight.nvstream.av.DecodeUnit;
 import com.limelight.nvstream.av.video.VideoDecoderRenderer;
+import com.limelight.nvstream.av.video.VideoDepacketizer;
 
 public class ConfigurableDecoderRenderer implements VideoDecoderRenderer {
 
@@ -26,8 +26,8 @@ public class ConfigurableDecoderRenderer implements VideoDecoderRenderer {
 	}
 
 	@Override
-	public void start() {
+	public void start(VideoDepacketizer depacketizer) {
-		decoderRenderer.start();
+		decoderRenderer.start(depacketizer);
 	}
 
 	@Override
@@ -35,11 +35,6 @@ public class ConfigurableDecoderRenderer implements VideoDecoderRenderer {
 		decoderRenderer.stop();
 	}
 
-	@Override
-	public boolean submitDecodeUnit(DecodeUnit du) {
-		return decoderRenderer.submitDecodeUnit(du);
-	}
-
 	@Override
 	public int getCapabilities() {
 		return decoderRenderer.getCapabilities();

src/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -8,6 +8,7 @@ import com.limelight.LimeLog;
 import com.limelight.nvstream.av.ByteBufferDescriptor;
 import com.limelight.nvstream.av.DecodeUnit;
 import com.limelight.nvstream.av.video.VideoDecoderRenderer;
+import com.limelight.nvstream.av.video.VideoDepacketizer;
 
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
@@ -23,15 +24,13 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 	private ByteBuffer[] videoDecoderInputBuffers;
 	private MediaCodec videoDecoder;
 	private Thread rendererThread;
-	private int redrawRate;
 	private boolean needsSpsBitstreamFixup;
 	private boolean needsSpsNumRefFixup;
-	private boolean fastInputQueueing;
+	private VideoDepacketizer depacketizer;
 	
 	public static final List<String> blacklistedDecoderPrefixes;
 	public static final List<String> spsFixupBitsreamFixupDecoderPrefixes;
 	public static final List<String> spsFixupNumRefFixupDecoderPrefixes;
-	public static final List<String> fastInputQueueingPrefixes;
 	
 	static {
 		blacklistedDecoderPrefixes = new LinkedList<String>();
@@ -47,11 +46,6 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 		spsFixupNumRefFixupDecoderPrefixes.add("omx.TI");
 	}
 		
-	static {
-		fastInputQueueingPrefixes = new LinkedList<String>();
-		fastInputQueueingPrefixes.add("omx.nvidia");
-	}
-		
 	private static boolean isDecoderInList(List<String> decoderList, String decoderName) {
 		for (String badPrefix : decoderList) {
 			if (decoderName.length() >= badPrefix.length()) {
@@ -125,8 +119,6 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 	
 	@Override
 	public void setup(int width, int height, int redrawRate, Object renderTarget, int drFlags) {
-		this.redrawRate = redrawRate;
-		
 		//dumpDecoders();
 		
 		MediaCodecInfo safeDecoder = findSafeDecoder();
@@ -140,16 +132,11 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 			if (needsSpsNumRefFixup) {
 				LimeLog.info("Decoder "+safeDecoder.getName()+" needs SPS ref num fixup");
 			}
-			fastInputQueueing = isDecoderInList(fastInputQueueingPrefixes, safeDecoder.getName());
-			if (fastInputQueueing) {
-				LimeLog.info("Decoder "+safeDecoder.getName()+" supports fast input queueing");
-			}
 		}
 		else {
 			videoDecoder = MediaCodec.createDecoderByType("video/avc");
 			needsSpsBitstreamFixup = false;
 			needsSpsNumRefFixup = false;
-			fastInputQueueing = false;
 		}
 		
 		MediaFormat videoFormat = MediaFormat.createVideoFormat("video/avc", width, height);
@@ -167,32 +154,18 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 		rendererThread = new Thread() {
 			@Override
 			public void run() {
-				long nextFrameTimeUs = 0;
 				BufferInfo info = new BufferInfo();
+				DecodeUnit du;
 				while (!isInterrupted())
 				{
-					// Block for a maximum of 100 ms
+					du = depacketizer.pollNextDecodeUnit();
-					int outIndex = videoDecoder.dequeueOutputBuffer(info, 100000);
+					if (du != null) {
-				    switch (outIndex) {
+						submitDecodeUnit(du);
-				    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
+					}
-				    	LimeLog.info("Output buffers changed");
-					    break;
-				    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
-				    	LimeLog.info("Output format changed");
-				    	LimeLog.info("New output Format: " + videoDecoder.getOutputFormat());
-				    	break;
-				    default:
-				      break;
-				    }
 					
+					int outIndex = videoDecoder.dequeueOutputBuffer(info, 0);
 				    if (outIndex >= 0) {
 					    int lastIndex = outIndex;
-				    	boolean render = false;
-					    
-				    	if (currentTimeUs() >= nextFrameTimeUs) {
-				    		render = true;
-				    		nextFrameTimeUs = computePresentationTime(redrawRate);
-				    	}
 
 					    // Get the last output buffer in the queue
 					    while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) {
@@ -200,26 +173,32 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 					    	lastIndex = outIndex;
 					    }
 					    
-					    // Render that buffer if it's time for the next frame
+					    // Render the last buffer
-				    	videoDecoder.releaseOutputBuffer(lastIndex, render);
+				    	videoDecoder.releaseOutputBuffer(lastIndex, true);
+				    } else {
+					    switch (outIndex) {
+					    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
+					    	LimeLog.info("Output buffers changed");
+						    break;
+					    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
+					    	LimeLog.info("Output format changed");
+					    	LimeLog.info("New output Format: " + videoDecoder.getOutputFormat());
+					    	break;
+					    default:
+					      break;
+					    }
 				    }
 				}
 			}
 		};
 		rendererThread.setName("Video - Renderer (MediaCodec)");
+		rendererThread.setPriority(Thread.MAX_PRIORITY);
 		rendererThread.start();
 	}
 
-	private static long currentTimeUs() {
-		return System.nanoTime() / 1000;
-	}
-
-	private long computePresentationTime(int frameRate) {
-		return currentTimeUs() + (1000000 / frameRate);
-	}
-
 	@Override
-	public void start() {
+	public void start(VideoDepacketizer depacketizer) {
+		this.depacketizer = depacketizer;
 		startRendererThread();
 	}
 
@@ -239,24 +218,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 		}
 	}
 
-	@Override
+	private boolean submitDecodeUnit(DecodeUnit decodeUnit) {
-	public boolean submitDecodeUnit(DecodeUnit decodeUnit) {
-		if (decodeUnit.getType() != DecodeUnit.TYPE_H264) {
-			System.err.println("Unknown decode unit type");
-			return false;
-		}
-		
-		int mcFlags = 0;
-		
-		if ((decodeUnit.getFlags() & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
-			LimeLog.info("Codec config");
-			mcFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
-		}
-		if ((decodeUnit.getFlags() & DecodeUnit.DU_FLAG_SYNC_FRAME) != 0) {
-			LimeLog.info("Sync frame");
-			mcFlags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME;
-		}
-		
 		int inputIndex = videoDecoder.dequeueInputBuffer(-1);
 		if (inputIndex >= 0)
 		{
@@ -312,7 +274,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 
 					videoDecoder.queueInputBuffer(inputIndex,
 							0, spsLength,
-							0, mcFlags);
+							0, 0);
 					return true;
 				}
 			}
@@ -325,7 +287,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 
 			videoDecoder.queueInputBuffer(inputIndex,
 					0, decodeUnit.getDataLength(),
-					0, mcFlags);
+					0, 0);
 		}
 		
 		return true;
@@ -333,7 +295,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer {
 
 	@Override
 	public int getCapabilities() {
-		return fastInputQueueing ? VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT : 0;
+		return 0;
 	}
 
 	/**