From a639143e9497122511d2d2f402440e5407d7639e Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 30 Jun 2014 21:14:27 -0700 Subject: [PATCH] Add a latency toast after the stream stops --- libs/limelight-common.jar | Bin 402899 -> 403065 bytes src/com/limelight/Game.java | 22 ++++- .../video/AndroidCpuDecoderRenderer.java | 10 ++ .../video/ConfigurableDecoderRenderer.java | 19 ++++ .../video/MediaCodecDecoderRenderer.java | 92 +++++++++++++----- 5 files changed, 117 insertions(+), 26 deletions(-) diff --git a/libs/limelight-common.jar b/libs/limelight-common.jar index 1d597abe8de563cb6abc152807f3011a09c81c76..db3959e30bac619b086fb57f91de2a47f4286152 100644 GIT binary patch delta 1078 zcmYk44@{dy6u`gxzV_Bi+m+JwKVKQbLDO1cwA>n9Eji*qP1FOee!A zjjWw)I9NJ!3nH^&V!Am2V^bhPw#c&JSh3rf2!UZlBhGaK5>}&&*V?UK@{;%a-OIgP z-n)K95*w3nww6*1A;Ds?i0VKeXFNz*0<*r^U|Hjp@+O;bs8;C=)|_vB>2LQzwbA(6 zU^Y|FPX9DmlK=8EuKju@St{Px1H-e~`}@6Yb$x!8Q4ZAk3VeWp@Ib{*F`-26@YdGDP+-&lR|&9CGWksW1s ze`r4x88@xJWum4f_ph;Xwp*9z4;D@_TONOweR3*uF_0Uc;AH&PMVEU|NJKwN8d@1O zJKs9C|Ma#Gg|R0^e4UJdW~;>W!7#^&*Bw6Ky?ETouvJFO+5#Gta*F^r4--`g>D?9`^9?NWM+rnQ$z3 z^)t<)er^HN>#4r3s^8mgfvu%$SdQ8%RUIIRVv-;pj>Gr4A>HR5G&YTf zj*Nx?4iX22gAp1HF}c<9nXXwo@xQYM=XEaEWA*#?o-muxO)e}Ys{|Vin0Xiyz zeAJu?Wu#&?arsRqG?8twc%T02NgLBMPgb9t@zE8k)g7$e@Awz=fO|XaY{usewfx8_j8; z733hL7SlfD&_aVQzAs-kl2JL{6Df}PzNF3_E_|>($fL#6wD3@$77BTG!cNSH3Bt>R GxcXoE-igfs delta 1015 zcmex)M&j~m37!CNW)=|!4h{~6JBv5+oZmWmt%1Pg*IOOxcl#Z75V&*tv;YUoA#EY~ z2_|zU2>5E9?oT**=s}A@#+vTUT;ejfH|gvBFyXRm=y$p^Q_Y5JmTa%dZfoQB^6T>% z3i#BeLO5btPEK;G=@guA&nYFo^TY&QP3??>=`${L&QfbT$zv>^Y-xCTf!l(;39O3+ zyPsc~t!E&@Vzq5Ge|@v`{Y|>|JN90VP53X01VGhK~~w_E#MBTTa~ z?|h-m_vLpRFV}}i9W=`PyXpDA&7D(P&ujePPGc@zFs)lf*WQmKL(=U;P?5?78P(M~ zF0($%%jDCO`NpJN=a`P|fdeK)%9sGdVV6ru?5^xl=#orb`L|-M3v1=suM{KtZkP zf8~IRFL6)bsLUog{ecW4+jKo1M)B$K@<3rRez-8_^tJMA)=arVa1plYoC-jZiy}xO zx}uDt({C#Pt>{t!TCrIiNseEFQ4AypbiNEkSV57^ooR_2Ld`UJMxp7m6oE>vt3gEX zE3$bqZP$TwB&J6z0R#Ms0bE3U`aUIK=%ri1ML4G)vts0%uAmGw=Y}m@SZKO~9Z>kM zFwk{VmD!{ig{N;&W{YN02?QyTn_eHtC^Fq%g-wfb#`G)|HZ#VU=?hiZ%o+Wr-&A2U zWptm;ugd1g2uuO0Y<9pO@8V`tny#(JrloR*myJO$J+mY;Jukl~HNcyd SjR6RpfpCR51A{0ZhzkIAUv@+Q diff --git a/src/com/limelight/Game.java b/src/com/limelight/Game.java index 6a7bd4aa..1267b4a3 100644 --- a/src/com/limelight/Game.java +++ b/src/com/limelight/Game.java @@ -65,6 +65,8 @@ public class Game extends Activity implements SurfaceHolder.Callback, OnGenericM private boolean connecting = false; private boolean connected = false; + private ConfigurableDecoderRenderer decoderRenderer; + private WifiManager.WifiLock wifiLock; private int drFlags = 0; @@ -194,6 +196,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, OnGenericM enableLargePackets ? 1460 : 1024), PlatformBinding.getCryptoProvider(this)); keybTranslator = new KeyboardTranslator(conn); controllerHandler = new ControllerHandler(conn); + decoderRenderer = new ConfigurableDecoderRenderer(); // The connection will be started when the surface gets created sh.addCallback(this); @@ -330,6 +333,23 @@ public class Game extends Activity implements SurfaceHolder.Callback, OnGenericM displayedFailureDialog = true; conn.stop(); + int averageEndToEndLat = decoderRenderer.getAverageEndToEndLatency(); + int averageDecoderLat = decoderRenderer.getAverageDecoderLatency(); + String message = null; + if (averageEndToEndLat > 0) { + message = "Average frame latency from receive to display: "+averageEndToEndLat+" ms"; + if (averageDecoderLat > 0) { + message += " (decoder latency: "+averageDecoderLat+" ms)"; + } + } + else if (averageDecoderLat > 0) { + message = "Average decoder latency: "+averageDecoderLat+" ms"; + } + + if (message != null) { + Toast.makeText(this, message, Toast.LENGTH_LONG).show(); + } + finish(); } @@ -647,7 +667,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, OnGenericM if (!connected && !connecting) { connecting = true; conn.start(PlatformBinding.getDeviceName(), holder, drFlags, - PlatformBinding.getAudioRenderer(), new ConfigurableDecoderRenderer()); + PlatformBinding.getAudioRenderer(), decoderRenderer); } } diff --git a/src/com/limelight/binding/video/AndroidCpuDecoderRenderer.java b/src/com/limelight/binding/video/AndroidCpuDecoderRenderer.java index aa54f5d5..8378dff6 100644 --- a/src/com/limelight/binding/video/AndroidCpuDecoderRenderer.java +++ b/src/com/limelight/binding/video/AndroidCpuDecoderRenderer.java @@ -222,4 +222,14 @@ public class AndroidCpuDecoderRenderer implements VideoDecoderRenderer { public int getCapabilities() { return 0; } + + @Override + public int getAverageDecoderLatency() { + return 0; + } + + @Override + public int getAverageEndToEndLatency() { + return 0; + } } diff --git a/src/com/limelight/binding/video/ConfigurableDecoderRenderer.java b/src/com/limelight/binding/video/ConfigurableDecoderRenderer.java index 6cca9c9a..5ee2ce24 100644 --- a/src/com/limelight/binding/video/ConfigurableDecoderRenderer.java +++ b/src/com/limelight/binding/video/ConfigurableDecoderRenderer.java @@ -40,4 +40,23 @@ public class ConfigurableDecoderRenderer implements VideoDecoderRenderer { return decoderRenderer.getCapabilities(); } + @Override + public int getAverageDecoderLatency() { + if (decoderRenderer != null) { + return decoderRenderer.getAverageDecoderLatency(); + } + else { + return 0; + } + } + + @Override + public int getAverageEndToEndLatency() { + if (decoderRenderer != null) { + return decoderRenderer.getAverageEndToEndLatency(); + } + else { + return 0; + } + } } diff --git a/src/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/src/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 7d5afc5c..16bb6ebb 100644 --- a/src/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/src/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -28,6 +28,12 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { private boolean needsSpsNumRefFixup; private VideoDepacketizer depacketizer; + private long totalTimeMs; + private long decoderTimeMs; + private int totalFrames; + + private final static byte[] BITSTREAM_RESTRICTIONS = new byte[] {(byte) 0xF1, (byte) 0x83, 0x2A, 0x00}; + public static final List blacklistedDecoderPrefixes; public static final List spsFixupBitsreamFixupDecoderPrefixes; public static final List spsFixupNumRefFixupDecoderPrefixes; @@ -177,6 +183,13 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { if (outIndex >= 0) { int lastIndex = outIndex; + // Add delta time to the totals (excluding probable outliers) + long delta = System.currentTimeMillis()-info.presentationTimeUs; + if (delta > 5 && delta < 300) { + decoderTimeMs += delta; + totalTimeMs += delta; + } + // Get the last output buffer in the queue while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) { videoDecoder.releaseOutputBuffer(lastIndex, false); @@ -216,7 +229,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { @Override public void stop() { rendererThread.interrupt(); - + try { rendererThread.join(); } catch (InterruptedException e) { } @@ -235,6 +248,13 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { { ByteBuffer buf = videoDecoderInputBuffers[inputIndex]; + long currentTime = System.currentTimeMillis(); + long delta = currentTime-decodeUnit.getReceiveTimestamp(); + if (delta >= 0 && delta < 300) { + totalTimeMs += currentTime-decodeUnit.getReceiveTimestamp(); + totalFrames++; + } + // Clear old input data buf.clear(); @@ -251,6 +271,8 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { (needsSpsBitstreamFixup || needsSpsNumRefFixup)) { ByteBufferDescriptor header = decodeUnit.getBufferList().get(0); if (header.data[header.offset+4] == 0x67) { + byte last = header.data[header.length+header.offset-1]; + // TI OMAP4 requires a reference frame count of 1 to decode successfully if (needsSpsNumRefFixup) { LimeLog.info("Fixing up num ref frames"); @@ -262,31 +284,41 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { // We manually modify the SPS here to speed-up decoding if the decoder was flagged as needing it. int spsLength; if (needsSpsBitstreamFixup) { - switch (header.length) { - case 26: - LimeLog.info("Adding bitstream restrictions to SPS (26)"); - buf.put(header.data, header.offset, 24); - buf.put((byte) 0x11); - buf.put((byte) 0xe3); - buf.put((byte) 0x06); - buf.put((byte) 0x50); - spsLength = header.length + 2; - break; - case 27: - LimeLog.info("Adding bitstream restrictions to SPS (27)"); - buf.put(header.data, header.offset, 25); - buf.put((byte) 0x04); - buf.put((byte) 0x78); - buf.put((byte) 0xc1); - buf.put((byte) 0x94); - spsLength = header.length + 2; - break; - default: - LimeLog.warning("Unknown SPS of length "+header.length); + if (!needsSpsNumRefFixup) { + switch (header.length) { + case 26: + LimeLog.info("Adding bitstream restrictions to SPS (26)"); + buf.put(header.data, header.offset, 24); + buf.put((byte) 0x11); + buf.put((byte) 0xe3); + buf.put((byte) 0x06); + buf.put((byte) 0x50); + spsLength = header.length + 2; + break; + case 27: + LimeLog.info("Adding bitstream restrictions to SPS (27)"); + buf.put(header.data, header.offset, 25); + buf.put((byte) 0x04); + buf.put((byte) 0x78); + buf.put((byte) 0xc1); + buf.put((byte) 0x94); + spsLength = header.length + 2; + break; + default: + LimeLog.warning("Unknown SPS of length "+header.length); + buf.put(header.data, header.offset, header.length); + spsLength = header.length; + break; + } + } + else { + // Set bitstream restrictions to only buffer single frame + // (starts 9 bits before stop bit and 6 bits earlier because of the shortening above) + this.replace(header, header.length*8+Integer.numberOfLeadingZeros(last & - last)%8-9-6, 2, BITSTREAM_RESTRICTIONS, 3*8); buf.put(header.data, header.offset, header.length); spsLength = header.length; - break; } + } else { buf.put(header.data, header.offset, header.length); @@ -295,7 +327,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { videoDecoder.queueInputBuffer(inputIndex, 0, spsLength, - 0, codecFlags); + currentTime, codecFlags); return true; } } @@ -308,7 +340,7 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { videoDecoder.queueInputBuffer(inputIndex, 0, decodeUnit.getDataLength(), - 0, codecFlags); + currentTime, codecFlags); } return true; @@ -397,4 +429,14 @@ public class MediaCodecDecoderRenderer implements VideoDecoderRenderer { source.offset = offset; source.length = length; } + + @Override + public int getAverageDecoderLatency() { + return (int)(decoderTimeMs / totalFrames); + } + + @Override + public int getAverageEndToEndLatency() { + return (int)(totalTimeMs / totalFrames); + } }