Merge pull request #716 from kevinxucs/kevinxucs/stats-overlay

Implement performance stats overlay

app/src/main/java/com/limelight/Game.java

@@ -13,6 +13,7 @@ import com.limelight.binding.input.virtual_controller.VirtualController;
 import com.limelight.binding.video.CrashListener;
 import com.limelight.binding.video.MediaCodecDecoderRenderer;
 import com.limelight.binding.video.MediaCodecHelper;
+import com.limelight.binding.video.PerfOverlayListener;
 import com.limelight.nvstream.NvConnection;
 import com.limelight.nvstream.NvConnectionListener;
 import com.limelight.nvstream.StreamConfiguration;
@@ -78,7 +79,8 @@ import java.util.Locale;
 
 public class Game extends Activity implements SurfaceHolder.Callback,
     OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener,
-    OnSystemUiVisibilityChangeListener, GameGestures, StreamView.InputCallbacks
+    OnSystemUiVisibilityChangeListener, GameGestures, StreamView.InputCallbacks,
+    PerfOverlayListener
 {
     private int lastMouseX = Integer.MIN_VALUE;
     private int lastMouseY = Integer.MIN_VALUE;
@@ -113,6 +115,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     private boolean grabComboDown = false;
     private StreamView streamView;
     private TextView notificationOverlayView;
+    private TextView performanceOverlayView;
 
     private ShortcutHelper shortcutHelper;
 
@@ -207,6 +210,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
         notificationOverlayView = findViewById(R.id.notificationOverlay);
 
+        performanceOverlayView = findViewById(R.id.performanceOverlay);
+
         inputCaptureProvider = InputCaptureManager.getInputCaptureProvider(this, this);
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
@@ -310,7 +315,14 @@ public class Game extends Activity implements SurfaceHolder.Callback,
             willStreamHdr = false;
         }
 
-        decoderRenderer = new MediaCodecDecoderRenderer(prefConfig,
+        // Check if the user has enabled performance stats overlay
+        if (prefConfig.enablePerfOverlay) {
+            performanceOverlayView.setVisibility(View.VISIBLE);
+        }
+
+        decoderRenderer = new MediaCodecDecoderRenderer(
+                this,
+                prefConfig,
                 new CrashListener() {
                     @Override
                     public void notifyCrash(Exception e) {
@@ -325,8 +337,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
                 tombstonePrefs.getInt("CrashCount", 0),
                 connMgr.isActiveNetworkMetered(),
                 willStreamHdr,
-                glPrefs.glRenderer
+                glPrefs.glRenderer,
-                );
+                this);
 
         // Don't stream HDR if the decoder can't support it
         if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported()) {
@@ -1574,4 +1586,14 @@ public class Game extends Activity implements SurfaceHolder.Callback,
             hideSystemUi(2000);
         }
     }
+
+    @Override
+    public void onPerfUpdate(final String text) {
+        runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                performanceOverlayView.setText(text);
+            }
+        });
+    }
 }

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -8,10 +8,12 @@ import org.jcodec.codecs.h264.io.model.SeqParameterSet;
 import org.jcodec.codecs.h264.io.model.VUIParameters;
 
 import com.limelight.LimeLog;
+import com.limelight.R;
 import com.limelight.nvstream.av.video.VideoDecoderRenderer;
 import com.limelight.nvstream.jni.MoonBridge;
 import com.limelight.preferences.PreferenceConfiguration;
 
+import android.content.Context;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
@@ -38,6 +40,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
     private boolean submittedCsd;
     private boolean submitCsdNextCall;
 
+    private Context context;
     private MediaCodec videoDecoder;
     private Thread rendererThread;
     private boolean needsSpsBitstreamFixup, isExynos4;
@@ -55,6 +58,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
     private String glRenderer;
     private boolean foreground = true;
     private boolean legacyFrameDropRendering = false;
+    private PerfOverlayListener perfListener;
 
     private boolean needsBaselineSpsHack;
     private SeqParameterSet savedSps;
@@ -63,13 +67,11 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
     private long initialExceptionTimestamp;
     private static final int EXCEPTION_REPORT_DELAY_MS = 3000;
 
+    private VideoStats activeWindowVideoStats;
+    private VideoStats lastWindowVideoStats;
+    private VideoStats globalVideoStats;
+
     private long lastTimestampUs;
-    private long decoderTimeMs;
-    private long totalTimeMs;
-    private int totalFramesReceived;
-    private int totalFramesRendered;
-    private int frameLossEvents;
-    private int framesLost;
     private int lastFrameNumber;
     private int refreshRate;
     private PreferenceConfiguration prefs;
@@ -119,16 +121,22 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
         this.renderTarget = renderTarget;
     }
 
-    public MediaCodecDecoderRenderer(PreferenceConfiguration prefs,
+    public MediaCodecDecoderRenderer(Context context, PreferenceConfiguration prefs,
                                      CrashListener crashListener, int consecutiveCrashCount,
                                      boolean meteredData, boolean requestedHdr,
-                                     String glRenderer) {
+                                     String glRenderer, PerfOverlayListener perfListener) {
         //dumpDecoders();
 
+        this.context = context;
         this.prefs = prefs;
         this.crashListener = crashListener;
         this.consecutiveCrashCount = consecutiveCrashCount;
         this.glRenderer = glRenderer;
+        this.perfListener = perfListener;
+
+        this.activeWindowVideoStats = new VideoStats();
+        this.lastWindowVideoStats = new VideoStats();
+        this.globalVideoStats = new VideoStats();
 
         avcDecoder = findAvcDecoder();
         if (avcDecoder != null) {
@@ -311,7 +319,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
                         long delta = (renderTimeNanos / 1000000L) - (presentationTimeUs / 1000);
                         if (delta >= 0 && delta < 1000) {
                             if (USE_FRAME_RENDER_TIME) {
-                                totalTimeMs += delta;
+                                activeWindowVideoStats.totalTimeMs += delta;
                             }
                         }
                     }
@@ -421,14 +429,14 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
                                 videoDecoder.releaseOutputBuffer(lastIndex, true);
                             }
 
-                            totalFramesRendered++;
+                            activeWindowVideoStats.totalFramesRendered++;
 
                             // Add delta time to the totals (excluding probable outliers)
                             long delta = MediaCodecHelper.getMonotonicMillis() - (presentationTimeUs / 1000);
                             if (delta >= 0 && delta < 1000) {
-                                decoderTimeMs += delta;
+                                activeWindowVideoStats.decoderTimeMs += delta;
                                 if (!USE_FRAME_RENDER_TIME) {
-                                    totalTimeMs += delta;
+                                    activeWindowVideoStats.totalTimeMs += delta;
                                 }
                             }
                         } else {
@@ -585,17 +593,57 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
             return MoonBridge.DR_OK;
         }
 
-        totalFramesReceived++;
+        if (lastFrameNumber == 0) {
-
+            activeWindowVideoStats.measurementStartTimestamp = System.currentTimeMillis();
+        } else if (frameNumber != lastFrameNumber && frameNumber != lastFrameNumber + 1) {
             // We can receive the same "frame" multiple times if it's an IDR frame.
             // In that case, each frame start NALU is submitted independently.
-        if (frameNumber != lastFrameNumber && frameNumber != lastFrameNumber + 1) {
+            activeWindowVideoStats.framesLost += frameNumber - lastFrameNumber - 1;
-            framesLost += frameNumber - lastFrameNumber - 1;
+            activeWindowVideoStats.totalFrames += frameNumber - lastFrameNumber - 1;
-            frameLossEvents++;
+            activeWindowVideoStats.frameLossEvents++;
         }
 
         lastFrameNumber = frameNumber;
 
+        // Flip stats windows roughly every second
+        if (System.currentTimeMillis() >= activeWindowVideoStats.measurementStartTimestamp + 1000) {
+            if (prefs.enablePerfOverlay) {
+                VideoStats lastTwo = new VideoStats();
+                lastTwo.add(lastWindowVideoStats);
+                lastTwo.add(activeWindowVideoStats);
+                VideoStatsFps fps = lastTwo.getFps();
+                String decoder;
+
+                if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H264) != 0) {
+                    decoder = avcDecoder.getName();
+                } else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) {
+                    decoder = hevcDecoder.getName();
+                } else {
+                    decoder = "(unknown)";
+                }
+
+                String perfText = context.getString(
+                        R.string.perf_overlay_text,
+                        initialWidth + "x" + initialHeight,
+                        decoder,
+                        fps.totalFps,
+                        fps.receivedFps,
+                        fps.renderedFps,
+                        (float)lastTwo.framesLost / lastTwo.totalFrames * 100,
+                        (float)lastTwo.totalTimeMs / lastTwo.totalFramesReceived,
+                        (float)lastTwo.decoderTimeMs / lastTwo.totalFramesReceived);
+                perfListener.onPerfUpdate(perfText);
+            }
+
+            globalVideoStats.add(activeWindowVideoStats);
+            lastWindowVideoStats.copy(activeWindowVideoStats);
+            activeWindowVideoStats.clear();
+            activeWindowVideoStats.measurementStartTimestamp = System.currentTimeMillis();
+        }
+
+        activeWindowVideoStats.totalFramesReceived++;
+        activeWindowVideoStats.totalFrames++;
+
         int inputBufferIndex;
         ByteBuffer buf;
 
@@ -603,7 +651,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
 
         if (!FRAME_RENDER_TIME_ONLY) {
             // Count time from first packet received to decode start
-            totalTimeMs += (timestampUs / 1000) - receiveTimeMs;
+            activeWindowVideoStats.totalTimeMs += (timestampUs / 1000) - receiveTimeMs;
         }
 
         if (timestampUs <= lastTimestampUs) {
@@ -910,17 +958,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
     }
 
     public int getAverageEndToEndLatency() {
-        if (totalFramesReceived == 0) {
+        if (globalVideoStats.totalFramesReceived == 0) {
             return 0;
         }
-        return (int)(totalTimeMs / totalFramesReceived);
+        return (int)(globalVideoStats.totalTimeMs / globalVideoStats.totalFramesReceived);
     }
 
     public int getAverageDecoderLatency() {
-        if (totalFramesReceived == 0) {
+        if (globalVideoStats.totalFramesReceived == 0) {
             return 0;
         }
-        return (int)(decoderTimeMs / totalFramesReceived);
+        return (int)(globalVideoStats.decoderTimeMs / globalVideoStats.totalFramesReceived);
     }
 
     static class DecoderHungException extends RuntimeException {
@@ -981,9 +1029,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
             str += "FPS target: "+renderer.refreshRate+"\n";
             str += "Bitrate: "+renderer.prefs.bitrate+" Kbps \n";
             str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+"\n";
-            str += "Total frames received: "+renderer.totalFramesReceived+"\n";
+            str += "Total frames received: "+renderer.globalVideoStats.totalFramesReceived+"\n";
-            str += "Total frames rendered: "+renderer.totalFramesRendered+"\n";
+            str += "Total frames rendered: "+renderer.globalVideoStats.totalFramesRendered+"\n";
-            str += "Frame losses: "+renderer.framesLost+" in "+renderer.frameLossEvents+" loss events\n";
+            str += "Frame losses: "+renderer.globalVideoStats.framesLost+" in "+renderer.globalVideoStats.frameLossEvents+" loss events\n";
             str += "Average end-to-end client latency: "+renderer.getAverageEndToEndLatency()+"ms\n";
             str += "Average hardware decoder latency: "+renderer.getAverageDecoderLatency()+"ms\n";
 

app/src/main/java/com/limelight/binding/video/PerfOverlayListener.java

@@ -0,0 +1,5 @@
+package com.limelight.binding.video;
+
+public interface PerfOverlayListener {
+    void onPerfUpdate(final String text);
+}

app/src/main/java/com/limelight/binding/video/VideoStats.java

@@ -0,0 +1,70 @@
+package com.limelight.binding.video;
+
+class VideoStats {
+
+    long decoderTimeMs;
+    long totalTimeMs;
+    int totalFrames;
+    int totalFramesReceived;
+    int totalFramesRendered;
+    int frameLossEvents;
+    int framesLost;
+    long measurementStartTimestamp;
+
+    void add(VideoStats other) {
+        this.decoderTimeMs += other.decoderTimeMs;
+        this.totalTimeMs += other.totalTimeMs;
+        this.totalFrames += other.totalFrames;
+        this.totalFramesReceived += other.totalFramesReceived;
+        this.totalFramesRendered += other.totalFramesRendered;
+        this.frameLossEvents += other.frameLossEvents;
+        this.framesLost += other.framesLost;
+
+        if (this.measurementStartTimestamp == 0) {
+            this.measurementStartTimestamp = other.measurementStartTimestamp;
+        }
+
+        assert other.measurementStartTimestamp <= this.measurementStartTimestamp;
+    }
+
+    void copy(VideoStats other) {
+        this.decoderTimeMs = other.decoderTimeMs;
+        this.totalTimeMs = other.totalTimeMs;
+        this.totalFrames = other.totalFrames;
+        this.totalFramesReceived = other.totalFramesReceived;
+        this.totalFramesRendered = other.totalFramesRendered;
+        this.frameLossEvents = other.frameLossEvents;
+        this.framesLost = other.framesLost;
+        this.measurementStartTimestamp = other.measurementStartTimestamp;
+    }
+
+    void clear() {
+        this.decoderTimeMs = 0;
+        this.totalTimeMs = 0;
+        this.totalFrames = 0;
+        this.totalFramesReceived = 0;
+        this.totalFramesRendered = 0;
+        this.frameLossEvents = 0;
+        this.framesLost = 0;
+        this.measurementStartTimestamp = 0;
+    }
+
+    VideoStatsFps getFps() {
+        float elapsed = (System.currentTimeMillis() - this.measurementStartTimestamp) / (float) 1000;
+
+        VideoStatsFps fps = new VideoStatsFps();
+        if (elapsed > 0) {
+            fps.totalFps = this.totalFrames / elapsed;
+            fps.receivedFps = this.totalFramesReceived / elapsed;
+            fps.renderedFps = this.totalFramesRendered / elapsed;
+        }
+        return fps;
+    }
+}
+
+class VideoStatsFps {
+
+    float totalFps;
+    float receivedFps;
+    float renderedFps;
+}

app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java

@@ -31,6 +31,7 @@ public class PreferenceConfiguration {
     private static final String DISABLE_FRAME_DROP_PREF_STRING = "checkbox_disable_frame_drop";
     private static final String ENABLE_HDR_PREF_STRING = "checkbox_enable_hdr";
     private static final String ENABLE_PIP_PREF_STRING = "checkbox_enable_pip";
+    private static final String ENABLE_PERF_OVERLAY_STRING = "checkbox_enable_perf_overlay";
     private static final String BIND_ALL_USB_STRING = "checkbox_usb_bind_all";
     private static final String MOUSE_EMULATION_STRING = "checkbox_mouse_emulation";
     private static final String MOUSE_NAV_BUTTONS_STRING = "checkbox_mouse_nav_buttons";
@@ -56,6 +57,7 @@ public class PreferenceConfiguration {
     private static final boolean DEFAULT_DISABLE_FRAME_DROP = false;
     private static final boolean DEFAULT_ENABLE_HDR = false;
     private static final boolean DEFAULT_ENABLE_PIP = false;
+    private static final boolean DEFAULT_ENABLE_PERF_OVERLAY = false;
     private static final boolean DEFAULT_BIND_ALL_USB = false;
     private static final boolean DEFAULT_MOUSE_EMULATION = true;
     private static final boolean DEFAULT_MOUSE_NAV_BUTTONS = false;
@@ -79,6 +81,7 @@ public class PreferenceConfiguration {
     public boolean disableFrameDrop;
     public boolean enableHdr;
     public boolean enablePip;
+    public boolean enablePerfOverlay;
     public boolean bindAllUsb;
     public boolean mouseEmulation;
     public boolean mouseNavButtons;
@@ -331,6 +334,7 @@ public class PreferenceConfiguration {
         config.disableFrameDrop = prefs.getBoolean(DISABLE_FRAME_DROP_PREF_STRING, DEFAULT_DISABLE_FRAME_DROP);
         config.enableHdr = prefs.getBoolean(ENABLE_HDR_PREF_STRING, DEFAULT_ENABLE_HDR);
         config.enablePip = prefs.getBoolean(ENABLE_PIP_PREF_STRING, DEFAULT_ENABLE_PIP);
+        config.enablePerfOverlay = prefs.getBoolean(ENABLE_PERF_OVERLAY_STRING, DEFAULT_ENABLE_PERF_OVERLAY);
         config.bindAllUsb = prefs.getBoolean(BIND_ALL_USB_STRING, DEFAULT_BIND_ALL_USB);
         config.mouseEmulation = prefs.getBoolean(MOUSE_EMULATION_STRING, DEFAULT_MOUSE_EMULATION);
         config.mouseNavButtons = prefs.getBoolean(MOUSE_NAV_BUTTONS_STRING, DEFAULT_MOUSE_NAV_BUTTONS);

app/src/main/res/layout/activity_game.xml

@@ -10,6 +10,17 @@
         android:layout_height="match_parent"
         android:layout_gravity="center" />
 
+    <TextView
+        android:id="@+id/performanceOverlay"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_marginLeft="3dp"
+        android:layout_gravity="left"
+        android:textAppearance="?android:attr/textAppearanceSmall"
+        android:textSize="@dimen/font_size_tiny"
+        android:gravity="left"
+        android:visibility="gone" />
+
     <TextView
         android:id="@+id/notificationOverlay"
         android:layout_width="wrap_content"

app/src/main/res/values/dimens.xml

@@ -4,4 +4,7 @@
     <dimen name="activity_horizontal_margin">16dp</dimen>
     <dimen name="activity_vertical_margin">16dp</dimen>
 
+    <!-- General sizes -->
+    <dimen name="font_size_tiny">8sp</dimen>
+
 </resources>

app/src/main/res/values/strings.xml

@@ -86,6 +86,7 @@
     <string name="delete_pc_msg">Are you sure you want to delete this PC?</string>
     <string name="slow_connection_msg">Slow connection to PC\nReduce your bitrate</string>
     <string name="poor_connection_msg">Poor connection to PC</string>
+    <string name="perf_overlay_text">Video dimensions: %1$s\nDecoder: %2$s\nEstimated host PC frame rate: %3$.2f FPS\nIncoming frame rate from network: %4$.2f FPS\nRendering frame rate: %5$.2f FPS\nFrames dropped by your network connection: %6$.2f%%\nAverage frame time: %7$.2f ms\nAverage decoding time: %8$.2f ms</string>
 
     <!-- AppList activity -->
     <string name="applist_connect_msg">Connecting to PC…</string>
@@ -185,5 +186,7 @@
     <string name="summary_video_format">H.265 lowers video bandwidth requirements but requires a very recent device</string>
     <string name="title_enable_hdr">Enable HDR (Experimental)</string>
     <string name="summary_enable_hdr">Stream HDR when the game and PC GPU support it. HDR requires a GTX 1000 series GPU or later.</string>
+    <string name="title_enable_perf_overlay">Enable performance overlay</string>
+    <string name="summary_enable_perf_overlay">Display performance stats overlay</string>
 
 </resources>

app/src/main/res/xml/preferences.xml

@@ -171,5 +171,10 @@
             android:title="@string/title_enable_hdr"
             android:summary="@string/summary_enable_hdr"
             android:defaultValue="false" />
+        <CheckBoxPreference
+            android:key="checkbox_enable_perf_overlay"
+            android:title="@string/title_enable_perf_overlay"
+            android:summary="@string/summary_enable_perf_overlay"
+            android:defaultValue="false"/>
     </PreferenceCategory>
 </PreferenceScreen>