Add host processing latency to performance stats overlay

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -1292,7 +1292,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
     @SuppressWarnings("deprecation")
     @Override
     public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType,
-                                int frameNumber, int frameType, long receiveTimeMs, long enqueueTimeMs) {
+                                int frameNumber, int frameType, char frameHostProcessingLatency,
+                                long receiveTimeMs, long enqueueTimeMs) {
         if (stopping) {
             // Don't bother if we're stopping
             return MoonBridge.DR_OK;
@@ -1334,6 +1335,10 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
                 sb.append(context.getString(R.string.perf_overlay_decoder, decoder)).append('\n');
                 sb.append(context.getString(R.string.perf_overlay_incomingfps, fps.receivedFps)).append('\n');
                 sb.append(context.getString(R.string.perf_overlay_renderingfps, fps.renderedFps)).append('\n');
+                sb.append(context.getString(R.string.perf_overlay_hostprocessinglatency,
+                        (float)lastTwo.minHostProcessingLatency / 10,
+                        (float)lastTwo.maxHostProcessingLatency / 10,
+                        (float)lastTwo.totalHostProcessingLatency / 10 / Math.max(lastTwo.framesWithHostProcessingLatency, 1))).append('\n');
                 sb.append(context.getString(R.string.perf_overlay_netdrops,
                         (float)lastTwo.framesLost / lastTwo.totalFrames * 100)).append('\n');
                 sb.append(context.getString(R.string.perf_overlay_netlatency,
@@ -1533,6 +1538,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
             }
         }
         else {
+            if (frameHostProcessingLatency != 0) {
+                if (activeWindowVideoStats.minHostProcessingLatency != 0) {
+                    activeWindowVideoStats.minHostProcessingLatency = (char) Math.min(activeWindowVideoStats.minHostProcessingLatency, frameHostProcessingLatency);
+                } else {
+                    activeWindowVideoStats.minHostProcessingLatency = frameHostProcessingLatency;
+                }
+                activeWindowVideoStats.framesWithHostProcessingLatency += 1;
+            }
+            activeWindowVideoStats.maxHostProcessingLatency = (char) Math.max(activeWindowVideoStats.maxHostProcessingLatency, frameHostProcessingLatency);
+            activeWindowVideoStats.totalHostProcessingLatency += frameHostProcessingLatency;
+
             activeWindowVideoStats.totalFramesReceived++;
             activeWindowVideoStats.totalFrames++;
 

app/src/main/java/com/limelight/binding/video/VideoStats.java

@@ -11,6 +11,10 @@ class VideoStats {
     int totalFramesRendered;
     int frameLossEvents;
     int framesLost;
+    char minHostProcessingLatency;
+    char maxHostProcessingLatency;
+    int totalHostProcessingLatency;
+    int framesWithHostProcessingLatency;
     long measurementStartTimestamp;
 
     void add(VideoStats other) {
@@ -22,6 +26,15 @@ class VideoStats {
         this.frameLossEvents += other.frameLossEvents;
         this.framesLost += other.framesLost;
 
+        if (this.minHostProcessingLatency == 0) {
+            this.minHostProcessingLatency = other.minHostProcessingLatency;
+        } else {
+            this.minHostProcessingLatency = (char) Math.min(this.minHostProcessingLatency, other.minHostProcessingLatency);
+        }
+        this.maxHostProcessingLatency = (char) Math.max(this.maxHostProcessingLatency, other.maxHostProcessingLatency);
+        this.totalHostProcessingLatency += other.totalHostProcessingLatency;
+        this.framesWithHostProcessingLatency += other.framesWithHostProcessingLatency;
+
         if (this.measurementStartTimestamp == 0) {
             this.measurementStartTimestamp = other.measurementStartTimestamp;
         }
@@ -37,6 +50,10 @@ class VideoStats {
         this.totalFramesRendered = other.totalFramesRendered;
         this.frameLossEvents = other.frameLossEvents;
         this.framesLost = other.framesLost;
+        this.minHostProcessingLatency = other.minHostProcessingLatency;
+        this.maxHostProcessingLatency = other.maxHostProcessingLatency;
+        this.totalHostProcessingLatency = other.totalHostProcessingLatency;
+        this.framesWithHostProcessingLatency = other.framesWithHostProcessingLatency;
         this.measurementStartTimestamp = other.measurementStartTimestamp;
     }
 
@@ -48,6 +65,10 @@ class VideoStats {
         this.totalFramesRendered = 0;
         this.frameLossEvents = 0;
         this.framesLost = 0;
+        this.minHostProcessingLatency = 0;
+        this.maxHostProcessingLatency = 0;
+        this.totalHostProcessingLatency = 0;
+        this.framesWithHostProcessingLatency = 0;
         this.measurementStartTimestamp = 0;
     }
 

app/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java

@@ -10,7 +10,8 @@ public abstract class VideoDecoderRenderer {
     // This is called once for each frame-start NALU. This means it will be called several times
     // for an IDR frame which contains several parameter sets and the I-frame data.
     public abstract int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType,
-                                         int frameNumber, int frameType, long receiveTimeMs, long enqueueTimeMs);
+                                         int frameNumber, int frameType, char frameHostProcessingLatency,
+                                         long receiveTimeMs, long enqueueTimeMs);
     
     public abstract void cleanup();
 

app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java

@@ -167,11 +167,11 @@ public class MoonBridge {
     }
 
     public static int bridgeDrSubmitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType,
-                                               int frameNumber, int frameType,
+                                               int frameNumber, int frameType, char frameHostProcessingLatency,
                                                long receiveTimeMs, long enqueueTimeMs) {
         if (videoRenderer != null) {
             return videoRenderer.submitDecodeUnit(decodeUnitData, decodeUnitLength,
-                    decodeUnitType, frameNumber, frameType, receiveTimeMs, enqueueTimeMs);
+                    decodeUnitType, frameNumber, frameType, frameHostProcessingLatency, receiveTimeMs, enqueueTimeMs);
         }
         else {
             return DR_OK;

app/src/main/jni/moonlight-core/callbacks.c

@@ -80,7 +80,7 @@ Java_com_limelight_nvstream_jni_MoonBridge_init(JNIEnv *env, jclass clazz) {
     BridgeDrStartMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrStart", "()V");
     BridgeDrStopMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrStop", "()V");
     BridgeDrCleanupMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrCleanup", "()V");
-    BridgeDrSubmitDecodeUnitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrSubmitDecodeUnit", "([BIIIIJJ)I");
+    BridgeDrSubmitDecodeUnitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrSubmitDecodeUnit", "([BIIIICJJ)I");
     BridgeArInitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArInit", "(III)I");
     BridgeArStartMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArStart", "()V");
     BridgeArStopMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArStop", "()V");
@@ -159,7 +159,7 @@ int BridgeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) {
 
             ret = (*env)->CallStaticIntMethod(env, GlobalBridgeClass, BridgeDrSubmitDecodeUnitMethod,
                                               DecodedFrameBuffer, currentEntry->length, currentEntry->bufferType,
-                                              decodeUnit->frameNumber, decodeUnit->frameType,
+                                              decodeUnit->frameNumber, decodeUnit->frameType, (jchar)decodeUnit->frameHostProcessingLatency,
                                               (jlong)decodeUnit->receiveTimeMs, (jlong)decodeUnit->enqueueTimeMs);
             if ((*env)->ExceptionCheck(env)) {
                 // We will crash here
@@ -180,7 +180,7 @@ int BridgeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) {
 
     ret = (*env)->CallStaticIntMethod(env, GlobalBridgeClass, BridgeDrSubmitDecodeUnitMethod,
                                        DecodedFrameBuffer, offset, BUFFER_TYPE_PICDATA,
-                                       decodeUnit->frameNumber, decodeUnit->frameType,
+                                       decodeUnit->frameNumber, decodeUnit->frameType, (jchar)decodeUnit->frameHostProcessingLatency,
                                        (jlong)decodeUnit->receiveTimeMs, (jlong)decodeUnit->enqueueTimeMs);
     if ((*env)->ExceptionCheck(env)) {
         // We will crash here

app/src/main/jni/moonlight-core/moonlight-common-c

@@ -1 +1 @@
-Subproject commit b77072d39984019b4234d9f2adc3b673b6d57812
+Subproject commit 284840bde75c0e30dd72aaa5e4e136dedf084ee1

app/src/main/res/values/strings.xml

@@ -108,6 +108,7 @@
     <string name="perf_overlay_decoder">Decoder: %1$s</string>
     <string name="perf_overlay_incomingfps">Incoming frame rate from network: %1$.2f FPS</string>
     <string name="perf_overlay_renderingfps">Rendering frame rate: %1$.2f FPS</string>
+    <string name="perf_overlay_hostprocessinglatency">Host processing latency min/max/average: %1$.1f/%2$.1f/%3$.1f ms</string>
     <string name="perf_overlay_netdrops">Frames dropped by your network connection: %1$.2f%%</string>
     <string name="perf_overlay_netlatency">Average network latency: %1$d ms (variance: %2$d ms)</string>
     <string name="perf_overlay_dectime">Average decoding time: %1$.2f ms</string>