Request unbuffered input events to reduce input latency

app/src/main/java/com/limelight/Game.java

@@ -241,6 +241,24 @@ public class Game extends Activity implements SurfaceHolder.Callback,
         streamView.setOnTouchListener(this);
         streamView.setInputCallbacks(this);
 
+        boolean needsInputBatching = false;
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+            // Request unbuffered input event dispatching for all input classes we handle here.
+            // Without this, input events are buffered to be delivered in lock-step with VBlank,
+            // artificially increasing input latency while streaming.
+            streamView.requestUnbufferedDispatch(
+                    InputDevice.SOURCE_CLASS_BUTTON | // Keyboards
+                    InputDevice.SOURCE_CLASS_JOYSTICK | // Gamepads
+                    InputDevice.SOURCE_CLASS_POINTER | // Touchscreens and mice (w/o pointer capture)
+                    InputDevice.SOURCE_CLASS_POSITION | // Touchpads
+                    InputDevice.SOURCE_CLASS_TRACKBALL // Mice (pointer capture)
+            );
+
+            // Since the OS isn't going to batch for us, we have to batch mouse events to
+            // avoid triggering a bug in GeForce Experience that can lead to massive latency.
+            needsInputBatching = true;
+        }
+
         notificationOverlayView = findViewById(R.id.notificationOverlay);
 
         performanceOverlayView = findViewById(R.id.performanceOverlay);
@@ -453,7 +471,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
                 .build();
 
         // Initialize the connection
-        conn = new NvConnection(host, uniqueId, config, PlatformBinding.getCryptoProvider(this), serverCert);
+        conn = new NvConnection(host, uniqueId, config, PlatformBinding.getCryptoProvider(this), serverCert, needsInputBatching);
         controllerHandler = new ControllerHandler(this, conn, this, prefConfig);
         keyboardTranslator = new KeyboardTranslator();
 
@@ -1652,6 +1670,12 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     @SuppressLint("ClickableViewAccessibility")
     @Override
     public boolean onTouch(View view, MotionEvent event) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            if (event.getAction() == MotionEvent.ACTION_DOWN) {
+                // Tell the OS not to buffer input events for us
+                view.requestUnbufferedDispatch(event);
+            }
+        }
         return handleMotionEvent(view, event);
     }
 

app/src/main/java/com/limelight/nvstream/NvConnection.java

@@ -7,6 +7,8 @@ import java.nio.ByteBuffer;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.concurrent.Semaphore;
 
 import javax.crypto.KeyGenerator;
@@ -33,12 +35,20 @@ public class NvConnection {
     private ConnectionContext context;
     private static Semaphore connectionAllowed = new Semaphore(1);
     private final boolean isMonkey;
+    private final boolean batchMouseInput;
 
-    public NvConnection(String host, String uniqueId, StreamConfiguration config, LimelightCryptoProvider cryptoProvider, X509Certificate serverCert)
+    private static final int MOUSE_BATCH_PERIOD_MS = 5;
+    private Timer mouseInputTimer;
+    private final Object mouseInputLock = new Object();
+    private short relMouseX, relMouseY, relMouseWidth, relMouseHeight;
+    private short absMouseX, absMouseY, absMouseWidth, absMouseHeight;
+
+    public NvConnection(String host, String uniqueId, StreamConfiguration config, LimelightCryptoProvider cryptoProvider, X509Certificate serverCert, boolean batchMouseInput)
     {       
         this.host = host;
         this.cryptoProvider = cryptoProvider;
         this.uniqueId = uniqueId;
+        this.batchMouseInput = batchMouseInput;
 
         this.context = new ConnectionContext();
         this.context.streamConfig = config;
@@ -70,6 +80,11 @@ public class NvConnection {
     }
 
     public void stop() {
+        // Stop sending additional input
+        if (mouseInputTimer != null) {
+            mouseInputTimer.cancel();
+        }
+
         // Interrupt any pending connection. This is thread-safe.
         MoonBridge.interruptConnection();
 
@@ -84,6 +99,24 @@ public class NvConnection {
         connectionAllowed.release();
     }
 
+    private void flushMousePosition() {
+        synchronized (mouseInputLock) {
+            if (relMouseX != 0 || relMouseY != 0) {
+                if (relMouseWidth != 0 || relMouseHeight != 0) {
+                    MoonBridge.sendMouseMoveAsMousePosition(relMouseX, relMouseY, relMouseWidth, relMouseHeight);
+                }
+                else {
+                    MoonBridge.sendMouseMove(relMouseX, relMouseY);
+                }
+                relMouseX = relMouseY = relMouseWidth = relMouseHeight = 0;
+            }
+            if (absMouseX != 0 || absMouseY != 0 || absMouseWidth != 0 || absMouseHeight != 0) {
+                MoonBridge.sendMousePosition(absMouseX, absMouseY, absMouseWidth, absMouseHeight);
+                absMouseX = absMouseY = absMouseWidth = absMouseHeight = 0;
+            }
+        }
+    }
+    
     private boolean startApp() throws XmlPullParserException, IOException
     {
         NvHTTP h = new NvHTTP(context.serverAddress, uniqueId, context.serverCert, cryptoProvider);
@@ -292,6 +325,21 @@ public class NvConnection {
                         // to stop the connection themselves. We need to release their
                         // semaphore count for them.
                         connectionAllowed.release();
+                        return;
+                    }
+
+                    if (batchMouseInput) {
+                        // High polling rate mice can cause GeForce Experience's input queue to get backed up,
+                        // causing massive input latency. We counter this by limiting our mouse events to 200 Hz
+                        // which appears to avoid triggering the issue on all known configurations.
+                        mouseInputTimer = new Timer("MouseInput", true);
+                        mouseInputTimer.schedule(new TimerTask() {
+                            @Override
+                            public void run() {
+                                // Flush the mouse position every 5 ms
+                                flushMousePosition();
+                            }
+                        }, MOUSE_BATCH_PERIOD_MS, MOUSE_BATCH_PERIOD_MS);
                     }
                 }
             }
@@ -301,27 +349,65 @@ public class NvConnection {
     public void sendMouseMove(final short deltaX, final short deltaY)
     {
         if (!isMonkey) {
-            MoonBridge.sendMouseMove(deltaX, deltaY);
+            synchronized (mouseInputLock) {
+                relMouseX += deltaX;
+                relMouseY += deltaY;
+
+                // Reset these to ensure we don't send this as a position update
+                relMouseWidth = 0;
+                relMouseHeight = 0;
+            }
+
+            if (!batchMouseInput) {
+                flushMousePosition();
+            }
         }
     }
 
     public void sendMousePosition(short x, short y, short referenceWidth, short referenceHeight)
     {
         if (!isMonkey) {
-            MoonBridge.sendMousePosition(x, y, referenceWidth, referenceHeight);
+            synchronized (mouseInputLock) {
+                absMouseX = x;
+                absMouseY = y;
+                absMouseWidth = referenceWidth;
+                absMouseHeight = referenceHeight;
+            }
+
+            if (!batchMouseInput) {
+                flushMousePosition();
+            }
         }
     }
 
     public void sendMouseMoveAsMousePosition(short deltaX, short deltaY, short referenceWidth, short referenceHeight)
     {
         if (!isMonkey) {
-            MoonBridge.sendMouseMoveAsMousePosition(deltaX, deltaY, referenceWidth, referenceHeight);
+            synchronized (mouseInputLock) {
+                // Only accumulate the delta if the reference size is the same
+                if (relMouseWidth == referenceWidth && relMouseHeight == referenceHeight) {
+                    relMouseX += deltaX;
+                    relMouseY += deltaY;
+                }
+                else {
+                    relMouseX = deltaX;
+                    relMouseY = deltaY;
+                }
+
+                relMouseWidth = referenceWidth;
+                relMouseHeight = referenceHeight;
+            }
+
+            if (!batchMouseInput) {
+                flushMousePosition();
+            }
         }
     }
 
     public void sendMouseButtonDown(final byte mouseButton)
     {
         if (!isMonkey) {
+            flushMousePosition();
             MoonBridge.sendMouseButton(MouseButtonPacket.PRESS_EVENT, mouseButton);
         }
     }
@@ -329,6 +415,7 @@ public class NvConnection {
     public void sendMouseButtonUp(final byte mouseButton)
     {
         if (!isMonkey) {
+            flushMousePosition();
             MoonBridge.sendMouseButton(MouseButtonPacket.RELEASE_EVENT, mouseButton);
         }
     }
@@ -364,12 +451,14 @@ public class NvConnection {
     
     public void sendMouseScroll(final byte scrollClicks) {
         if (!isMonkey) {
+            flushMousePosition();
             MoonBridge.sendMouseScroll(scrollClicks);
         }
     }
 
     public void sendMouseHighResScroll(final short scrollAmount) {
         if (!isMonkey) {
+            flushMousePosition();
             MoonBridge.sendMouseHighResScroll(scrollAmount);
         }
     }