From b65369486016973bb79ee87c50e16315e91d5bc3 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 6 Sep 2022 19:09:31 -0500 Subject: [PATCH] Request unbuffered input events to reduce input latency --- app/src/main/java/com/limelight/Game.java | 26 ++++- .../com/limelight/nvstream/NvConnection.java | 99 ++++++++++++++++++- 2 files changed, 119 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 91a1f9f1..575986cd 100644 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -241,6 +241,24 @@ public class Game extends Activity implements SurfaceHolder.Callback, streamView.setOnTouchListener(this); streamView.setInputCallbacks(this); + boolean needsInputBatching = false; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + // Request unbuffered input event dispatching for all input classes we handle here. + // Without this, input events are buffered to be delivered in lock-step with VBlank, + // artificially increasing input latency while streaming. + streamView.requestUnbufferedDispatch( + InputDevice.SOURCE_CLASS_BUTTON | // Keyboards + InputDevice.SOURCE_CLASS_JOYSTICK | // Gamepads + InputDevice.SOURCE_CLASS_POINTER | // Touchscreens and mice (w/o pointer capture) + InputDevice.SOURCE_CLASS_POSITION | // Touchpads + InputDevice.SOURCE_CLASS_TRACKBALL // Mice (pointer capture) + ); + + // Since the OS isn't going to batch for us, we have to batch mouse events to + // avoid triggering a bug in GeForce Experience that can lead to massive latency. + needsInputBatching = true; + } + notificationOverlayView = findViewById(R.id.notificationOverlay); performanceOverlayView = findViewById(R.id.performanceOverlay); @@ -453,7 +471,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, .build(); // Initialize the connection - conn = new NvConnection(host, uniqueId, config, PlatformBinding.getCryptoProvider(this), serverCert); + conn = new NvConnection(host, uniqueId, config, PlatformBinding.getCryptoProvider(this), serverCert, needsInputBatching); controllerHandler = new ControllerHandler(this, conn, this, prefConfig); keyboardTranslator = new KeyboardTranslator(); @@ -1652,6 +1670,12 @@ public class Game extends Activity implements SurfaceHolder.Callback, @SuppressLint("ClickableViewAccessibility") @Override public boolean onTouch(View view, MotionEvent event) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + if (event.getAction() == MotionEvent.ACTION_DOWN) { + // Tell the OS not to buffer input events for us + view.requestUnbufferedDispatch(event); + } + } return handleMotionEvent(view, event); } diff --git a/app/src/main/java/com/limelight/nvstream/NvConnection.java b/app/src/main/java/com/limelight/nvstream/NvConnection.java index 588a846c..b99be0df 100644 --- a/app/src/main/java/com/limelight/nvstream/NvConnection.java +++ b/app/src/main/java/com/limelight/nvstream/NvConnection.java @@ -7,6 +7,8 @@ import java.nio.ByteBuffer; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.X509Certificate; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.Semaphore; import javax.crypto.KeyGenerator; @@ -33,12 +35,20 @@ public class NvConnection { private ConnectionContext context; private static Semaphore connectionAllowed = new Semaphore(1); private final boolean isMonkey; - - public NvConnection(String host, String uniqueId, StreamConfiguration config, LimelightCryptoProvider cryptoProvider, X509Certificate serverCert) + private final boolean batchMouseInput; + + private static final int MOUSE_BATCH_PERIOD_MS = 5; + private Timer mouseInputTimer; + private final Object mouseInputLock = new Object(); + private short relMouseX, relMouseY, relMouseWidth, relMouseHeight; + private short absMouseX, absMouseY, absMouseWidth, absMouseHeight; + + public NvConnection(String host, String uniqueId, StreamConfiguration config, LimelightCryptoProvider cryptoProvider, X509Certificate serverCert, boolean batchMouseInput) { this.host = host; this.cryptoProvider = cryptoProvider; this.uniqueId = uniqueId; + this.batchMouseInput = batchMouseInput; this.context = new ConnectionContext(); this.context.streamConfig = config; @@ -70,6 +80,11 @@ public class NvConnection { } public void stop() { + // Stop sending additional input + if (mouseInputTimer != null) { + mouseInputTimer.cancel(); + } + // Interrupt any pending connection. This is thread-safe. MoonBridge.interruptConnection(); @@ -83,6 +98,24 @@ public class NvConnection { // Now a pending connection can be processed connectionAllowed.release(); } + + private void flushMousePosition() { + synchronized (mouseInputLock) { + if (relMouseX != 0 || relMouseY != 0) { + if (relMouseWidth != 0 || relMouseHeight != 0) { + MoonBridge.sendMouseMoveAsMousePosition(relMouseX, relMouseY, relMouseWidth, relMouseHeight); + } + else { + MoonBridge.sendMouseMove(relMouseX, relMouseY); + } + relMouseX = relMouseY = relMouseWidth = relMouseHeight = 0; + } + if (absMouseX != 0 || absMouseY != 0 || absMouseWidth != 0 || absMouseHeight != 0) { + MoonBridge.sendMousePosition(absMouseX, absMouseY, absMouseWidth, absMouseHeight); + absMouseX = absMouseY = absMouseWidth = absMouseHeight = 0; + } + } + } private boolean startApp() throws XmlPullParserException, IOException { @@ -292,6 +325,21 @@ public class NvConnection { // to stop the connection themselves. We need to release their // semaphore count for them. connectionAllowed.release(); + return; + } + + if (batchMouseInput) { + // High polling rate mice can cause GeForce Experience's input queue to get backed up, + // causing massive input latency. We counter this by limiting our mouse events to 200 Hz + // which appears to avoid triggering the issue on all known configurations. + mouseInputTimer = new Timer("MouseInput", true); + mouseInputTimer.schedule(new TimerTask() { + @Override + public void run() { + // Flush the mouse position every 5 ms + flushMousePosition(); + } + }, MOUSE_BATCH_PERIOD_MS, MOUSE_BATCH_PERIOD_MS); } } } @@ -301,27 +349,65 @@ public class NvConnection { public void sendMouseMove(final short deltaX, final short deltaY) { if (!isMonkey) { - MoonBridge.sendMouseMove(deltaX, deltaY); + synchronized (mouseInputLock) { + relMouseX += deltaX; + relMouseY += deltaY; + + // Reset these to ensure we don't send this as a position update + relMouseWidth = 0; + relMouseHeight = 0; + } + + if (!batchMouseInput) { + flushMousePosition(); + } } } public void sendMousePosition(short x, short y, short referenceWidth, short referenceHeight) { if (!isMonkey) { - MoonBridge.sendMousePosition(x, y, referenceWidth, referenceHeight); + synchronized (mouseInputLock) { + absMouseX = x; + absMouseY = y; + absMouseWidth = referenceWidth; + absMouseHeight = referenceHeight; + } + + if (!batchMouseInput) { + flushMousePosition(); + } } } public void sendMouseMoveAsMousePosition(short deltaX, short deltaY, short referenceWidth, short referenceHeight) { if (!isMonkey) { - MoonBridge.sendMouseMoveAsMousePosition(deltaX, deltaY, referenceWidth, referenceHeight); + synchronized (mouseInputLock) { + // Only accumulate the delta if the reference size is the same + if (relMouseWidth == referenceWidth && relMouseHeight == referenceHeight) { + relMouseX += deltaX; + relMouseY += deltaY; + } + else { + relMouseX = deltaX; + relMouseY = deltaY; + } + + relMouseWidth = referenceWidth; + relMouseHeight = referenceHeight; + } + + if (!batchMouseInput) { + flushMousePosition(); + } } } public void sendMouseButtonDown(final byte mouseButton) { if (!isMonkey) { + flushMousePosition(); MoonBridge.sendMouseButton(MouseButtonPacket.PRESS_EVENT, mouseButton); } } @@ -329,6 +415,7 @@ public class NvConnection { public void sendMouseButtonUp(final byte mouseButton) { if (!isMonkey) { + flushMousePosition(); MoonBridge.sendMouseButton(MouseButtonPacket.RELEASE_EVENT, mouseButton); } } @@ -364,12 +451,14 @@ public class NvConnection { public void sendMouseScroll(final byte scrollClicks) { if (!isMonkey) { + flushMousePosition(); MoonBridge.sendMouseScroll(scrollClicks); } } public void sendMouseHighResScroll(final short scrollAmount) { if (!isMonkey) { + flushMousePosition(); MoonBridge.sendMouseHighResScroll(scrollAmount); } }