From 67b2853ef037b03660cc9a8a71129ea16b39a09f Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sat, 22 Jul 2023 17:18:57 -0500 Subject: [PATCH] Add contact area and orientation for pen/touch events --- app/src/main/java/com/limelight/Game.java | 112 ++++++++++++++++-- .../com/limelight/nvstream/NvConnection.java | 12 +- .../limelight/nvstream/jni/MoonBridge.java | 6 +- .../jni/moonlight-core/moonlight-common-c | 2 +- app/src/main/jni/moonlight-core/simplejni.c | 13 +- 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 2aefb986..c2cba6b0 100644 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -1532,6 +1532,100 @@ public class Game extends Activity implements SurfaceHolder.Callback, return new float[] { normalizedX, normalizedY }; } + private static float normalizeValueInRange(float value, InputDevice.MotionRange range) { + return (value - range.getMin()) / range.getRange(); + } + + private static float getPressureOrDistance(MotionEvent event) { + InputDevice dev = event.getDevice(); + switch (event.getActionMasked()) { + case MotionEvent.ACTION_HOVER_ENTER: + case MotionEvent.ACTION_HOVER_MOVE: + case MotionEvent.ACTION_HOVER_EXIT: + // Hover events report distance + if (dev != null) { + InputDevice.MotionRange distanceRange = dev.getMotionRange(MotionEvent.AXIS_DISTANCE, event.getSource()); + if (distanceRange != null) { + return normalizeValueInRange(event.getAxisValue(MotionEvent.AXIS_DISTANCE, event.getActionIndex()), distanceRange); + } + } + return 0.0f; + + default: + // Other events report pressure + return event.getPressure(event.getActionIndex()); + } + } + + private static short getRotationDegrees(MotionEvent event) { + InputDevice dev = event.getDevice(); + if (dev != null) { + if (dev.getMotionRange(MotionEvent.AXIS_ORIENTATION, event.getSource()) != null) { + short rotationDegrees = (short) Math.toDegrees(event.getOrientation(event.getActionIndex())); + if (rotationDegrees < 0) { + rotationDegrees += 360; + } + return rotationDegrees; + } + } + return MoonBridge.LI_ROT_UNKNOWN; + } + + private static float[] polarToCartesian(float r, float theta) { + return new float[] { (float)(r * Math.cos(theta)), (float)(r * Math.sin(theta)) }; + } + + private static float cartesianToR(float[] point) { + return (float)Math.sqrt(Math.pow(point[0], 2) + Math.pow(point[1], 2)); + } + + private float[] getStreamViewNormalizedContactArea(MotionEvent event) { + float orientation; + + // If the orientation is unknown, we'll just assume it's at a 45 degree angle and scale it by + // X and Y scaling factors evenly. + if (event.getDevice() == null || event.getDevice().getMotionRange(MotionEvent.AXIS_ORIENTATION, event.getSource()) == null) { + orientation = (float)(Math.PI / 4); + } + else { + orientation = event.getOrientation(event.getActionIndex()); + } + + float contactAreaMajor, contactAreaMinor; + switch (event.getActionMasked()) { + // Hover events report the tool size + case MotionEvent.ACTION_HOVER_ENTER: + case MotionEvent.ACTION_HOVER_MOVE: + case MotionEvent.ACTION_HOVER_EXIT: + contactAreaMajor = event.getToolMajor(event.getActionIndex()); + contactAreaMinor = event.getToolMinor(event.getActionIndex()); + break; + + // Other events report contact area + default: + contactAreaMajor = event.getTouchMajor(event.getActionIndex()); + contactAreaMinor = event.getTouchMinor(event.getActionIndex()); + break; + } + + // The contact area major axis is parallel to the orientation, so we simply convert + // polar to cartesian coordinates using the orientation as theta. + float[] contactAreaMajorCartesian = polarToCartesian(contactAreaMajor, orientation); + + // The contact area minor axis is perpendicular to the contact area major axis (and thus + // the orientation), so rotate the orientation angle by 90 degrees. + float[] contactAreaMinorCartesian = polarToCartesian(contactAreaMinor, (float)(orientation + (Math.PI / 2))); + + // Normalize the contact area to the stream view size + contactAreaMajorCartesian[0] = Math.min(Math.abs(contactAreaMajorCartesian[0]), streamView.getWidth()) / streamView.getWidth(); + contactAreaMinorCartesian[0] = Math.min(Math.abs(contactAreaMinorCartesian[0]), streamView.getWidth()) / streamView.getWidth(); + contactAreaMajorCartesian[1] = Math.min(Math.abs(contactAreaMajorCartesian[1]), streamView.getHeight()) / streamView.getHeight(); + contactAreaMinorCartesian[1] = Math.min(Math.abs(contactAreaMinorCartesian[1]), streamView.getHeight()) / streamView.getHeight(); + + // Convert the normalized values back into polar coordinates + return new float[] { cartesianToR(contactAreaMajorCartesian), cartesianToR(contactAreaMinorCartesian) }; + } + private boolean trySendPenEvent(View view, MotionEvent event) { byte eventType = getLiTouchTypeFromEvent(event); if (eventType < 0) { @@ -1558,26 +1652,21 @@ public class Game extends Activity implements SurfaceHolder.Callback, penButtons |= MoonBridge.LI_PEN_BUTTON_SECONDARY; } - short rotationDegrees = MoonBridge.LI_ROT_UNKNOWN; byte tiltDegrees = MoonBridge.LI_TILT_UNKNOWN; InputDevice dev = event.getDevice(); if (dev != null) { - if (dev.getMotionRange(MotionEvent.AXIS_ORIENTATION, event.getSource()) != null) { - rotationDegrees = (short)Math.toDegrees(event.getOrientation(event.getActionIndex())); - if (rotationDegrees < 0) { - rotationDegrees += 360; - } - } if (dev.getMotionRange(MotionEvent.AXIS_TILT, event.getSource()) != null) { tiltDegrees = (byte)Math.toDegrees(event.getAxisValue(MotionEvent.AXIS_TILT, event.getActionIndex())); } } float[] normalizedCoords = getStreamViewRelativeNormalizedXY(view, event); + float[] normalizedContactArea = getStreamViewNormalizedContactArea(event); return conn.sendPenEvent(eventType, toolType, penButtons, normalizedCoords[0], normalizedCoords[1], - event.getPressure(event.getActionIndex()), - rotationDegrees, tiltDegrees) != MoonBridge.LI_ERR_UNSUPPORTED; + getPressureOrDistance(event), + normalizedContactArea[0], normalizedContactArea[1], + getRotationDegrees(event), tiltDegrees) != MoonBridge.LI_ERR_UNSUPPORTED; } private boolean trySendTouchEvent(View view, MotionEvent event) { @@ -1587,9 +1676,12 @@ public class Game extends Activity implements SurfaceHolder.Callback, } float[] normalizedCoords = getStreamViewRelativeNormalizedXY(view, event); + float[] normalizedContactArea = getStreamViewNormalizedContactArea(event); return conn.sendTouchEvent(eventType, event.getPointerId(event.getActionIndex()), normalizedCoords[0], normalizedCoords[1], - event.getPressure(event.getActionIndex())) != MoonBridge.LI_ERR_UNSUPPORTED; + getPressureOrDistance(event), + normalizedContactArea[0], normalizedContactArea[1], + getRotationDegrees(event)) != MoonBridge.LI_ERR_UNSUPPORTED; } // Returns true if the event was consumed diff --git a/app/src/main/java/com/limelight/nvstream/NvConnection.java b/app/src/main/java/com/limelight/nvstream/NvConnection.java index a3a898ab..7fd24f47 100644 --- a/app/src/main/java/com/limelight/nvstream/NvConnection.java +++ b/app/src/main/java/com/limelight/nvstream/NvConnection.java @@ -530,9 +530,11 @@ public class NvConnection { } } - public int sendTouchEvent(byte eventType, int pointerId, float x, float y, float pressure) { + public int sendTouchEvent(byte eventType, int pointerId, float x, float y, float pressureOrDistance, + float contactAreaMajor, float contactAreaMinor, short rotation) { if (!isMonkey) { - return MoonBridge.sendTouchEvent(eventType, pointerId, x, y, pressure); + return MoonBridge.sendTouchEvent(eventType, pointerId, x, y, pressureOrDistance, + contactAreaMajor, contactAreaMinor, rotation); } else { return MoonBridge.LI_ERR_UNSUPPORTED; @@ -540,9 +542,11 @@ public class NvConnection { } public int sendPenEvent(byte eventType, byte toolType, byte penButtons, float x, float y, - float pressure, short rotation, byte tilt) { + float pressureOrDistance, float contactAreaMajor, float contactAreaMinor, + short rotation, byte tilt) { if (!isMonkey) { - return MoonBridge.sendPenEvent(eventType, toolType, penButtons, x, y, pressure, rotation, tilt); + return MoonBridge.sendPenEvent(eventType, toolType, penButtons, x, y, pressureOrDistance, + contactAreaMajor, contactAreaMinor, rotation, tilt); } else { return MoonBridge.LI_ERR_UNSUPPORTED; diff --git a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java index 8b522650..c6764ee0 100644 --- a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java +++ b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java @@ -372,10 +372,12 @@ public class MoonBridge { short leftStickX, short leftStickY, short rightStickX, short rightStickY); - public static native int sendTouchEvent(byte eventType, int pointerId, float x, float y, float pressure); + public static native int sendTouchEvent(byte eventType, int pointerId, float x, float y, float pressure, + float contactAreaMajor, float contactAreaMinor, short rotation); public static native int sendPenEvent(byte eventType, byte toolType, byte penButtons, float x, float y, - float pressure, short rotation, byte tilt); + float pressure, float contactAreaMajor, float contactAreaMinor, + short rotation, byte tilt); public static native int sendControllerArrivalEvent(byte controllerNumber, short activeGamepadMask, byte type, int supportedButtonFlags, short capabilities); diff --git a/app/src/main/jni/moonlight-core/moonlight-common-c b/app/src/main/jni/moonlight-core/moonlight-common-c index 2d0badde..70a2e305 160000 --- a/app/src/main/jni/moonlight-core/moonlight-common-c +++ b/app/src/main/jni/moonlight-core/moonlight-common-c @@ -1 +1 @@ -Subproject commit 2d0badde9aa9a48480e24a289569de1046c6acba +Subproject commit 70a2e305bc7170eccbd48d8c49b64a814e64ecb7 diff --git a/app/src/main/jni/moonlight-core/simplejni.c b/app/src/main/jni/moonlight-core/simplejni.c index cab0e992..9fbbfd08 100644 --- a/app/src/main/jni/moonlight-core/simplejni.c +++ b/app/src/main/jni/moonlight-core/simplejni.c @@ -45,16 +45,21 @@ Java_com_limelight_nvstream_jni_MoonBridge_sendMultiControllerInput(JNIEnv *env, JNIEXPORT jint JNICALL Java_com_limelight_nvstream_jni_MoonBridge_sendTouchEvent(JNIEnv *env, jclass clazz, jbyte eventType, jint pointerId, - jfloat x, jfloat y, jfloat pressure) { - return LiSendTouchEvent(eventType, pointerId, x, y, pressure); + jfloat x, jfloat y, jfloat pressureOrDistance, + jfloat contactAreaMajor, jfloat contactAreaMinor, + jshort rotation) { + return LiSendTouchEvent(eventType, pointerId, x, y, pressureOrDistance, + contactAreaMajor, contactAreaMinor, rotation); } JNIEXPORT jint JNICALL Java_com_limelight_nvstream_jni_MoonBridge_sendPenEvent(JNIEnv *env, jclass clazz, jbyte eventType, jbyte toolType, jbyte penButtons, - jfloat x, jfloat y, jfloat pressure, + jfloat x, jfloat y, jfloat pressureOrDistance, + jfloat contactAreaMajor, jfloat contactAreaMinor, jshort rotation, jbyte tilt) { - return LiSendPenEvent(eventType, toolType, penButtons, x, y, pressure, rotation, tilt); + return LiSendPenEvent(eventType, toolType, penButtons, x, y, pressureOrDistance, + contactAreaMajor, contactAreaMinor, rotation, tilt); } JNIEXPORT jint JNICALL