diff --git a/app/src/main/java/com/limelight/binding/input/touch/RelativeTouchContext.java b/app/src/main/java/com/limelight/binding/input/touch/RelativeTouchContext.java
index cf3871db..794f2e68 100644
--- a/app/src/main/java/com/limelight/binding/input/touch/RelativeTouchContext.java
+++ b/app/src/main/java/com/limelight/binding/input/touch/RelativeTouchContext.java
@@ -160,6 +160,11 @@ public class RelativeTouchContext implements TouchContext {
                         return;
                     }
 
+                    // The drag should only be processed for the primary finger
+                    if (actionIndex != maxPointerCountInGesture - 1) {
+                        return;
+                    }
+
                     // Check if someone cancelled us
                     if (dragTimer == null) {
                         return;