feat(scrcpy): improve support for scroll and hover

apps/demo/src/pages/scrcpy.tsx

@@ -40,9 +40,11 @@ import {
     AndroidKeyEventAction,
     AndroidMotionEventAction,
     AndroidScreenPowerMode,
+    clamp,
     CodecOptions,
     DEFAULT_SERVER_PATH,
     ScrcpyDeviceMessageType,
+    ScrcpyHoverHelper,
     ScrcpyLogLevel,
     ScrcpyOptions1_25,
     ScrcpyOptionsInit1_24,
@@ -157,18 +159,6 @@ function fetchServer(
     return cachedValue.promise;
 }
 
-function clamp(value: number, min: number, max: number): number {
-    if (value < min) {
-        return min;
-    }
-
-    if (value > max) {
-        return max;
-    }
-
-    return value;
-}
-
 export interface H264Decoder extends Disposable {
     readonly maxProfile: AndroidCodecProfile | undefined;
     readonly maxLevel: AndroidCodecLevel | undefined;
@@ -258,6 +248,7 @@ const useClasses = makeStyles({
     },
     video: {
         transformOrigin: "center center",
+        touchAction: "none",
     },
 });
 
@@ -369,6 +360,7 @@ class ScrcpyPageState {
     }
 
     client: AdbScrcpyClient | undefined = undefined;
+    hoverHelper: ScrcpyHoverHelper | undefined = undefined;
 
     async pushServer() {
         const serverBuffer = await fetchServer();
@@ -891,6 +883,7 @@ class ScrcpyPageState {
             handlePointerDown: false,
             handlePointerMove: false,
             handlePointerUp: false,
+            handlePointerLeave: false,
             handleWheel: false,
             handleContextMenu: false,
             handleKeyDown: false,
@@ -1117,6 +1110,7 @@ class ScrcpyPageState {
 
             runInAction(() => {
                 this.client = client;
+                this.hoverHelper = new ScrcpyHoverHelper();
                 this.running = true;
             });
         } catch (e: any) {
@@ -1301,37 +1295,46 @@ class ScrcpyPageState {
 
         const { pointerType } = e;
         let pointerId: bigint;
-        let { pressure } = e;
         if (pointerType === "mouse") {
             // ScrcpyPointerId.Mouse doesn't work with Chrome browser
             // https://github.com/Genymobile/scrcpy/issues/3635
             pointerId = ScrcpyPointerId.Finger;
-            pressure = pressure === 0 ? 0 : 1;
         } else {
             pointerId = BigInt(e.pointerId);
         }
 
         const { x, y } = this.calculatePointerPosition(e.clientX, e.clientY);
-        this.client!.controlMessageSerializer!.injectTouch({
+
+        const messages = this.hoverHelper!.process({
             action,
             pointerId,
-            screenWidth: this.client!.screenWidth!,
+            screenWidth: this.client.screenWidth!,
-            screenHeight: this.client!.screenHeight!,
+            screenHeight: this.client.screenHeight!,
             pointerX: x,
             pointerY: y,
-            pressure,
+            pressure: e.pressure,
             buttons: e.buttons,
         });
+        for (const message of messages) {
+            this.client.controlMessageSerializer!.injectTouch(message);
+        }
     };
 
     handlePointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
         this.rendererContainer!.focus();
         e.preventDefault();
+        e.stopPropagation();
         e.currentTarget.setPointerCapture(e.pointerId);
         this.injectTouch(AndroidMotionEventAction.Down, e);
     };
 
     handlePointerMove = (e: React.PointerEvent<HTMLDivElement>) => {
+        if (!this.client) {
+            return;
+        }
+
+        e.preventDefault();
+        e.stopPropagation();
         this.injectTouch(
             e.buttons === 0
                 ? AndroidMotionEventAction.HoverMove
@@ -1341,6 +1344,16 @@ class ScrcpyPageState {
     };
 
     handlePointerUp = (e: React.PointerEvent<HTMLDivElement>) => {
+        e.preventDefault();
+        e.stopPropagation();
+        this.injectTouch(AndroidMotionEventAction.Up, e);
+    };
+
+    handlePointerLeave = (e: React.PointerEvent<HTMLDivElement>) => {
+        e.preventDefault();
+        e.stopPropagation();
+        // Prevent hover state on device from "stucking" at the last position
+        this.injectTouch(AndroidMotionEventAction.HoverExit, e);
         this.injectTouch(AndroidMotionEventAction.Up, e);
     };
 
@@ -1358,8 +1371,8 @@ class ScrcpyPageState {
             screenHeight: this.client!.screenHeight!,
             pointerX: x,
             pointerY: y,
-            scrollX: e.deltaX / 100,
+            scrollX: -e.deltaX / 100,
-            scrollY: e.deltaY / 100,
+            scrollY: -e.deltaY / 100,
             buttons: 0,
         });
     };
@@ -1591,6 +1604,7 @@ const Scrcpy: NextPage = () => {
                         onPointerMove={state.handlePointerMove}
                         onPointerUp={state.handlePointerUp}
                         onPointerCancel={state.handlePointerUp}
+                        onPointerLeave={state.handlePointerLeave}
                         onKeyDown={state.handleKeyDown}
                         onContextMenu={state.handleContextMenu}
                     />

libraries/scrcpy/src/control/hover-helper.ts

@@ -0,0 +1,51 @@
+import {
+    AndroidMotionEventAction,
+    type ScrcpyInjectTouchControlMessage,
+} from "./inject-touch.js";
+import { ScrcpyControlMessageType } from "./type.js";
+
+/**
+ * On Android, touching the screen with a finger will disable mouse cursor.
+ * However, Scrcpy doesn't do that, and can inject two pointers at the same time.
+ * This can cause finger events to be "ignored" because mouse is still the primary pointer.
+ *
+ * This helper class injects an extra `ACTION_UP` event,
+ * so Scrcpy server can remove the previously hovering pointer.
+ */
+export class ScrcpyHoverHelper {
+    // AFAIK, only mouse and pen can have hover state
+    // and you can't have two mouses or pens.
+    private lastHoverMessage: ScrcpyInjectTouchControlMessage | undefined;
+
+    public process(
+        message: Omit<ScrcpyInjectTouchControlMessage, "type">
+    ): ScrcpyInjectTouchControlMessage[] {
+        const result: ScrcpyInjectTouchControlMessage[] = [];
+
+        // A different pointer appeared,
+        // Cancel previously hovering pointer so Scrcpy server can free up the pointer ID.
+        if (
+            this.lastHoverMessage &&
+            this.lastHoverMessage.pointerId !== message.pointerId
+        ) {
+            // TODO: Inject MotionEvent.ACTION_HOVER_EXIT
+            // From testing, it seems no App cares about this event.
+            result.push({
+                ...this.lastHoverMessage,
+                action: AndroidMotionEventAction.Up,
+            });
+            this.lastHoverMessage = undefined;
+        }
+
+        if (message.action === AndroidMotionEventAction.HoverMove) {
+            // TODO: Inject MotionEvent.ACTION_HOVER_ENTER
+            this.lastHoverMessage = message as ScrcpyInjectTouchControlMessage;
+        }
+
+        (message as ScrcpyInjectTouchControlMessage).type =
+            ScrcpyControlMessageType.InjectTouch;
+        result.push(message as ScrcpyInjectTouchControlMessage);
+
+        return result;
+    }
+}

libraries/scrcpy/src/control/index.ts

@@ -1,4 +1,5 @@
 export * from "./back-or-screen-on.js";
+export * from "./hover-helper.js";
 export * from "./inject-keycode.js";
 export * from "./inject-scroll.js";
 export * from "./inject-text.js";

libraries/scrcpy/src/control/inject-touch.ts

@@ -31,6 +31,18 @@ export namespace ScrcpyPointerId {
     export const VirtualFinger = BigInt(-4);
 }
 
+export function clamp(value: number, min: number, max: number): number {
+    if (value < min) {
+        return min;
+    }
+
+    if (value > max) {
+        return max;
+    }
+
+    return value;
+}
+
 const Uint16Max = (1 << 16) - 1;
 
 const ScrcpyFloatToUint16NumberType: NumberFieldType = {
@@ -41,7 +53,7 @@ const ScrcpyFloatToUint16NumberType: NumberFieldType = {
         return value / Uint16Max;
     },
     serialize(dataView, offset, value, littleEndian) {
-        value = value * Uint16Max;
+        value = clamp(value, 0, 1) * Uint16Max;
         NumberFieldType.Uint16.serialize(dataView, offset, value, littleEndian);
     },
 };

libraries/scrcpy/src/options/1_16/scroll.ts

@@ -40,18 +40,18 @@ export class ScrcpyScrollController1_16 implements ScrcpyScrollController {
         let scrollY = 0;
         if (this.accumulatedX >= 1) {
             scrollX = 1;
-            this.accumulatedX -= 1;
+            this.accumulatedX = 0;
         } else if (this.accumulatedX <= -1) {
             scrollX = -1;
-            this.accumulatedX += 1;
+            this.accumulatedX = 0;
         }
 
         if (this.accumulatedY >= 1) {
             scrollY = 1;
-            this.accumulatedY -= 1;
+            this.accumulatedY = 0;
         } else if (this.accumulatedY <= -1) {
             scrollY = -1;
-            this.accumulatedY += 1;
+            this.accumulatedY = 0;
         }
 
         if (scrollX === 0 && scrollY === 0) {

libraries/scrcpy/src/options/1_25/scroll.ts

@@ -4,6 +4,7 @@ import Struct, {
 } from "@yume-chan/struct";
 
 import {
+    clamp,
     ScrcpyControlMessageType,
     type ScrcpyInjectScrollControlMessage,
 } from "../../control/index.js";
@@ -19,7 +20,7 @@ const ScrcpyFloatToInt16NumberType: NumberFieldType = {
         return value / Int16Max;
     },
     serialize(dataView, offset, value, littleEndian) {
-        value = value * Int16Max;
+        value = clamp(value, -1, 1) * Int16Max;
         NumberFieldType.Int16.serialize(dataView, offset, value, littleEndian);
     },
 };