refactor(scrcpy): revert changing control message client to stream

refs #427

apps/demo/src/pages/scrcpy.tsx

@@ -9,7 +9,7 @@ import { CSSProperties, ReactNode, useEffect, useState } from "react";
 
 import { ADB_SYNC_MAX_PACKET_SIZE } from '@yume-chan/adb';
 import { EventEmitter } from "@yume-chan/event";
-import { AdbScrcpyClient, AdbScrcpyOptions1_22, AndroidKeyCode, AndroidKeyEventAction, AndroidMotionEventAction, CodecOptions, DEFAULT_SERVER_PATH, ScrcpyControlMessage, ScrcpyControlMessageType, ScrcpyLogLevel, ScrcpyOptions1_24, ScrcpyVideoOrientation, TinyH264Decoder, WebCodecsDecoder, type H264Decoder, type H264DecoderConstructor, type VideoStreamPacket } from "@yume-chan/scrcpy";
+import { AdbScrcpyClient, AdbScrcpyOptions1_22, AndroidKeyCode, AndroidKeyEventAction, AndroidMotionEventAction, CodecOptions, DEFAULT_SERVER_PATH, ScrcpyDeviceMessageType, ScrcpyLogLevel, ScrcpyOptions1_24, ScrcpyVideoOrientation, TinyH264Decoder, WebCodecsDecoder, type H264Decoder, type H264DecoderConstructor, type VideoStreamPacket } from "@yume-chan/scrcpy";
 import SCRCPY_SERVER_VERSION from '@yume-chan/scrcpy/bin/version';
 import { ChunkStream, InspectStream, ReadableStream, WritableStream } from '@yume-chan/stream-extra';
 
@@ -243,7 +243,6 @@ class ScrcpyPageState {
     get rotatedHeight() { return state.rotate & 1 ? state.width : state.height; }
 
     client: AdbScrcpyClient | undefined = undefined;
-    controlMessageWriter: WritableStreamDefaultWriter<ScrcpyControlMessage> | undefined = undefined;
 
     async pushServer() {
         const serverBuffer = await fetchServer();
@@ -364,7 +363,7 @@ class ScrcpyPageState {
             iconProps: { iconName: Icons.Orientation },
             iconOnly: true,
             text: 'Rotate Device',
-            onClick: () => { this.controlMessageWriter!.write({ type: ScrcpyControlMessageType.RotateDevice }); },
+            onClick: () => { this.client!.controlMessageSerializer!.rotateDevice(); },
         });
 
         result.push({
@@ -795,13 +794,16 @@ class ScrcpyPageState {
 
             client.deviceMessageStream!.pipeTo(new WritableStream({
                 write(message) {
-                    window.navigator.clipboard.writeText(message.content);
+                    switch (message.type) {
+                        case ScrcpyDeviceMessageType.Clipboard:
+                            window.navigator.clipboard.writeText(message.content);
+                            break;
+                    }
                 }
             })).catch(() => { });
 
             runInAction(() => {
                 this.client = client;
-                this.controlMessageWriter = client.controlMessageStream!.getWriter();
                 this.running = true;
             });
         } catch (e: any) {
@@ -850,10 +852,7 @@ class ScrcpyPageState {
         }
         e.currentTarget.setPointerCapture(e.pointerId);
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.backOrScreenOn(AndroidKeyEventAction.Down);
-            type: ScrcpyControlMessageType.BackOrScreenOn,
-            action: AndroidKeyEventAction.Down,
-        });
     };
 
     handleBackPointerUp = (e: React.PointerEvent<HTMLDivElement>) => {
@@ -865,10 +864,7 @@ class ScrcpyPageState {
             return;
         }
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.backOrScreenOn(AndroidKeyEventAction.Up);
-            type: ScrcpyControlMessageType.BackOrScreenOn,
-            action: AndroidKeyEventAction.Up,
-        });
     };
 
     handleHomePointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
@@ -881,8 +877,7 @@ class ScrcpyPageState {
         }
         e.currentTarget.setPointerCapture(e.pointerId);
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectKeyCode({
-            type: ScrcpyControlMessageType.InjectKeyCode,
             action: AndroidKeyEventAction.Down,
             keyCode: AndroidKeyCode.Home,
             repeat: 0,
@@ -899,8 +894,7 @@ class ScrcpyPageState {
             return;
         }
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectKeyCode({
-            type: ScrcpyControlMessageType.InjectKeyCode,
             action: AndroidKeyEventAction.Up,
             keyCode: AndroidKeyCode.Home,
             repeat: 0,
@@ -918,8 +912,7 @@ class ScrcpyPageState {
         }
         e.currentTarget.setPointerCapture(e.pointerId);
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectKeyCode({
-            type: ScrcpyControlMessageType.InjectKeyCode,
             action: AndroidKeyEventAction.Down,
             keyCode: AndroidKeyCode.AppSwitch,
             repeat: 0,
@@ -936,8 +929,7 @@ class ScrcpyPageState {
             return;
         }
 
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectKeyCode({
-            type: ScrcpyControlMessageType.InjectKeyCode,
             action: AndroidKeyEventAction.Up,
             keyCode: AndroidKeyCode.AppSwitch,
             repeat: 0,
@@ -981,8 +973,7 @@ class ScrcpyPageState {
         }
 
         const { x, y } = this.calculatePointerPosition(e.clientX, e.clientY);
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectTouch({
-            type: ScrcpyControlMessageType.InjectTouch,
             action,
             pointerId: e.pointerType === "mouse" ? BigInt(-1) : BigInt(e.pointerId),
             screenWidth: this.client!.screenWidth!,
@@ -1021,8 +1012,7 @@ class ScrcpyPageState {
         e.stopPropagation();
 
         const { x, y } = this.calculatePointerPosition(e.clientX, e.clientY);
-        this.controlMessageWriter!.write({
+        this.client!.controlMessageSerializer!.injectScroll({
-            type: ScrcpyControlMessageType.InjectScroll,
             screenWidth: this.client!.screenWidth!,
             screenHeight: this.client!.screenHeight!,
             pointerX: x,
@@ -1044,10 +1034,7 @@ class ScrcpyPageState {
 
         const { key, code } = e;
         if (key.match(/^[!-`{-~]$/i)) {
-            this.controlMessageWriter!.write({
+            this.client!.controlMessageSerializer!.injectText(key);
-                type: ScrcpyControlMessageType.InjectText,
-                text: key,
-            });
             return;
         }
 
@@ -1058,15 +1045,13 @@ class ScrcpyPageState {
         } as Record<string, AndroidKeyCode | undefined>)[code];
 
         if (keyCode) {
-            this.controlMessageWriter!.write({
+            this.client!.controlMessageSerializer!.injectKeyCode({
-                type: ScrcpyControlMessageType.InjectKeyCode,
                 action: AndroidKeyEventAction.Down,
                 keyCode,
                 metaState: 0,
                 repeat: 0,
             });
-            this.controlMessageWriter!.write({
+            this.client!.controlMessageSerializer!.injectKeyCode({
-                type: ScrcpyControlMessageType.InjectKeyCode,
                 action: AndroidKeyEventAction.Up,
                 keyCode,
                 metaState: 0,

libraries/scrcpy/src/adb/client.ts

@@ -1,7 +1,7 @@
 import { Adb, AdbSubprocessNoneProtocol, AdbSubprocessProtocol, AdbSync } from '@yume-chan/adb';
-import { DecodeUtf8Stream, InspectStream, pipeFrom, ReadableStream, SplitStringStream, WrapWritableStream, WritableStream, type ReadableWritablePair } from '@yume-chan/stream-extra';
+import { DecodeUtf8Stream, InspectStream, ReadableStream, SplitStringStream, WrapWritableStream, WritableStream, type ReadableWritablePair } from '@yume-chan/stream-extra';
 
-import { ScrcpyControlMessageSerializeStream, type ScrcpyControlMessage } from '../control/index.js';
+import { ScrcpyControlMessageSerializer } from '../control/index.js';
 import { ScrcpyDeviceMessageDeserializeStream, type ScrcpyDeviceMessage } from '../device-message/index.js';
 import { DEFAULT_SERVER_PATH, type VideoStreamPacket } from '../options/index.js';
 import type { AdbScrcpyOptions } from './options/index.js';
@@ -255,8 +255,8 @@ export class AdbScrcpyClient {
     private _videoStream: ReadableStream<VideoStreamPacket>;
     public get videoStream() { return this._videoStream; }
 
-    private _controlMessageStream: WritableStream<ScrcpyControlMessage> | undefined;
+    private _controlMessageSerializer: ScrcpyControlMessageSerializer | undefined;
-    public get controlMessageStream() { return this._controlMessageStream; }
+    public get controlMessageSerializer() { return this._controlMessageSerializer; }
 
     private _deviceMessageStream: ReadableStream<ScrcpyDeviceMessage> | undefined;
     public get deviceMessageStream() { return this._deviceMessageStream; }
@@ -281,7 +281,7 @@ export class AdbScrcpyClient {
             }));
 
         if (controlStream) {
-            this._controlMessageStream = pipeFrom(controlStream.writable, new ScrcpyControlMessageSerializeStream(options));
+            this._controlMessageSerializer = new ScrcpyControlMessageSerializer(controlStream.writable, options);
             this._deviceMessageStream = controlStream.readable.pipeThrough(new ScrcpyDeviceMessageDeserializeStream());
         }
     }

libraries/scrcpy/src/control/index.ts

@@ -2,5 +2,5 @@ export * from './inject-keycode.js';
 export * from './inject-text.js';
 export * from './inject-touch.js';
 export * from './rotate-device.js';
-export * from './stream.js';
+export * from './serializer.js';
 export * from './type.js';

libraries/scrcpy/src/control/serializer.ts

@@ -0,0 +1,78 @@
+import type { WritableStreamDefaultWriter } from '@yume-chan/stream-extra';
+
+import type { ScrcpyInjectScrollControlMessage1_22, ScrcpyOptions } from '../options/index.js';
+import { AndroidKeyEventAction, ScrcpyInjectKeyCodeControlMessage } from './inject-keycode.js';
+import { ScrcpyInjectTextControlMessage } from './inject-text.js';
+import { ScrcpyInjectTouchControlMessage } from './inject-touch.js';
+import { ScrcpyRotateDeviceControlMessage } from './rotate-device.js';
+import { ScrcpyControlMessageType } from './type.js';
+
+export class ScrcpyControlMessageSerializer {
+    private options: ScrcpyOptions<any>;
+    /** Control message type values for current version of server */
+    private types: ScrcpyControlMessageType[];
+    private writer: WritableStreamDefaultWriter<Uint8Array>;
+
+    public constructor(stream: WritableStream<Uint8Array>, options: ScrcpyOptions<any>) {
+        this.options = options;
+        this.types = options.getControlMessageTypes();
+        this.writer = stream.getWriter();
+    }
+
+    public getTypeValue(type: ScrcpyControlMessageType): number {
+        const value = this.types.indexOf(type);
+        if (value === -1) {
+            throw new Error('Not supported');
+        }
+        return value;
+    }
+
+    public injectKeyCode(message: Omit<ScrcpyInjectKeyCodeControlMessage, 'type'>) {
+        return this.writer.write(ScrcpyInjectKeyCodeControlMessage.serialize({
+            ...message,
+            type: this.getTypeValue(ScrcpyControlMessageType.InjectKeyCode),
+        }));
+    }
+
+    public injectText(text: string) {
+        return this.writer.write(ScrcpyInjectTextControlMessage.serialize({
+            text,
+            type: this.getTypeValue(ScrcpyControlMessageType.InjectText),
+        }));
+    }
+
+    public injectTouch(message: Omit<ScrcpyInjectTouchControlMessage, 'type'>) {
+        return this.writer.write(ScrcpyInjectTouchControlMessage.serialize({
+            ...message,
+            type: this.getTypeValue(ScrcpyControlMessageType.InjectTouch),
+        }));
+    }
+
+    public injectScroll(message: Omit<ScrcpyInjectScrollControlMessage1_22, 'type'>) {
+        return this.writer.write(this.options.serializeInjectScrollControlMessage({
+            ...message,
+            type: this.getTypeValue(ScrcpyControlMessageType.InjectScroll),
+        }));
+    }
+
+    public async backOrScreenOn(action: AndroidKeyEventAction) {
+        const buffer = this.options.serializeBackOrScreenOnControlMessage({
+            action,
+            type: this.getTypeValue(ScrcpyControlMessageType.BackOrScreenOn),
+        });
+
+        if (buffer) {
+            return await this.writer.write(buffer);
+        }
+    }
+
+    public rotateDevice() {
+        return this.writer.write(ScrcpyRotateDeviceControlMessage.serialize({
+            type: this.getTypeValue(ScrcpyControlMessageType.RotateDevice),
+        }));
+    }
+
+    public close() {
+        return this.writer.close();
+    }
+};

libraries/scrcpy/src/control/stream.ts

@@ -1,79 +0,0 @@
-import { TransformStream } from '@yume-chan/stream-extra';
-
-import type { ScrcpyBackOrScreenOnControlMessage1_18, ScrcpyInjectScrollControlMessage1_22, ScrcpyOptions } from '../options/index.js';
-import { ScrcpyInjectKeyCodeControlMessage } from './inject-keycode.js';
-import { ScrcpyInjectTextControlMessage } from './inject-text.js';
-import { ScrcpyInjectTouchControlMessage } from './inject-touch.js';
-import { ScrcpyRotateDeviceControlMessage } from './rotate-device.js';
-import { ScrcpyControlMessageType } from './type.js';
-
-export type ScrcpyControlMessage =
-    | ScrcpyInjectKeyCodeControlMessage
-    | ScrcpyInjectTextControlMessage
-    | ScrcpyInjectTouchControlMessage
-    | ScrcpyInjectScrollControlMessage1_22
-    | ScrcpyBackOrScreenOnControlMessage1_18
-    | ScrcpyRotateDeviceControlMessage;
-
-export class ScrcpyControlMessageSerializeStream extends TransformStream<ScrcpyControlMessage, Uint8Array> {
-    public constructor(options: ScrcpyOptions<any>) {
-        // Get control message types for current version of server
-        const types = options.getControlMessageTypes();
-
-        super({
-            transform(message, controller) {
-                const type = types.indexOf(message.type);
-                if (type === -1) {
-                    throw new Error('Not supported');
-                }
-
-                switch (message.type) {
-                    case ScrcpyControlMessageType.InjectKeyCode:
-                        controller.enqueue(ScrcpyInjectKeyCodeControlMessage.serialize({
-                            ...message,
-                            type
-                        }));
-                        break;
-                    case ScrcpyControlMessageType.InjectText:
-                        controller.enqueue(ScrcpyInjectTextControlMessage.serialize({
-                            ...message,
-                            type,
-                        }));
-                        break;
-                    case ScrcpyControlMessageType.InjectTouch:
-                        // ADB streams are actually pretty low-bandwidth and laggy
-                        // Re-sample move events to avoid flooding the connection
-
-                        controller.enqueue(ScrcpyInjectTouchControlMessage.serialize({
-                            ...message,
-                            type,
-                        }));
-                        break;
-                    case ScrcpyControlMessageType.InjectScroll:
-                        controller.enqueue(options.serializeInjectScrollControlMessage({
-                            ...message,
-                            type,
-                        }))
-                        break;
-                    case ScrcpyControlMessageType.BackOrScreenOn:
-                        {
-                            const buffer = options.serializeBackOrScreenOnControlMessage({
-                                ...message,
-                                type,
-                            });
-
-                            if (buffer) {
-                                controller.enqueue(buffer);
-                            }
-                        }
-                        break;
-                    case ScrcpyControlMessageType.RotateDevice:
-                        controller.enqueue(ScrcpyRotateDeviceControlMessage.serialize({
-                            type,
-                        }));
-                        break;
-                }
-            }
-        })
-    }
-}

libraries/scrcpy/src/decoder/tinyh264/wrapper.ts

@@ -1,5 +1,5 @@
 import { PromiseResolver } from '@yume-chan/async';
-import { AutoDisposable, EventEmitter } from '@yume-chan/event';
+import { AutoDisposable, Disposable, EventEmitter } from '@yume-chan/event';
 
 let worker: Worker | undefined;
 let workerReady = false;
@@ -16,7 +16,17 @@ export interface PictureReadyEventArgs {
     data: ArrayBuffer;
 }
 
-const pictureReadyEvent = new EventEmitter<PictureReadyEventArgs>();
+const PICTURE_READY_SUBSCRIPTIONS = new Map<number, (e: PictureReadyEventArgs) => void>();
+
+function subscribePictureReady(streamId: number, handler: (e: PictureReadyEventArgs) => void): Disposable {
+    PICTURE_READY_SUBSCRIPTIONS.set(streamId, handler);
+
+    return {
+        dispose() {
+            PICTURE_READY_SUBSCRIPTIONS.delete(streamId);
+        }
+    };
+}
 
 export class TinyH264Wrapper extends AutoDisposable {
     public readonly streamId: number;
@@ -28,14 +38,12 @@ export class TinyH264Wrapper extends AutoDisposable {
         super();
 
         this.streamId = streamId;
-        this.addDisposable(pictureReadyEvent.event(this.handlePictureReady, this));
+        this.addDisposable(subscribePictureReady(streamId, this.handlePictureReady));
     }
 
-    private handlePictureReady(e: PictureReadyEventArgs) {
+    private handlePictureReady = (e: PictureReadyEventArgs) => {
-        if (e.renderStateId === this.streamId) {
+        this.pictureReadyEvent.fire(e);
-            this.pictureReadyEvent.fire(e);
+    };
-        }
-    }
 
     public feed(data: ArrayBuffer) {
         worker!.postMessage({
@@ -71,7 +79,7 @@ export function createTinyH264Wrapper(): Promise<TinyH264Wrapper> {
                     pendingResolvers.length = 0;
                     break;
                 case 'pictureReady':
-                    pictureReadyEvent.fire(data);
+                    PICTURE_READY_SUBSCRIPTIONS.get(data.renderStateId)?.(data);
                     break;
             }
         });

libraries/scrcpy/src/device-message/clipboard.ts

@@ -1,9 +1,11 @@
 import Struct from '@yume-chan/struct';
+import { ScrcpyDeviceMessageType } from './type.js';
 
 export const ScrcpyClipboardDeviceMessage =
     new Struct()
         .uint32('length')
-        .string('content', { lengthField: 'length' });
+        .string('content', { lengthField: 'length' })
+        .extra({ type: ScrcpyDeviceMessageType.Clipboard as const });
 
 export type ScrcpyClipboardDeviceMessage =
     typeof ScrcpyClipboardDeviceMessage['TDeserializeResult'];

libraries/scrcpy/src/device-message/index.ts

@@ -1,2 +1,3 @@
 export * from './clipboard.js';
 export * from './stream.js';
+export * from './type.js';

libraries/scrcpy/src/device-message/type.ts

@@ -0,0 +1,5 @@
+// https://github.com/Genymobile/scrcpy/blob/41abe021e2a73efd4899b0efcd0b9eef9ec68c9b/server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java#L5
+export enum ScrcpyDeviceMessageType {
+    Clipboard,
+    AckClipboard,
+}

libraries/scrcpy/src/index.ts

@@ -2,4 +2,5 @@ export * from './adb/index.js';
 export * from './codec.js';
 export * from './control/index.js';
 export * from './decoder/index.js';
+export * from './device-message/index.js';
 export * from './options/index.js';