feat(scrcpy): video stream works

.vscode/settings.json

@@ -5,14 +5,17 @@
         "CNXN",
         "Callout",
         "Deserialization",
+        "Muxer",
         "PKCS",
         "RSASSA",
         "Scrcpy",
         "WRTE",
         "addrs",
+        "brotli",
         "fluentui",
         "genymobile",
         "getprop",
+        "jmuxer",
         "killforward",
         "lapo",
         "localabstract",
@@ -32,6 +35,7 @@
         "webusb",
         "wifi",
         "wirelessly",
-        "yume"
+        "yume",
+        "zstd"
     ]
 }

packages/adb/src/adb.ts

@@ -17,7 +17,8 @@ export enum AdbPropKey {
 export class Adb {
     private packetDispatcher: AdbPacketDispatcher;
 
-    private backend: AdbBackend;
+    public readonly backend: AdbBackend;
+
     public get onDisconnected() { return this.backend.onDisconnected; }
 
     private _connected = false;

packages/adb/src/commands/reverse.ts

@@ -65,15 +65,13 @@ export class AdbReverseCommand extends AutoDisposable {
 
         const success = this.dispatcher.backend.decodeUtf8(await buffered.read(4)) === 'OKAY';
         if (success) {
-            const response = await AdbReverseStringResponse.deserialize(buffered);
+            if (deviceAddress.startsWith('tcp:')) {
-
+                const response = await AdbReverseStringResponse.deserialize(buffered);
-            if (deviceAddress === 'tcp:0') {
                 deviceAddress = `tcp:${Number.parseInt(response.content!, 10)}`;
             }
 
             this.localPortToHandler.set(localPort, handler);
             this.deviceAddressToLocalPort.set(deviceAddress, localPort);
-
             return deviceAddress;
         } else {
             return await AdbReverseErrorResponse.deserialize(buffered);

packages/adb/src/stream/buffered-stream.ts

@@ -37,15 +37,28 @@ export class BufferedStream<T extends Stream> {
             index = buffer.byteLength;
             this.buffer = undefined;
         } else {
+            const buffer = await this.stream.read(length);
+            if (buffer.byteLength === length) {
+                return buffer;
+            }
+
+            if (buffer.byteLength > length) {
+                this.buffer = new Uint8Array(buffer, length);
+                return buffer.slice(0, length);
+            }
+
             array = new Uint8Array(length);
-            index = 0;
+            array.set(new Uint8Array(buffer), 0);
+            index = buffer.byteLength;
         }
 
         while (index < length) {
-            const buffer = await this.stream.read(length - index);
+            const left = length - index;
-            if (buffer.byteLength > length - index) {
+
-                array.set(new Uint8Array(buffer, 0, length), index);
+            const buffer = await this.stream.read(left);
-                this.buffer = new Uint8Array(buffer, length);
+            if (buffer.byteLength > left) {
+                array.set(new Uint8Array(buffer, 0, left), index);
+                this.buffer = new Uint8Array(buffer, left);
                 return array.buffer;
             }
 

packages/demo/package-lock.json

@@ -3858,6 +3858,11 @@
 			"integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
 			"dev": true
 		},
+		"jmuxer": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/jmuxer/-/jmuxer-1.2.0.tgz",
+			"integrity": "sha512-ekJ/3D/poGdkNtwcpe4hXMOFTJuki9dkT19dmZec5eXB3nlAk1+DEtOjigwthx1d5+SA87llPKzxGiqBip2qrw=="
+		},
 		"js-tokens": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

packages/demo/package.json

@@ -43,6 +43,7 @@
         "@yume-chan/adb": "^0.0.1",
         "@yume-chan/adb-backend-web": "^0.0.1",
         "@yume-chan/struct": "^0.0.0",
+        "jmuxer": "^1.2.0",
         "react": "16.13.1",
         "react-dom": "16.13.1",
         "react-router-dom": "5.2.0",

packages/demo/src/index.tsx

@@ -8,7 +8,7 @@ import Connect from './connect';
 import ErrorDialogProvider from './error-dialog';
 import './index.css';
 import { CacheRoute, CacheSwitch } from './router';
-import { FileManager, FrameBuffer, Intro, Shell, TcpIp } from './routes';
+import { FileManager, FrameBuffer, Intro, Scrcpy, Shell, TcpIp } from './routes';
 
 initializeIcons();
 
@@ -87,6 +87,14 @@ function App(): JSX.Element | null {
                 <FrameBuffer device={device} />
             ),
         },
+        {
+            path: '/scrcpy',
+            name: 'Scrcpy',
+            noCache: true,
+            children: (
+                <Scrcpy device={device} />
+            ),
+        },
     ], [device]);
 
     return (

packages/demo/src/routes/index.ts

@@ -1,6 +1,7 @@
 export * from './file-manager';
 export * from './framebuffer';
 export * from './intro';
+export * from './scrcpy';
 export * from './shell';
 export * from './tcpip';
 export * from './type';

packages/demo/src/routes/scrcpy/index.tsx

@@ -1,27 +1,42 @@
-import { PrimaryButton } from '@fluentui/react';
+import { PrimaryButton, Stack, StackItem } from '@fluentui/react';
 import { AdbBufferedStream, AdbStream, EventQueue } from '@yume-chan/adb';
 import { Struct } from '@yume-chan/struct';
 import serverUrl from 'file-loader!./scrcpy-server';
+import JMuxer from 'jmuxer';
 import React, { useCallback, useRef, useState } from 'react';
-import { withDisplayName } from '../../utils';
+import { ResizeObserver, withDisplayName } from '../../utils';
 import { RouteProps } from '../type';
 
 const DeviceServerPath = '/data/local/tmp/scrcpy-server.jar';
 
 const Size =
-    new Struct({ littleEndian: true })
+    new Struct()
         .uint16('width')
         .uint16('height');
 
 const VideoPacket =
     new Struct()
-        .uint64('pts')
+        .int64('pts')
         .uint32('size')
         .arrayBuffer('data', { lengthField: 'size' });
 
-async function receiveVideo(stream: AdbBufferedStream) {
+const NoPts = BigInt(-1);
+
+async function receiveVideo(stream: AdbBufferedStream, jmuxer: JMuxer) {
+    let lastPts = BigInt(0);
     while (true) {
-        await VideoPacket.deserialize(stream);
+        const { pts, data } = await VideoPacket.deserialize(stream);
+
+        let duration: number | undefined;
+        if (pts !== NoPts) {
+            duration = Number(pts - lastPts) / 1000;
+            lastPts = pts;
+        }
+
+        jmuxer.feed({
+            video: new Uint8Array(data!),
+            duration,
+        });
     }
 }
 
@@ -44,16 +59,34 @@ async function receiveControl(stream: AdbBufferedStream) {
     }
 }
 
+export const enum ScrcpyLogLevel {
+    Debug = 'debug',
+    Info = 'info',
+    Warn = 'warn',
+    Error = 'error',
+}
+
+export const enum ScrcpyScreenOrientation {
+    Unlocked = -1,
+    Portrait = 0,
+    Landscape = 1,
+    PortraitFlipped = 2,
+    LandscapeFlipped = 3,
+}
+
 export const Scrcpy = withDisplayName('Scrcpy', ({
     device
 }: RouteProps): JSX.Element | null => {
-    const [width, setWidth] = useState(0);
+    const [running, setRunning] = useState(false);
-    const [height, setHeight] = useState(0);
+
+    const videoRef = useRef<HTMLVideoElement | null>(null);
+    const [videoWidth, setVideoWidth] = useState(0);
+    const [videoHeight, setVideoHeight] = useState(0);
     const [scale, setScale] = useState(1);
 
     const controlStreamRef = useRef<AdbBufferedStream | undefined>();
 
-    const connect = useCallback(async () => {
+    const start = useCallback(async () => {
         if (!device) {
             return;
         }
@@ -76,11 +109,11 @@ export const Scrcpy = withDisplayName('Scrcpy', ({
             '/', // unused
             'com.genymobile.scrcpy.Server',
             '1.16', // SCRCPY_VERSION
-            'error', // log_level
+            ScrcpyLogLevel.Debug,
             '0', // max_size (0: unlimited)
             '8000000', // bit_rate
             '0', // max_fps
-            '-1', // lock_video_orientation (-1: unlocked)
+            ScrcpyScreenOrientation.Unlocked.toString(), // lock_video_orientation (-1: unlocked)
             'false', // tunnel_forward
             '-', // crop
             'true', // always send frame meta (packet boundaries + timestamp)
@@ -90,61 +123,105 @@ export const Scrcpy = withDisplayName('Scrcpy', ({
             'true', // stay_awake
             '-', // codec_options
         );
+        server.onData(data => {
+            console.log(device.backend.decodeUtf8(data));
+        });
+        server.onClose(() => {
+            console.log('server stopped');
+        });
 
         const videoStream = new AdbBufferedStream(await listener.next());
         const controlStream = new AdbBufferedStream(await listener.next());
         controlStreamRef.current = controlStream;
 
-        await device.reverse.remove(reverseDeviceAddress);
+        device.reverse.remove(reverseDeviceAddress);
 
         // device name, we have already knew it from adb
         await videoStream.read(64);
 
         const { width, height } = await Size.deserialize(videoStream);
-        setWidth(width);
+        setVideoWidth(width);
-        setHeight(height);
+        setVideoHeight(height);
+
+        const jmuxer = new JMuxer({
+            node: videoRef.current!,
+            mode: 'video',
+            flushingTime: 0,
+        });
 
         await Promise.all([
-            receiveVideo(videoStream),
+            receiveVideo(videoStream, jmuxer),
             receiveControl(controlStream),
         ]);
 
+        jmuxer.destroy();
         await server.close();
     }, [device]);
 
-    const handleTouchStart = useCallback((e: React.TouchEvent<HTMLDivElement>) => {
+    const handleTouchStart = useCallback((e: React.TouchEvent<HTMLVideoElement>) => {
         controlStreamRef.current!.write(new ArrayBuffer(10));
     }, []);
 
-    const handleTouchMove = useCallback((e: React.TouchEvent<HTMLDivElement>) => {
+    const handleTouchMove = useCallback((e: React.TouchEvent<HTMLVideoElement>) => {
 
     }, []);
 
-    const handleTouchEnd = useCallback((e: React.TouchEvent<HTMLDivElement>) => {
+    const handleTouchEnd = useCallback((e: React.TouchEvent<HTMLVideoElement>) => {
 
     }, []);
 
-    const handleKeyPress = useCallback((e: React.KeyboardEvent<HTMLDivElement>) => {
+    const handleKeyPress = useCallback((e: React.KeyboardEvent<HTMLVideoElement>) => {
 
     }, []);
 
+    const handleCanPlay = useCallback(() => {
+        videoRef.current!.play();
+    }, []);
+
+    const handleResize = useCallback((width: number, height: number) => {
+        if (videoWidth === 0) {
+            setScale(1);
+            return;
+        }
+
+        const videoRatio = videoWidth / videoHeight;
+        const containerRatio = width / height;
+        if (videoRatio > containerRatio) {
+            setScale(width / videoWidth);
+        } else {
+            setScale(height / videoHeight);
+        }
+    }, [videoWidth, videoHeight]);
+
     return (
         <>
-            <PrimaryButton
+            <StackItem>
-                content="Connect"
+                <Stack horizontal>
-                onClick={connect}
+                    <PrimaryButton
-            />
+                        text="Start"
-            <video
+                        disabled={!device}
-                width={width}
+                        onClick={start}
-                height={height}
+                    />
-                style={{ transform: `scale(${scale})` }}
+                </Stack>
-            />
+            </StackItem>
-            <div
+            <StackItem grow>
-                onTouchStart={handleTouchStart}
+                <ResizeObserver
-                onTouchMove={handleTouchMove}
+                    style={{ position: 'relative', width: '100%', height: '100%' }}
-                onTouchEnd={handleTouchEnd}
+                    onResize={handleResize}
-                onKeyPress={handleKeyPress}
+                >
-            />
+                    <video
+                        ref={videoRef}
+                        width={videoWidth}
+                        height={videoHeight}
+                        style={{ position: 'absolute', transform: `scale(${scale})`, transformOrigin: 'top left' }}
+                        onCanPlay={handleCanPlay}
+                        onTouchStart={handleTouchStart}
+                        onTouchMove={handleTouchMove}
+                        onTouchEnd={handleTouchEnd}
+                        onKeyPress={handleKeyPress}
+                    />
+                </ResizeObserver>
+            </StackItem>
         </>
     );
 });

packages/demo/src/types.d.ts

@@ -41,3 +41,37 @@ declare module 'streamsaver' {
 }
 
 declare module 'file-loader!*';
+
+declare module 'jmuxer' {
+    export interface JMuxerOptions {
+        node: string | HTMLVideoElement;
+
+        mode?: 'video' | 'audio' | 'both';
+
+        flushingTime?: number;
+
+        clearBuffer?: boolean;
+
+        fps?: number;
+
+        onReady?: () => void;
+
+        debug?: boolean;
+    }
+
+    export interface JMuxerData {
+        video?: Uint8Array;
+
+        audio?: Uint8Array;
+
+        duration?: number;
+    }
+
+    export default class JMuxer {
+        constructor(options: JMuxerOptions);
+
+        feed(data: JMuxerData): void;
+
+        destroy(): void;
+    }
+}

packages/demo/src/utils/resize-observer.tsx

@@ -1,8 +1,9 @@
+import { createMergedRef } from '@fluentui/react';
 import React, { CSSProperties, HTMLAttributes, PropsWithChildren, useCallback, useRef } from 'react';
-import { withDisplayName } from './with-display-name';
+import { forwardRef } from './with-display-name';
 
 export interface ResizeObserverProps extends HTMLAttributes<HTMLDivElement>, PropsWithChildren<{}> {
-    onResize: () => void;
+    onResize: (width: number, height: number) => void;
 }
 
 const iframeStyle: CSSProperties = {
@@ -14,17 +15,21 @@ const iframeStyle: CSSProperties = {
     visibility: 'hidden',
 };
 
-export const ResizeObserver = withDisplayName('ResizeObserver', ({
+export const ResizeObserver = forwardRef('ResizeObserver', ({
     onResize,
     style,
     children,
     ...rest
-}: ResizeObserverProps): JSX.Element | null => {
+}: ResizeObserverProps, ref): JSX.Element | null => {
-    const onResizeRef = useRef<() => void>(onResize);
+    const onResizeRef = useRef<(width: number, height: number) => void>(onResize);
     onResizeRef.current = onResize;
 
+    const containerRef = useRef<HTMLDivElement | null>(null);
+    const mergedRef = createMergedRef()(ref, containerRef);
+
     const handleResize = useCallback(() => {
-        onResizeRef.current();
+        const { width, height } = containerRef.current!.getBoundingClientRect();
+        onResizeRef.current(width, height);
     }, []);
 
     const handleIframeRef = useCallback((element: HTMLIFrameElement | null) => {
@@ -46,7 +51,7 @@ export const ResizeObserver = withDisplayName('ResizeObserver', ({
     }, [style]);
 
     return (
-        <div style={containerStyle} {...rest}>
+        <div ref={mergedRef} style={containerStyle} {...rest}>
             <iframe ref={handleIframeRef} style={iframeStyle} />
             {children}
         </div>

packages/demo/src/utils/with-display-name.ts

@@ -1,3 +1,4 @@
+import React from 'react';
 import { memo, NamedExoticComponent } from 'react';
 
 export function withDisplayName<P extends object>(
@@ -7,3 +8,11 @@ export function withDisplayName<P extends object>(
     Component.displayName = name;
     return memo(Component);
 }
+
+export function forwardRef<P extends object>(
+    name: string,
+    Component: React.ForwardRefRenderFunction<unknown, P>
+) {
+    Component.displayName = name;
+    return memo(React.forwardRef(Component));
+}

packages/struct/src/field/number.ts

@@ -4,13 +4,16 @@ import { FieldDescriptorBase, FieldDescriptorBaseOptions, FieldType } from './de
 
 export namespace Number {
     export type TypeScriptType<T extends SubType> =
-        T extends SubType.Uint64 ? bigint : number;
+        T extends SubType.Uint64 ? bigint :
+        T extends SubType.Int64 ? bigint :
+        number;
 
     export const enum SubType {
         Uint16,
         Int32,
         Uint32,
         Uint64,
+        Int64,
     }
 
     export const SizeMap: Record<SubType, number> = {
@@ -18,6 +21,7 @@ export namespace Number {
         [SubType.Int32]: 4,
         [SubType.Uint32]: 4,
         [SubType.Uint64]: 8,
+        [SubType.Int64]: 8,
     };
 
     export const DataViewGetterMap = {
@@ -25,6 +29,7 @@ export namespace Number {
         [SubType.Int32]: 'getInt32',
         [SubType.Uint32]: 'getUint32',
         [SubType.Uint64]: 'getBigUint64',
+        [SubType.Int64]: 'getBigInt64',
     } as const;
 
     export const DataViewSetterMap = {
@@ -32,6 +37,7 @@ export namespace Number {
         [SubType.Int32]: 'setInt32',
         [SubType.Uint32]: 'setUint32',
         [SubType.Uint64]: 'setBigUint64',
+        [SubType.Int64]: 'setBigInt64',
     } as const;
 }
 

packages/struct/src/struct.ts

@@ -256,6 +256,22 @@ export default class Struct<
         );
     }
 
+    public int64<
+        TName extends string,
+        TTypeScriptType = Number.TypeScriptType<Number.SubType.Int64>
+    >(
+        name: TName,
+        options: FieldDescriptorBaseOptions = {},
+        _typescriptType?: TTypeScriptType,
+    ) {
+        return this.number(
+            name,
+            Number.SubType.Int64,
+            options,
+            _typescriptType
+        );
+    }
+
     private array: AddArrayFieldDescriptor<TResult, TInit, TExtra, TAfterParsed> = (
         name: string,
         type: Array.SubType,