feat(scrcpy): support sendFrameMeta: false

fixes #374

apps/demo/pages/index.mdx

@@ -3,6 +3,10 @@ import Head from 'next/head';
 import { ExternalLink } from '../components';
 import { RouteStackProps } from "../utils";
 
+<!--
+cspell: ignore cybojenix
+-->
+
 This is a demo for my <ExternalLink href="https://github.com/yume-chan/ya-webadb/">ya-webadb</ExternalLink> project, which can use ADB protocol to control Android phones, directly from Web browsers (or Node.js).
 
 It started because I want to try the <ExternalLink href="https://developer.mozilla.org/en-US/docs/Web/API/USB">WebUSB</ExternalLink> API, and because I have an Android phone. It's not production-ready, and I don't recommend normal users to try it. If you have any questions or suggestions, please file an issue at <ExternalLink href="https://github.com/yume-chan/ya-webadb/issues">here</ExternalLink>.
@@ -10,7 +14,6 @@ It started because I want to try the <ExternalLink href="https://developer.mozil
 It was called "ya-webadb" (Yet Another WebADB), because there have already been several similar projects, for example:
 
 * <ExternalLink href="https://github.com/webadb/webadb.js">webadb/webadb.js</ExternalLink>
-<!-- cspell: disable-next-line -->
 * <ExternalLink href="https://github.com/cybojenix/WebADB">cybojenix/WebADB</ExternalLink>
 
 However, they are all pretty simple and not maintained, so I decided to make my own.

apps/demo/pages/scrcpy.tsx

@@ -336,7 +336,7 @@ class ScrcpyPageState {
                 while (this.rendererContainer.firstChild) {
                     this.rendererContainer.firstChild.remove();
                 }
-                this.rendererContainer.appendChild(this.decoder.element);
+                this.rendererContainer.appendChild(this.decoder.renderer);
             }
         });
 
@@ -454,19 +454,19 @@ class ScrcpyPageState {
             client.onOutput(action(line => this.log.push(line)));
             client.onClose(this.stop);
 
-            client.onSizeChanged(action((size) => {
+            client.onEncodingChanged(action((encoding) => {
-                const { croppedWidth, croppedHeight, } = size;
+                const { croppedWidth, croppedHeight, } = encoding;
 
                 this.log.push(`[client] Video size changed: ${croppedWidth}x${croppedHeight}`);
 
                 this.width = croppedWidth;
                 this.height = croppedHeight;
 
-                decoder.setSize(size);
+                decoder.changeEncoding(encoding);
             }));
 
-            client.onVideoData(({ data }) => {
+            client.onVideoData((data) => {
-                decoder.feed(data);
+                decoder.feedData(data);
             });
 
             client.onClipboardChange(content => {

libraries/adb/src/stream/buffered-stream.ts

@@ -1,13 +1,25 @@
-import { StructAsyncDeserializeStream } from '@yume-chan/struct';
+import { StructAsyncDeserializeStream, ValueOrPromise } from '@yume-chan/struct';
 import { AdbSocket, AdbSocketInfo } from '../socket';
 import { AdbSocketStream } from './stream';
 
+export class StreamEndedError extends Error {
+    public constructor() {
+        super('Stream ended');
+
+        // Fix Error's prototype chain when compiling to ES5
+        Object.setPrototypeOf(this, new.target.prototype);
+    }
+}
+
 export interface Stream {
     /**
+     * When the stream is ended (no more data can be read),
+     * An `StreamEndedError` should be thrown.
+     *
      * @param length A hint of how much data should be read.
-     * @returns Data, which can be either more or less than `length`
+     * @returns Data, which can be either more or less than `length`.
      */
-    read(length: number): ArrayBuffer | Promise<ArrayBuffer>;
+    read(length: number): ValueOrPromise<ArrayBuffer>;
 
     close?(): void;
 }
@@ -21,7 +33,13 @@ export class BufferedStream<T extends Stream> {
         this.stream = stream;
     }
 
-    public async read(length: number): Promise<ArrayBuffer> {
+    /**
+     *
+     * @param length
+     * @param readToEnd When `true`, allow less data to be returned if the stream has reached its end.
+     * @returns
+     */
+    public async read(length: number, readToEnd: boolean = false): Promise<ArrayBuffer> {
         let array: Uint8Array;
         let index: number;
         if (this.buffer) {
@@ -51,6 +69,7 @@ export class BufferedStream<T extends Stream> {
             index = buffer.byteLength;
         }
 
+        try {
             while (index < length) {
                 const left = length - index;
 
@@ -64,6 +83,14 @@ export class BufferedStream<T extends Stream> {
                 array.set(new Uint8Array(buffer), index);
                 index += buffer.byteLength;
             }
+        }
+        catch (e) {
+            if (readToEnd && e instanceof StreamEndedError) {
+                return array.buffer;
+            }
+
+            throw e;
+        }
 
         return array.buffer;
     }

libraries/adb/src/stream/stream.ts

@@ -2,6 +2,7 @@ import { once } from '@yume-chan/event';
 import { ValueOrPromise } from '@yume-chan/struct';
 import { AdbSocket, AdbSocketInfo } from '../socket';
 import { EventQueue } from '../utils';
+import { StreamEndedError } from "./buffered-stream";
 
 export class AdbSocketStream implements AdbSocketInfo {
     private socket: AdbSocket;
@@ -36,7 +37,7 @@ export class AdbSocketStream implements AdbSocketInfo {
         try {
             return await this.queue.dequeue();
         } catch {
-            throw new Error('Can not read after AdbSocketStream has been closed');
+            throw new StreamEndedError();
         }
     }
 

libraries/event/src/utils.ts

@@ -1,7 +1,7 @@
 import { PromiseResolver } from '@yume-chan/async';
 import { Event } from './event';
 
-export async function once<T>(event: Event<T>): Promise<T> {
+export async function once<T>(event: Event<T, any>): Promise<T> {
     const resolver = new PromiseResolver<T>();
     const dispose = event(resolver.resolve);
     const result = await resolver.promise;

libraries/scrcpy/src/client.ts

@@ -3,10 +3,9 @@ import { PromiseResolver } from '@yume-chan/async';
 import { EventEmitter } from '@yume-chan/event';
 import Struct from '@yume-chan/struct';
 import { AndroidKeyEventAction, AndroidMotionEventAction, ScrcpyControlMessageType, ScrcpyInjectKeyCodeControlMessage, ScrcpyInjectTextControlMessage, ScrcpyInjectTouchControlMessage } from './message';
-import { ScrcpyOptions } from "./options";
+import { H264EncodingInfo, ScrcpyOptions } from "./options";
 import { ScrcpyInjectScrollControlMessage1_22 } from "./options/1_22";
 import { pushServer, PushServerOptions } from "./push-server";
-import { parse_sequence_parameter_set, SequenceParameterSet } from './sps';
 import { decodeUtf8 } from "./utils";
 
 function* splitLines(text: string): Generator<string, void, void> {
@@ -25,37 +24,11 @@ function* splitLines(text: string): Generator<string, void, void> {
     }
 }
 
-const VideoPacket =
-    new Struct()
-        .int64('pts')
-        .uint32('size')
-        .arrayBuffer('data', { lengthField: 'size' });
-
-export const NoPts = BigInt(-1);
-
-export type VideoPacket = typeof VideoPacket['TDeserializeResult'];
-
 const ClipboardMessage =
     new Struct()
         .uint32('length')
         .string('content', { lengthField: 'length' });
 
-export interface FrameSize {
-    sequenceParameterSet: SequenceParameterSet;
-
-    width: number;
-    height: number;
-
-    cropLeft: number;
-    cropRight: number;
-
-    cropTop: number;
-    cropBottom: number;
-
-    croppedWidth: number;
-    croppedHeight: number;
-}
-
 export class ScrcpyClient {
     public static pushServer(
         device: Adb,
@@ -129,10 +102,10 @@ export class ScrcpyClient {
     private _screenHeight: number | undefined;
     public get screenHeight() { return this._screenHeight; }
 
-    private readonly sizeChangedEvent = new EventEmitter<FrameSize>();
+    private readonly encodingChangedEvent = new EventEmitter<H264EncodingInfo>();
-    public get onSizeChanged() { return this.sizeChangedEvent.event; }
+    public get onEncodingChanged() { return this.encodingChangedEvent.event; }
 
-    private readonly videoDataEvent = new DataEventEmitter<VideoPacket>();
+    private readonly videoDataEvent = new DataEventEmitter<ArrayBuffer>();
     public get onVideoData() { return this.videoDataEvent.event; }
 
     private readonly clipboardChangeEvent = new EventEmitter<string>();
@@ -225,68 +198,16 @@ export class ScrcpyClient {
         }
 
         try {
-            let buffer: ArrayBuffer | undefined;
             while (this._running) {
-                const { pts, data } = await VideoPacket.deserialize(this.videoStream);
+                const { encodingInfo, videoData } = await this.options!.parseVideoStream(this.videoStream);
-                if (!data || data.byteLength === 0) {
+                if (encodingInfo) {
-                    continue;
+                    this._screenWidth = encodingInfo.croppedWidth;
+                    this._screenHeight = encodingInfo.croppedHeight;
+                    this.encodingChangedEvent.fire(encodingInfo);
                 }
-
+                if (videoData) {
-                if (pts === NoPts) {
+                    this.videoDataEvent.fire(videoData);
-                    const sequenceParameterSet = parse_sequence_parameter_set(data.slice(0));
-
-                    const {
-                        pic_width_in_mbs_minus1,
-                        pic_height_in_map_units_minus1,
-                        frame_mbs_only_flag,
-                        frame_crop_left_offset,
-                        frame_crop_right_offset,
-                        frame_crop_top_offset,
-                        frame_crop_bottom_offset,
-                    } = sequenceParameterSet;
-                    const width = (pic_width_in_mbs_minus1 + 1) * 16;
-                    const height = (pic_height_in_map_units_minus1 + 1) * (2 - frame_mbs_only_flag) * 16;
-                    const cropLeft = frame_crop_left_offset * 2;
-                    const cropRight = frame_crop_right_offset * 2;
-                    const cropTop = frame_crop_top_offset * 2;
-                    const cropBottom = frame_crop_bottom_offset * 2;
-
-                    const screenWidth = width - cropLeft - cropRight;
-                    const screenHeight = height - cropTop - cropBottom;
-                    this._screenWidth = screenWidth;
-                    this._screenHeight = screenHeight;
-
-                    this.sizeChangedEvent.fire({
-                        sequenceParameterSet,
-                        width,
-                        height,
-                        cropLeft: cropLeft,
-                        cropRight: cropRight,
-                        cropTop: cropTop,
-                        cropBottom: cropBottom,
-                        croppedWidth: screenWidth,
-                        croppedHeight: screenHeight,
-                    });
-
-                    buffer = data;
-                    continue;
                 }
-
-                let array: Uint8Array;
-                if (buffer) {
-                    array = new Uint8Array(buffer.byteLength + data!.byteLength);
-                    array.set(new Uint8Array(buffer));
-                    array.set(new Uint8Array(data!), buffer.byteLength);
-                    buffer = undefined;
-                } else {
-                    array = new Uint8Array(data!);
-                }
-
-                await this.videoDataEvent.fire({
-                    pts,
-                    size: array.byteLength,
-                    data: array.buffer,
-                });
             }
         } catch (e) {
             if (!this._running) {

libraries/scrcpy/src/decoder/index.ts

@@ -1,17 +1,17 @@
 import { Disposable } from "@yume-chan/event";
-import type { FrameSize } from "../client";
 import type { AndroidCodecLevel, AndroidCodecProfile } from "../codec";
+import type { H264EncodingInfo } from "../options";
 
 export interface H264Decoder extends Disposable {
     readonly maxProfile: AndroidCodecProfile;
 
     readonly maxLevel: AndroidCodecLevel;
 
-    readonly element: HTMLElement;
+    readonly renderer: HTMLElement;
 
-    setSize(size: FrameSize): void;
+    changeEncoding(size: H264EncodingInfo): void;
 
-    feed(data: ArrayBuffer): void;
+    feedData(data: ArrayBuffer): void;
 }
 
 export interface H264DecoderConstructor {

libraries/scrcpy/src/decoder/tinyh264/index.ts

@@ -1,7 +1,7 @@
 import { PromiseResolver } from "@yume-chan/async";
 import type { H264Decoder } from "..";
-import type { FrameSize } from '../../client';
 import { AndroidCodecLevel, AndroidCodecProfile } from "../../codec";
+import type { H264EncodingInfo } from '../../options';
 import { createTinyH264Wrapper, TinyH264Wrapper } from "./wrapper";
 
 let cachedInitializePromise: Promise<{ YuvBuffer: typeof import('yuv-buffer'), YuvCanvas: typeof import('yuv-canvas').default; }> | undefined;
@@ -23,7 +23,7 @@ export class TinyH264Decoder implements H264Decoder {
     public readonly maxLevel = AndroidCodecLevel.Level4;
 
     private _element: HTMLCanvasElement;
-    public get element() { return this._element; }
+    public get renderer() { return this._element; }
 
     private _yuvCanvas: import('yuv-canvas').default | undefined;
     private _initializer: PromiseResolver<TinyH264Wrapper> | undefined;
@@ -33,7 +33,7 @@ export class TinyH264Decoder implements H264Decoder {
         this._element = document.createElement('canvas');
     }
 
-    public async setSize(size: FrameSize) {
+    public async changeEncoding(size: H264EncodingInfo) {
         this.dispose();
 
         this._initializer = new PromiseResolver<TinyH264Wrapper>();
@@ -43,15 +43,15 @@ export class TinyH264Decoder implements H264Decoder {
             this._yuvCanvas = YuvCanvas.attach(this._element);;
         }
 
-        const { width, height } = size;
+        const { encodedWidth, encodedHeight } = size;
-        const chromaWidth = width / 2;
+        const chromaWidth = encodedWidth / 2;
-        const chromaHeight = height / 2;
+        const chromaHeight = encodedHeight / 2;
 
         this._element.width = size.croppedWidth;
         this._element.height = size.croppedHeight;
         const format = YuvBuffer.format({
-            width,
+            width: encodedWidth,
-            height,
+            height: encodedHeight,
             chromaWidth,
             chromaHeight,
             cropLeft: size.cropLeft,
@@ -65,12 +65,12 @@ export class TinyH264Decoder implements H264Decoder {
         const wrapper = await createTinyH264Wrapper();
         this._initializer.resolve(wrapper);
 
-        const uPlaneOffset = width * height;
+        const uPlaneOffset = encodedWidth * encodedHeight;
         const vPlaneOffset = uPlaneOffset + chromaWidth * chromaHeight;
         wrapper.onPictureReady(({ data }) => {
             const array = new Uint8Array(data);
             const frame = YuvBuffer.frame(format,
-                YuvBuffer.lumaPlane(format, array, width, 0),
+                YuvBuffer.lumaPlane(format, array, encodedWidth, 0),
                 YuvBuffer.chromaPlane(format, array, chromaWidth, uPlaneOffset),
                 YuvBuffer.chromaPlane(format, array, chromaWidth, vPlaneOffset)
             );
@@ -78,7 +78,7 @@ export class TinyH264Decoder implements H264Decoder {
         });
     }
 
-    public async feed(data: ArrayBuffer) {
+    public async feedData(data: ArrayBuffer) {
         if (!this._initializer) {
             throw new Error('Decoder not initialized');
         }

libraries/scrcpy/src/decoder/web-codecs/index.ts

@@ -1,7 +1,7 @@
 import type { ValueOrPromise } from "@yume-chan/struct";
 import type { H264Decoder } from "..";
-import type { FrameSize } from "../../client";
 import { AndroidCodecLevel, AndroidCodecProfile } from "../../codec";
+import type { H264EncodingInfo } from "../../options";
 
 function toHex(value: number) {
     return value.toString(16).padStart(2, '0').toUpperCase();
@@ -13,7 +13,7 @@ export class WebCodecsDecoder implements H264Decoder {
     public readonly maxLevel = AndroidCodecLevel.Level5;
 
     private _element: HTMLCanvasElement;
-    public get element() { return this._element; }
+    public get renderer() { return this._element; }
 
     private context: CanvasRenderingContext2D;
     private decoder: VideoDecoder;
@@ -31,22 +31,22 @@ export class WebCodecsDecoder implements H264Decoder {
         });
     }
 
-    public setSize(size: FrameSize): ValueOrPromise<void> {
+    public changeEncoding(encoding: H264EncodingInfo): ValueOrPromise<void> {
-        const { sequenceParameterSet: { profile_idc, constraint_set, level_idc } } = size;
+        const { profileIndex, constraintSet, levelIndex } = encoding;
 
-        this._element.width = size.croppedWidth;
+        this._element.width = encoding.croppedWidth;
-        this._element.height = size.croppedHeight;
+        this._element.height = encoding.croppedHeight;
 
         // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
         // ISO Base Media File Format Name Space
-        const codec = `avc1.${[profile_idc, constraint_set, level_idc].map(toHex).join('')}`;
+        const codec = `avc1.${[profileIndex, constraintSet, levelIndex].map(toHex).join('')}`;
         this.decoder.configure({
             codec: codec,
             optimizeForLatency: true,
         });
     }
 
-    feed(data: ArrayBuffer): ValueOrPromise<void> {
+    feedData(data: ArrayBuffer): ValueOrPromise<void> {
         this.decoder.decode(new EncodedVideoChunk({
             type: 'key',
             timestamp: 0,

libraries/scrcpy/src/options/1_16/index.ts

@@ -1,10 +1,11 @@
-import type { Adb } from "@yume-chan/adb";
+import type { Adb, AdbBufferedStream } from "@yume-chan/adb";
 import Struct, { placeholder } from "@yume-chan/struct";
-import { AndroidCodecLevel, AndroidCodecProfile } from "../codec";
+import { AndroidCodecLevel, AndroidCodecProfile } from "../../codec";
-import { ScrcpyClientConnection, ScrcpyClientConnectionOptions, ScrcpyClientForwardConnection, ScrcpyClientReverseConnection } from "../connection";
+import { ScrcpyClientConnection, ScrcpyClientConnectionOptions, ScrcpyClientForwardConnection, ScrcpyClientReverseConnection } from "../../connection";
-import { AndroidKeyEventAction, ScrcpyControlMessageType } from "../message";
+import { AndroidKeyEventAction, ScrcpyControlMessageType } from "../../message";
-import type { ScrcpyInjectScrollControlMessage1_22 } from "./1_22";
+import type { ScrcpyInjectScrollControlMessage1_22 } from "../1_22";
-import { ScrcpyLogLevel, ScrcpyOptions, ScrcpyOptionValue, ScrcpyScreenOrientation, toScrcpyOptionValue } from "./common";
+import { ScrcpyLogLevel, ScrcpyOptions, ScrcpyOptionValue, ScrcpyScreenOrientation, toScrcpyOptionValue, VideoStreamPacket } from "../common";
+import { parse_sequence_parameter_set } from "./sps";
 
 export interface CodecOptionsType {
     profile: AndroidCodecProfile;
@@ -65,9 +66,13 @@ export interface ScrcpyOptions1_16Type {
     /**
      * Send PTS so that the client may record properly
      *
-     * @default true
+     * Note: When `sendFrameMeta: false` is specified,
+     * `onChangeEncoding` event won't fire and `onVideoData` event doesn't
+     * merge sps/pps frame and first video frame. Which means you can't use
+     * the shipped decoders to render the video
+     * (You can still record the stream into a file).
      *
-     * TODO: Add support for `sendFrameMeta: false`
+     * @default true
      */
     sendFrameMeta: boolean;
 
@@ -87,6 +92,14 @@ export interface ScrcpyOptions1_16Type {
     encoderName: string;
 }
 
+export const VideoPacket =
+    new Struct()
+        .int64('pts')
+        .uint32('size')
+        .arrayBuffer('data', { lengthField: 'size' });
+
+export const NoPts = BigInt(-1);
+
 export const ScrcpyBackOrScreenOnEvent1_16 =
     new Struct()
         .uint8('type', placeholder<ScrcpyControlMessageType.BackOrScreenOn>());
@@ -104,6 +117,8 @@ export const ScrcpyInjectScrollControlMessage1_16 =
 export class ScrcpyOptions1_16<T extends ScrcpyOptions1_16Type = ScrcpyOptions1_16Type> implements ScrcpyOptions<T> {
     public value: Partial<T>;
 
+    private _streamHeader: ArrayBuffer | undefined;
+
     public constructor(value: Partial<ScrcpyOptions1_16Type>) {
         if (new.target === ScrcpyOptions1_16 &&
             value.logLevel === ScrcpyLogLevel.Verbose) {
@@ -180,6 +195,77 @@ export class ScrcpyOptions1_16<T extends ScrcpyOptions1_16Type = ScrcpyOptions1_
         return /\s+scrcpy --encoder-name '(.*?)'/;
     }
 
+    public async parseVideoStream(stream: AdbBufferedStream): Promise<VideoStreamPacket> {
+        if (this.value.sendFrameMeta === false) {
+            return {
+                videoData: await stream.read(1 * 1024 * 1024, true),
+            };
+        }
+
+        const { pts, data } = await VideoPacket.deserialize(stream);
+        if (!data || data.byteLength === 0) {
+            return {};
+        }
+
+        if (pts === NoPts) {
+            const sequenceParameterSet = parse_sequence_parameter_set(data.slice(0));
+
+            const {
+                profile_idc: profileIndex,
+                constraint_set: constraintSet,
+                level_idc: levelIndex,
+                pic_width_in_mbs_minus1,
+                pic_height_in_map_units_minus1,
+                frame_mbs_only_flag,
+                frame_crop_left_offset,
+                frame_crop_right_offset,
+                frame_crop_top_offset,
+                frame_crop_bottom_offset,
+            } = sequenceParameterSet;
+
+            const encodedWidth = (pic_width_in_mbs_minus1 + 1) * 16;
+            const encodedHeight = (pic_height_in_map_units_minus1 + 1) * (2 - frame_mbs_only_flag) * 16;
+            const cropLeft = frame_crop_left_offset * 2;
+            const cropRight = frame_crop_right_offset * 2;
+            const cropTop = frame_crop_top_offset * 2;
+            const cropBottom = frame_crop_bottom_offset * 2;
+
+            const croppedWidth = encodedWidth - cropLeft - cropRight;
+            const croppedHeight = encodedHeight - cropTop - cropBottom;
+
+            this._streamHeader = data;
+            return {
+                encodingInfo: {
+                    profileIndex,
+                    constraintSet,
+                    levelIndex,
+                    encodedWidth,
+                    encodedHeight,
+                    cropLeft,
+                    cropRight,
+                    cropTop,
+                    cropBottom,
+                    croppedWidth,
+                    croppedHeight,
+                }
+            };
+        }
+
+        let array: Uint8Array;
+        if (this._streamHeader) {
+            array = new Uint8Array(this._streamHeader.byteLength + data!.byteLength);
+            array.set(new Uint8Array(this._streamHeader));
+            array.set(new Uint8Array(data!), this._streamHeader.byteLength);
+            this._streamHeader = undefined;
+        } else {
+            array = new Uint8Array(data!);
+        }
+
+        return {
+            videoData: array.buffer,
+        };
+    }
+
     public serializeBackOrScreenOnControlMessage(action: AndroidKeyEventAction, device: Adb) {
         if (action === AndroidKeyEventAction.Down) {
             return ScrcpyBackOrScreenOnEvent1_16.serialize(

libraries/scrcpy/src/options/1_22.ts

@@ -25,8 +25,6 @@ export interface ScrcpyOptions1_22Type extends ScrcpyOptions1_21Type {
      * Implies `sendDeviceMeta: false`, `sendFrameMeta: false` and `sendDummyByte: false`
      *
      * @default false
-     *
-     * TODO: Add support for `sendFrameMeta: false`
      */
     rawVideoStream: boolean;
 }
@@ -45,13 +43,6 @@ export class ScrcpyOptions1_22<T extends ScrcpyOptions1_22Type = ScrcpyOptions1_
             init.sendDeviceMeta = false;
             init.sendFrameMeta = false;
             init.sendDummyByte = false;
-            // TODO: Add support for `sendFrameMeta: false`
-            throw new Error('`rawVideoStream:true` is not supported');
-        }
-
-        if (init.sendFrameMeta === false) {
-            // TODO: Add support for `sendFrameMeta: false`
-            throw new Error('`sendFrameMeta:false` is not supported');
         }
 
         super(init);

libraries/scrcpy/src/options/common.ts

@@ -1,4 +1,4 @@
-import type { Adb } from "@yume-chan/adb";
+import type { Adb, AdbBufferedStream } from "@yume-chan/adb";
 import type { ScrcpyClientConnection } from "../connection";
 import type { AndroidKeyEventAction } from "../message";
 import type { ScrcpyInjectScrollControlMessage1_22 } from "./1_22";
@@ -22,6 +22,30 @@ export enum ScrcpyScreenOrientation {
     LandscapeFlipped = 3,
 }
 
+export interface H264EncodingInfo {
+    profileIndex: number;
+    constraintSet: number;
+    levelIndex: number;
+
+    encodedWidth: number;
+    encodedHeight: number;
+
+    cropLeft: number;
+    cropRight: number;
+
+    cropTop: number;
+    cropBottom: number;
+
+    croppedWidth: number;
+    croppedHeight: number;
+}
+
+export interface VideoStreamPacket {
+    encodingInfo?: H264EncodingInfo | undefined;
+
+    videoData?: ArrayBuffer | undefined;
+}
+
 export interface ScrcpyOptions<T> {
     value: Partial<T>;
 
@@ -31,6 +55,8 @@ export interface ScrcpyOptions<T> {
 
     createConnection(device: Adb): ScrcpyClientConnection;
 
+    parseVideoStream(stream: AdbBufferedStream): Promise<VideoStreamPacket>;
+
     serializeBackOrScreenOnControlMessage(
         action: AndroidKeyEventAction,
         device: Adb

libraries/struct/src/struct.ts

@@ -1,3 +1,5 @@
+// cspell: ignore Syncbird
+
 import type { StructAsyncDeserializeStream, StructDeserializeStream, StructFieldDefinition, StructFieldValue, StructOptions } from './basic';
 import { StructDefaultOptions, StructValue } from './basic';
 import { Syncbird } from "./syncbird";
@@ -103,7 +105,7 @@ interface ArrayBufferLikeFieldCreator<
 /**
  * Similar to `ArrayBufferLikeFieldCreator`, but bind to `TType`
  */
-interface BindedArrayBufferLikeFieldDefinitionCreator<
+interface BoundArrayBufferLikeFieldDefinitionCreator<
     TFields extends object,
     TOmitInitKey extends PropertyKey,
     TExtra extends object,
@@ -435,7 +437,7 @@ export class Struct<
             }
         };
 
-    public arrayBuffer: BindedArrayBufferLikeFieldDefinitionCreator<
+    public arrayBuffer: BoundArrayBufferLikeFieldDefinitionCreator<
         TFields,
         TOmitInitKey,
         TExtra,
@@ -448,7 +450,7 @@ export class Struct<
             return this.arrayBufferLike(name, ArrayBufferFieldType.instance, options);
         };
 
-    public uint8ClampedArray: BindedArrayBufferLikeFieldDefinitionCreator<
+    public uint8ClampedArray: BoundArrayBufferLikeFieldDefinitionCreator<
         TFields,
         TOmitInitKey,
         TExtra,
@@ -461,7 +463,7 @@ export class Struct<
             return this.arrayBufferLike(name, Uint8ClampedArrayFieldType.instance, options);
         };
 
-    public string: BindedArrayBufferLikeFieldDefinitionCreator<
+    public string: BoundArrayBufferLikeFieldDefinitionCreator<
         TFields,
         TOmitInitKey,
         TExtra,