feat(adb): support ls_v2 and fixed_push_mkdir features

.vscode/settings.json

@@ -16,6 +16,7 @@
         "genymobile",
         "Genymobile's",
         "getprop",
+        "griffel",
         "keyof",
         "laggy",
         "localabstract",

apps/demo/src/components/hex-viewer.tsx

@@ -4,9 +4,18 @@ import { withDisplayName } from "../utils";
 const useClasses = makeStyles({
 });
 
-export const HexViewer = withDisplayName('HexViewer')(({ }) => {
+export interface HexViewer {
+    data: Uint8Array;
+}
+
+export const HexViewer = withDisplayName('HexViewer')(({
+
+}) => {
     const classes = useClasses();
 
+    // Because ADB packets are usually small,
+    // so don't add virtualization now.
+
     return (
         <div>
 

apps/demo/src/pages/device-info.tsx

@@ -9,11 +9,11 @@ import { globalState } from '../state';
 import { Icons, RouteStackProps } from "../utils";
 
 const KNOWN_FEATURES: Record<string, string> = {
-    'shell_v2': `"shell" command now supports separating child process's stdout and stderr, and returning exit code`,
+    [AdbFeatures.ShellV2]: `"shell" command now supports separating child process's stdout and stderr, and returning exit code`,
     // 'cmd': '',
     [AdbFeatures.StatV2]: '"sync" command now supports "STA2" (returns more information of a file than old "STAT") and "LST2" (returns information of a directory) sub command',
-    'ls_v2': '"sync" command now supports "LST2" sub command which returns more information when listing a directory than old "LIST"',
+    [AdbFeatures.ListV2]: '"sync" command now supports "LST2" sub command which returns more information when listing a directory than old "LIST"',
-    // 'fixed_push_mkdir': '',
+    [AdbFeatures.FixedPushMkdir]: 'Android 9 (P) introduced a bug that pushing files to a non-existing directory would fail. This feature indicates it\'s fixed (Android 10)',
     // 'apex': '',
     // 'abb': '',
     // 'fixed_push_symlink_timestamp': '',

apps/demo/src/pages/file-manager.tsx

@@ -2,7 +2,7 @@ import { Breadcrumb, concatStyleSets, ContextualMenu, ContextualMenuItem, Detail
 import { FileIconType, getFileTypeIconProps, initializeFileTypeIcons } from "@fluentui/react-file-type-icons";
 import { useConst } from '@fluentui/react-hooks';
 import { getIcon } from '@fluentui/style-utilities';
-import { AdbSyncEntryResponse, ADB_SYNC_MAX_PACKET_SIZE, ChunkStream, LinuxFileType, ReadableStream } from '@yume-chan/adb';
+import { AdbFeatures, ADB_SYNC_MAX_PACKET_SIZE, ChunkStream, LinuxFileType, type AdbSyncEntry } from '@yume-chan/adb';
 import { action, autorun, makeAutoObservable, observable, runInAction } from "mobx";
 import { observer } from "mobx-react-lite";
 import { NextPage } from "next";
@@ -12,15 +12,15 @@ import path from 'path';
 import { useCallback, useEffect, useState } from 'react';
 import { CommandBar, NoSsr } from '../components';
 import { globalState } from '../state';
-import { asyncEffect, formatSize, formatSpeed, Icons, pickFile, ProgressStream, RouteStackProps, saveFile } from '../utils';
+import { asyncEffect, createFileStream, formatSize, formatSpeed, Icons, pickFile, ProgressStream, RouteStackProps, saveFile } from '../utils';
 
 initializeFileTypeIcons();
 
-interface ListItem extends AdbSyncEntryResponse {
+interface ListItem extends AdbSyncEntry {
     key: string;
 }
 
-function toListItem(item: AdbSyncEntryResponse): ListItem {
+function toListItem(item: AdbSyncEntry): ListItem {
     return { ...item, key: item.name! };
 }
 
@@ -139,7 +139,7 @@ class FileManagerState {
                                     const item = this.selectedItems[0];
                                     const itemPath = path.resolve(this.path, item.name);
                                     await sync.read(itemPath)
-                                        .pipeTo(saveFile(item.name, item.size));
+                                        .pipeTo(saveFile(item.name, Number(item.size)));
                                 } catch (e) {
                                     globalState.showErrorDialog(e instanceof Error ? e.message : `${e}`);
                                 } finally {
@@ -226,7 +226,7 @@ class FileManagerState {
                 minWidth: ICON_SIZE,
                 maxWidth: ICON_SIZE,
                 isCollapsible: true,
-                onRender(item: AdbSyncEntryResponse) {
+                onRender(item: AdbSyncEntry) {
                     let iconName: string;
 
                     switch (item.type) {
@@ -254,7 +254,7 @@ class FileManagerState {
                 name: 'Name',
                 minWidth: 0,
                 isRowHeader: true,
-                onRender(item: AdbSyncEntryResponse) {
+                onRender(item: AdbSyncEntry) {
                     return (
                         <span className={classNames.name} data-selection-invoke>
                             {item.name}
@@ -267,7 +267,7 @@ class FileManagerState {
                 name: 'Permission',
                 minWidth: 0,
                 isCollapsible: true,
-                onRender(item: AdbSyncEntryResponse) {
+                onRender(item: AdbSyncEntry) {
                     return `${(item.mode >> 6 & 0b100).toString(8)}${(item.mode >> 3 & 0b100).toString(8)}${(item.mode & 0b100).toString(8)}`;
                 }
             },
@@ -276,9 +276,9 @@ class FileManagerState {
                 name: 'Size',
                 minWidth: 0,
                 isCollapsible: true,
-                onRender(item: AdbSyncEntryResponse) {
+                onRender(item: AdbSyncEntry) {
                     if (item.type === LinuxFileType.File) {
-                        return formatSize(item.size);
+                        return formatSize(Number(item.size));
                     }
                     return '';
                 }
@@ -288,12 +288,35 @@ class FileManagerState {
                 name: 'Last Modified Time',
                 minWidth: 150,
                 isCollapsible: true,
-                onRender(item: AdbSyncEntryResponse) {
+                onRender(item: AdbSyncEntry) {
-                    return new Date(item.mtime * 1000).toLocaleString();
+                    return new Date(Number(item.mtime) * 1000).toLocaleString();
                 },
             }
         ];
 
+        if (globalState.device?.features?.includes(AdbFeatures.ListV2)) {
+            list.push(
+                {
+                    key: 'ctime',
+                    name: 'Creation Time',
+                    minWidth: 150,
+                    isCollapsible: true,
+                    onRender(item: AdbSyncEntry) {
+                        return new Date(Number(item.ctime!) * 1000).toLocaleString();
+                    },
+                },
+                {
+                    key: 'atime',
+                    name: 'Last Access Time',
+                    minWidth: 150,
+                    isCollapsible: true,
+                    onRender(item: AdbSyncEntry) {
+                        return new Date(Number(item.atime!) * 1000).toLocaleString();
+                    },
+                },
+            );
+        }
+
         for (const item of list) {
             item.onColumnClick = (e, column) => {
                 if (this.sortKey === column.key) {
@@ -368,7 +391,7 @@ class FileManagerState {
         const sync = await globalState.device.sync();
 
         const items: ListItem[] = [];
-        const linkItems: AdbSyncEntryResponse[] = [];
+        const linkItems: AdbSyncEntry[] = [];
         const intervalId = setInterval(() => {
             if (signal.aborted) {
                 return;
@@ -401,7 +424,7 @@ class FileManagerState {
 
                 if (!await sync.isDirectory(path.resolve(currentPath, entry.name!))) {
                     entry.mode = (LinuxFileType.File << 12) | entry.permission;
-                    entry.size = 0;
+                    entry.size = 0n;
                 }
 
                 items.push(toListItem(entry));
@@ -440,7 +463,7 @@ class FileManagerState {
             }), 1000);
 
             try {
-                await (file.stream() as unknown as ReadableStream<Uint8Array>)
+                await createFileStream(file)
                     .pipeThrough(new ChunkStream(ADB_SYNC_MAX_PACKET_SIZE))
                     .pipeThrough(new ProgressStream(action((uploaded) => {
                         this.uploadedSize = uploaded;
@@ -535,7 +558,7 @@ const FileManager: NextPage = (): JSX.Element | null => {
         setPreviewUrl(undefined);
     }, []);
 
-    const handleItemInvoked = useCallback((item: AdbSyncEntryResponse) => {
+    const handleItemInvoked = useCallback((item: AdbSyncEntry) => {
         switch (item.type) {
             case LinuxFileType.Link:
             case LinuxFileType.Directory:
@@ -564,7 +587,7 @@ const FileManager: NextPage = (): JSX.Element | null => {
     }));
 
     const showContextMenu = useCallback((
-        item?: AdbSyncEntryResponse,
+        item?: AdbSyncEntry,
         index?: number,
         e?: Event
     ) => {

apps/demo/src/pages/install.tsx

@@ -5,7 +5,7 @@ import { observer } from "mobx-react-lite";
 import { NextPage } from "next";
 import Head from "next/head";
 import { globalState } from "../state";
-import { pickFile, ProgressStream, RouteStackProps } from "../utils";
+import { createFileStream, pickFile, ProgressStream, RouteStackProps } from "../utils";
 
 enum Stage {
     Uploading,
@@ -56,7 +56,7 @@ class InstallPageState {
             };
         });
 
-        await (file.stream() as unknown as ReadableStream<Uint8Array>)
+        await createFileStream(file)
             .pipeThrough(new ChunkStream(ADB_SYNC_MAX_PACKET_SIZE))
             .pipeThrough(new ProgressStream(action((uploaded) => {
                 if (uploaded !== file.size) {

apps/demo/src/utils/file.ts

@@ -1,5 +1,5 @@
 import getConfig from "next/config";
-import type { WritableStream } from '@yume-chan/adb';
+import { WrapReadableStream, WritableStream, type ReadableStream } from '@yume-chan/adb';
 
 interface PickFileOptions {
     accept?: string;
@@ -47,3 +47,10 @@ export function saveFile(fileName: string, size?: number | undefined) {
         { size }
     ) as unknown as WritableStream<Uint8Array>;
 }
+
+export function createFileStream(file: File) {
+    // `@types/node` typing messed things up
+    // https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/58079
+    // TODO: demo: remove the wrapper after switching to native stream implementation.
+    return new WrapReadableStream<Uint8Array>(file.stream() as unknown as ReadableStream<Uint8Array>);
+}

libraries/adb/src/adb.ts

@@ -78,12 +78,12 @@ export class Adb {
 
         try {
             // https://android.googlesource.com/platform/packages/modules/adb/+/79010dc6d5ca7490c493df800d4421730f5466ca/transport.cpp#1252
-            // There are more feature constants, but some of them are only used by ADB server, not devices.
+            // There are some other feature constants, but some of them are only used by ADB server, not devices.
             const features = [
-                'shell_v2',
+                AdbFeatures.ShellV2,
-                'cmd',
+                AdbFeatures.Cmd,
                 AdbFeatures.StatV2,
-                'ls_v2',
+                AdbFeatures.ListV2,
                 'fixed_push_mkdir',
                 'apex',
                 'abb',

libraries/adb/src/commands/sync/list.ts

@@ -2,7 +2,11 @@ import Struct from '@yume-chan/struct';
 import type { AdbBufferedStream, WritableStreamDefaultWriter } from '../../stream/index.js';
 import { AdbSyncRequestId, adbSyncWriteRequest } from './request.js';
 import { AdbSyncDoneResponse, adbSyncReadResponse, AdbSyncResponseId } from './response.js';
-import { AdbSyncLstatResponse } from './stat.js';
+import { AdbSyncLstatResponse, AdbSyncStatResponse, type AdbSyncStat } from './stat.js';
+
+export interface AdbSyncEntry extends AdbSyncStat {
+    name: string;
+}
 
 export const AdbSyncEntryResponse =
     new Struct({ littleEndian: true })
@@ -13,22 +17,64 @@ export const AdbSyncEntryResponse =
 
 export type AdbSyncEntryResponse = typeof AdbSyncEntryResponse['TDeserializeResult'];
 
-const ResponseTypes = {
+export const AdbSyncEntry2Response =
+    new Struct({ littleEndian: true })
+        .fields(AdbSyncStatResponse)
+        .uint32('nameLength')
+        .string('name', { lengthField: 'nameLength' })
+        .extra({ id: AdbSyncResponseId.Entry2 as const });
+
+export type AdbSyncEntry2Response = typeof AdbSyncEntry2Response['TDeserializeResult'];
+
+const LIST_V1_RESPONSE_TYPES = {
     [AdbSyncResponseId.Entry]: AdbSyncEntryResponse,
     [AdbSyncResponseId.Done]: new AdbSyncDoneResponse(AdbSyncEntryResponse.size),
 };
 
+const LIST_V2_RESPONSE_TYPES = {
+    [AdbSyncResponseId.Entry2]: AdbSyncEntry2Response,
+    [AdbSyncResponseId.Done]: new AdbSyncDoneResponse(AdbSyncEntry2Response.size),
+};
+
 export async function* adbSyncOpenDir(
     stream: AdbBufferedStream,
     writer: WritableStreamDefaultWriter<Uint8Array>,
     path: string,
-): AsyncGenerator<AdbSyncEntryResponse, void, void> {
+    v2: boolean,
-    await adbSyncWriteRequest(writer, AdbSyncRequestId.List, path);
+): AsyncGenerator<AdbSyncEntry, void, void> {
+    let requestId: AdbSyncRequestId.List | AdbSyncRequestId.List2;
+    let responseType: typeof LIST_V1_RESPONSE_TYPES | typeof LIST_V2_RESPONSE_TYPES;
+
+    if (v2) {
+        requestId = AdbSyncRequestId.List2;
+        responseType = LIST_V2_RESPONSE_TYPES;
+    } else {
+        requestId = AdbSyncRequestId.List;
+        responseType = LIST_V1_RESPONSE_TYPES;
+    }
+
+    await adbSyncWriteRequest(writer, requestId, path);
 
     while (true) {
-        const response = await adbSyncReadResponse(stream, ResponseTypes);
+        const response = await adbSyncReadResponse(stream, responseType);
         switch (response.id) {
             case AdbSyncResponseId.Entry:
+                yield {
+                    mode: response.mode,
+                    size: BigInt(response.size),
+                    mtime: BigInt(response.mtime),
+                    get type() { return response.type; },
+                    get permission() { return response.permission; },
+                    name: response.name,
+                };
+                break;
+            case AdbSyncResponseId.Entry2:
+                // `LST2` can return error codes for failed `lstat` calls.
+                // `LIST` just ignores them.
+                // But they only contain `name` so still pretty useless.
+                if (response.error !== 0) {
+                    continue;
+                }
                 yield response;
                 break;
             case AdbSyncResponseId.Done:

libraries/adb/src/commands/sync/push.ts

@@ -25,9 +25,6 @@ export function adbSyncPush(
     return pipeFrom(
         new WritableStream<Uint8Array>({
             async start() {
-                // TODO: AdbSyncPush: support create directory using `mkdir`
-                // if device doesn't support `fixed_push_mkdir` feature.
-
                 const pathAndMode = `${filename},${mode.toString()}`;
                 await adbSyncWriteRequest(writer, AdbSyncRequestId.Send, pathAndMode);
             },

libraries/adb/src/commands/sync/request.ts

@@ -4,6 +4,7 @@ import { encodeUtf8 } from "../../utils/index.js";
 
 export enum AdbSyncRequestId {
     List = 'LIST',
+    List2 = 'LIS2',
     Send = 'SEND',
     Lstat = 'STAT',
     Stat = 'STA2',

libraries/adb/src/commands/sync/response.ts

@@ -4,6 +4,7 @@ import { decodeUtf8 } from "../../utils/index.js";
 
 export enum AdbSyncResponseId {
     Entry = 'DENT',
+    Entry2 = 'DNT2',
     Lstat = 'STAT',
     Stat = 'STA2',
     Lstat2 = 'LST2',
@@ -43,7 +44,12 @@ export const AdbSyncFailResponse =
 export async function adbSyncReadResponse<T extends Record<string, StructLike<any>>>(
     stream: AdbBufferedStream,
     types: T,
-): Promise<StructValueType<T[keyof T]>> {
+    // When `T` is a union type, `T[keyof T]` only includes their common keys.
+    // For example, let `type T = { a: string, b: string } | { a: string, c: string}`,
+    // `keyof T` is `'a'`, not `'a' | 'b' | 'c'`.
+    // However, `T extends unknown ? keyof T : never` will distribute `T`,
+    // so returns all keys.
+): Promise<StructValueType<T extends unknown ? T[keyof T] : never>> {
     const id = decodeUtf8(await stream.read(4));
 
     if (id === AdbSyncResponseId.Fail) {
@@ -54,5 +60,5 @@ export async function adbSyncReadResponse<T extends Record<string, StructLike<an
         return types[id]!.deserialize(stream);
     }
 
-    throw new Error('Unexpected response id');
+    throw new Error(`Expected '${Object.keys(types).join(', ')}', but got '${id}'`);
 }

libraries/adb/src/commands/sync/stat.ts

@@ -10,6 +10,19 @@ export enum LinuxFileType {
     Link = 0o12,
 }
 
+export interface AdbSyncStat {
+    mode: number;
+    size: bigint;
+    mtime: bigint;
+    get type(): LinuxFileType;
+    get permission(): number;
+
+    uid?: number;
+    gid?: number;
+    atime?: bigint;
+    ctime?: bigint;
+}
+
 export const AdbSyncLstatResponse =
     new Struct({ littleEndian: true })
         .int32('mode')
@@ -32,6 +45,7 @@ export const AdbSyncLstatResponse =
 export type AdbSyncLstatResponse = typeof AdbSyncLstatResponse['TDeserializeResult'];
 
 export enum AdbSyncStatErrorCode {
+    SUCCESS = 0,
     EACCES = 13,
     EEXIST = 17,
     EFAULT = 14,
@@ -80,15 +94,15 @@ export const AdbSyncStatResponse =
 
 export type AdbSyncStatResponse = typeof AdbSyncStatResponse['TDeserializeResult'];
 
-const StatResponseType = {
+const STAT_RESPONSE_TYPES = {
     [AdbSyncResponseId.Stat]: AdbSyncStatResponse,
 };
 
-const LstatResponseType = {
+const LSTAT_RESPONSE_TYPES = {
     [AdbSyncResponseId.Lstat]: AdbSyncLstatResponse,
 };
 
-const Lstat2ResponseType = {
+const LSTAT_V2_RESPONSE_TYPES = {
     [AdbSyncResponseId.Lstat2]: AdbSyncStatResponse,
 };
 
@@ -97,20 +111,33 @@ export async function adbSyncLstat(
     writer: WritableStreamDefaultWriter<Uint8Array>,
     path: string,
     v2: boolean,
-): Promise<AdbSyncLstatResponse | AdbSyncStatResponse> {
+): Promise<AdbSyncStat> {
     let requestId: AdbSyncRequestId.Lstat | AdbSyncRequestId.Lstat2;
-    let responseType: typeof LstatResponseType | typeof Lstat2ResponseType;
+    let responseType: typeof LSTAT_RESPONSE_TYPES | typeof LSTAT_V2_RESPONSE_TYPES;
 
     if (v2) {
         requestId = AdbSyncRequestId.Lstat2;
-        responseType = Lstat2ResponseType;
+        responseType = LSTAT_V2_RESPONSE_TYPES;
     } else {
         requestId = AdbSyncRequestId.Lstat;
-        responseType = LstatResponseType;
+        responseType = LSTAT_RESPONSE_TYPES;
     }
 
     await adbSyncWriteRequest(writer, requestId, path);
-    return adbSyncReadResponse(stream, responseType);
+    const response = await adbSyncReadResponse(stream, responseType);
+
+    switch (response.id) {
+        case AdbSyncResponseId.Lstat:
+            return {
+                mode: response.mode,
+                size: BigInt(response.size),
+                mtime: BigInt(response.mtime),
+                get type() { return response.type; },
+                get permission() { return response.permission; },
+            };
+        default:
+            return response;
+    }
 }
 
 export async function adbSyncStat(
@@ -119,5 +146,5 @@ export async function adbSyncStat(
     path: string,
 ): Promise<AdbSyncStatResponse> {
     await adbSyncWriteRequest(writer, AdbSyncRequestId.Stat, path);
-    return await adbSyncReadResponse(stream, StatResponseType);
+    return await adbSyncReadResponse(stream, STAT_RESPONSE_TYPES);
 }

libraries/adb/src/commands/sync/sync.ts

@@ -4,11 +4,28 @@ import { AdbFeatures } from '../../features.js';
 import type { AdbSocket } from '../../socket/index.js';
 import { AdbBufferedStream, ReadableStream, WrapReadableStream, WrapWritableStream, WritableStream, WritableStreamDefaultWriter } from '../../stream/index.js';
 import { AutoResetEvent } from '../../utils/index.js';
-import { AdbSyncEntryResponse, adbSyncOpenDir } from './list.js';
+import { escapeArg } from "../index.js";
+import { adbSyncOpenDir, type AdbSyncEntry } from './list.js';
 import { adbSyncPull } from './pull.js';
 import { adbSyncPush } from './push.js';
 import { adbSyncLstat, adbSyncStat } from './stat.js';
 
+/**
+ * A simplified `dirname` function that only handles absolute unix paths.
+ * @param path an absolute unix path
+ * @returns the directory name of the input path
+ */
+export function dirname(path: string): string {
+    const end = path.lastIndexOf('/');
+    if (end === -1) {
+        throw new Error(`Invalid path`);
+    }
+    if (end === 0) {
+        return '/';
+    }
+    return path.substring(0, end);
+}
+
 export class AdbSync extends AutoDisposable {
     protected adb: Adb;
 
@@ -22,6 +39,19 @@ export class AdbSync extends AutoDisposable {
         return this.adb.features!.includes(AdbFeatures.StatV2);
     }
 
+    public get supportsList2(): boolean {
+        return this.adb.features!.includes(AdbFeatures.ListV2);
+    }
+
+    public get fixedPushMkdir(): boolean {
+        return this.adb.features!.includes(AdbFeatures.FixedPushMkdir);
+    }
+
+    public get needPushMkdirWorkaround(): boolean {
+        // https://android.googlesource.com/platform/packages/modules/adb/+/91768a57b7138166e0a3d11f79cd55909dda7014/client/file_sync_client.cpp#1361
+        return this.adb.features!.includes(AdbFeatures.ShellV2) && !this.fixedPushMkdir;
+    }
+
     public constructor(adb: Adb, socket: AdbSocket) {
         super();
 
@@ -65,18 +95,18 @@ export class AdbSync extends AutoDisposable {
 
     public async *opendir(
         path: string
-    ): AsyncGenerator<AdbSyncEntryResponse, void, void> {
+    ): AsyncGenerator<AdbSyncEntry, void, void> {
         await this.sendLock.wait();
 
         try {
-            yield* adbSyncOpenDir(this.stream, this.writer, path);
+            yield* adbSyncOpenDir(this.stream, this.writer, path, this.supportsList2);
         } finally {
             this.sendLock.notify();
         }
     }
 
     public async readdir(path: string) {
-        const results: AdbSyncEntryResponse[] = [];
+        const results: AdbSyncEntry[] = [];
         for await (const entry of this.opendir(path)) {
             results.push(entry);
         }
@@ -117,6 +147,18 @@ export class AdbSync extends AutoDisposable {
         return new WrapWritableStream({
             start: async () => {
                 await this.sendLock.wait();
+
+                if (this.needPushMkdirWorkaround) {
+                    // It may fail if the path is already existed.
+                    // Ignore the result.
+                    // TODO: test this
+                    await this.adb.subprocess.spawnAndWait([
+                        'mkdir',
+                        '-p',
+                        escapeArg(dirname(filename)),
+                    ]);
+                }
+
                 return adbSyncPush(
                     this.stream,
                     this.writer,

libraries/adb/src/features.ts

@@ -1,5 +1,9 @@
+// The order follows
+// https://android.googlesource.com/platform/packages/modules/adb/+/79010dc6d5ca7490c493df800d4421730f5466ca/transport.cpp#1252
 export enum AdbFeatures {
-    StatV2 = 'stat_v2',
+    ShellV2 = "shell_v2",
     Cmd = 'cmd',
-    ShellV2 = "shell_v2"
+    StatV2 = 'stat_v2',
+    ListV2 = 'ls_v2',
+    FixedPushMkdir = 'fixed_push_mkdir',
 }