yetangitu/ya-webadb
1
0
Fork 0
mirror of https://github.com/yume-chan/ya-webadb.git synced 2025-10-03 17:59:50 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(aoa): emulating HID keyboard via AOA protocol

Browse source
This commit is contained in:
Simon Chan 2023-02-28 00:25:07 +08:00
parent 258547c50b
commit d08a6891e7
No known key found for this signature in database
GPG key ID: A8B69F750B9BCEDD
28 changed files with 2118 additions and 881 deletions
Download patch file Download diff file Expand all files Collapse all files

2
apps/demo/package.json
View file

@ -22,6 +22,7 @@
"@yume-chan/adb-backend-ws": "workspace:^0.0.9", "@yume-chan/adb-backend-ws": "workspace:^0.0.9",
"@yume-chan/adb-credential-web": "workspace:^0.0.18", "@yume-chan/adb-credential-web": "workspace:^0.0.18",
"@yume-chan/android-bin": "workspace:^0.0.18", "@yume-chan/android-bin": "workspace:^0.0.18",
"@yume-chan/aoa": "workspace:^0.0.18",
"@yume-chan/async": "^2.2.0", "@yume-chan/async": "^2.2.0",
"@yume-chan/b-tree": "workspace:^0.0.16", "@yume-chan/b-tree": "workspace:^0.0.16",
"@yume-chan/event": "workspace:^0.0.18", "@yume-chan/event": "workspace:^0.0.18",
@ -46,6 +47,7 @@
"@mdx-js/loader": "^2.2.1", "@mdx-js/loader": "^2.2.1",
"@mdx-js/react": "^2.2.1", "@mdx-js/react": "^2.2.1",
"@next/mdx": "^13.1.1", "@next/mdx": "^13.1.1",
"@types/node": "^18.11.18",
"@types/react": "18.0.27", "@types/react": "18.0.27",
"eslint": "^8.31.0", "eslint": "^8.31.0",
"eslint-config-next": "13.1.5", "eslint-config-next": "13.1.5",

224
apps/demo/src/components/scrcpy/input.ts Normal file
View file

@ -0,0 +1,224 @@
import { AoaHidDevice, HidKeyCode, HidKeyboard } from "@yume-chan/aoa";
import { Disposable } from "@yume-chan/event";
import {
AdbScrcpyClient,
AndroidKeyCode,
AndroidKeyEventAction,
AndroidKeyEventMeta,
} from "@yume-chan/scrcpy";
export interface KeyboardInjector extends Disposable {
down(key: string): Promise<void>;
up(key: string): Promise<void>;
reset(): Promise<void>;
}
export class ScrcpyKeyboardInjector implements KeyboardInjector {
private readonly client: AdbScrcpyClient;
private _controlLeft = false;
private _controlRight = false;
private _shiftLeft = false;
private _shiftRight = false;
private _altLeft = false;
private _altRight = false;
private _metaLeft = false;
private _metaRight = false;
private _capsLock = false;
private _numLock = true;
private _keys: Set<AndroidKeyCode> = new Set();
public constructor(client: AdbScrcpyClient) {
this.client = client;
}
private setModifier(keyCode: AndroidKeyCode, value: boolean) {
switch (keyCode) {
case AndroidKeyCode.ControlLeft:
this._controlLeft = value;
break;
case AndroidKeyCode.ControlRight:
this._controlRight = value;
break;
case AndroidKeyCode.ShiftLeft:
this._shiftLeft = value;
break;
case AndroidKeyCode.ShiftRight:
this._shiftRight = value;
break;
case AndroidKeyCode.AltLeft:
this._altLeft = value;
break;
case AndroidKeyCode.AltRight:
this._altRight = value;
break;
case AndroidKeyCode.MetaLeft:
this._metaLeft = value;
break;
case AndroidKeyCode.MetaRight:
this._metaRight = value;
break;
case AndroidKeyCode.CapsLock:
if (value) {
this._capsLock = !this._capsLock;
}
break;
case AndroidKeyCode.NumLock:
if (value) {
this._numLock = !this._numLock;
}
break;
}
}
private getMetaState(): AndroidKeyEventMeta {
let metaState = 0;
if (this._altLeft) {
metaState |=
AndroidKeyEventMeta.AltOn | AndroidKeyEventMeta.AltLeftOn;
}
if (this._altRight) {
metaState |=
AndroidKeyEventMeta.AltOn | AndroidKeyEventMeta.AltRightOn;
}
if (this._shiftLeft) {
metaState |=
AndroidKeyEventMeta.ShiftOn | AndroidKeyEventMeta.ShiftLeftOn;
}
if (this._shiftRight) {
metaState |=
AndroidKeyEventMeta.ShiftOn | AndroidKeyEventMeta.ShiftRightOn;
}
if (this._controlLeft) {
metaState |=
AndroidKeyEventMeta.CtrlOn | AndroidKeyEventMeta.CtrlLeftOn;
}
if (this._controlRight) {
metaState |=
AndroidKeyEventMeta.CtrlOn | AndroidKeyEventMeta.CtrlRightOn;
}
if (this._metaLeft) {
metaState |=
AndroidKeyEventMeta.MetaOn | AndroidKeyEventMeta.MetaLeftOn;
}
if (this._metaRight) {
metaState |=
AndroidKeyEventMeta.MetaOn | AndroidKeyEventMeta.MetaRightOn;
}
if (this._capsLock) {
metaState |= AndroidKeyEventMeta.CapsLockOn;
}
if (this._numLock) {
metaState |= AndroidKeyEventMeta.NumLockOn;
}
return metaState;
}
public async down(key: string): Promise<void> {
const keyCode = AndroidKeyCode[key as keyof typeof AndroidKeyCode];
if (!keyCode) {
return;
}
this.setModifier(keyCode, true);
this._keys.add(keyCode);
await this.client.controlMessageSerializer?.injectKeyCode({
action: AndroidKeyEventAction.Down,
keyCode,
metaState: this.getMetaState(),
repeat: 0,
});
}
public async up(key: string): Promise<void> {
const keyCode = AndroidKeyCode[key as keyof typeof AndroidKeyCode];
if (!keyCode) {
return;
}
this.setModifier(keyCode, false);
this._keys.delete(keyCode);
await this.client.controlMessageSerializer?.injectKeyCode({
action: AndroidKeyEventAction.Up,
keyCode,
metaState: this.getMetaState(),
repeat: 0,
});
}
public async reset(): Promise<void> {
this._controlLeft = false;
this._controlRight = false;
this._shiftLeft = false;
this._shiftRight = false;
this._altLeft = false;
this._altRight = false;
this._metaLeft = false;
this._metaRight = false;
for (const key of this._keys) {
this.up(AndroidKeyCode[key]);
}
this._keys.clear();
}
public dispose(): void {
// do nothing
}
}
export class AoaKeyboardInjector implements KeyboardInjector {
public static async register(
device: USBDevice
): Promise<AoaKeyboardInjector> {
const keyboard = await AoaHidDevice.register(
device,
0,
HidKeyboard.DESCRIPTOR
);
return new AoaKeyboardInjector(keyboard);
}
private readonly aoaKeyboard: AoaHidDevice;
private readonly hidKeyboard = new HidKeyboard();
public constructor(aoaKeyboard: AoaHidDevice) {
this.aoaKeyboard = aoaKeyboard;
}
public async down(key: string): Promise<void> {
const keyCode = HidKeyCode[key as keyof typeof HidKeyCode];
if (!keyCode) {
return;
}
this.hidKeyboard.down(keyCode);
await this.aoaKeyboard.sendInputReport(
this.hidKeyboard.serializeInputReport()
);
}
public async up(key: string): Promise<void> {
const keyCode = HidKeyCode[key as keyof typeof HidKeyCode];
if (!keyCode) {
return;
}
this.hidKeyboard.up(keyCode);
await this.aoaKeyboard.sendInputReport(
this.hidKeyboard.serializeInputReport()
);
}
public async reset(): Promise<void> {
this.hidKeyboard.reset();
await this.aoaKeyboard.sendInputReport(
this.hidKeyboard.serializeInputReport()
);
}
public async dispose(): Promise<void> {
await this.aoaKeyboard.unregister();
}
}

20
apps/demo/src/components/scrcpy/state.tsx
View file

@ -1,8 +1,8 @@
import { ADB_SYNC_MAX_PACKET_SIZE } from "@yume-chan/adb"; import { ADB_SYNC_MAX_PACKET_SIZE } from "@yume-chan/adb";
import { AdbWebUsbBackend } from "@yume-chan/adb-backend-webusb";
import { import {
AdbScrcpyClient, AdbScrcpyClient,
AdbScrcpyOptions1_22, AdbScrcpyOptions1_22,
AndroidKeyCode,
AndroidScreenPowerMode, AndroidScreenPowerMode,
CodecOptions, CodecOptions,
DEFAULT_SERVER_PATH, DEFAULT_SERVER_PATH,
@ -25,6 +25,11 @@ import { action, autorun, makeAutoObservable, runInAction } from "mobx";
import { GLOBAL_STATE } from "../../state"; import { GLOBAL_STATE } from "../../state";
import { ProgressStream } from "../../utils"; import { ProgressStream } from "../../utils";
import { fetchServer } from "./fetch-server"; import { fetchServer } from "./fetch-server";
import {
AoaKeyboardInjector,
KeyboardInjector,
ScrcpyKeyboardInjector,
} from "./input";
import { MuxerStream, RECORD_STATE } from "./recorder"; import { MuxerStream, RECORD_STATE } from "./recorder";
import { H264Decoder, SETTING_STATE } from "./settings"; import { H264Decoder, SETTING_STATE } from "./settings";
@ -58,7 +63,7 @@ export class ScrcpyPageState {
client: AdbScrcpyClient | undefined = undefined; client: AdbScrcpyClient | undefined = undefined;
hoverHelper: ScrcpyHoverHelper | undefined = undefined; hoverHelper: ScrcpyHoverHelper | undefined = undefined;
pressedKeys: Set<AndroidKeyCode> = new Set(); keyboard: KeyboardInjector | undefined = undefined;
async pushServer() { async pushServer() {
const serverBuffer = await fetchServer(); const serverBuffer = await fetchServer();
@ -351,6 +356,14 @@ export class ScrcpyPageState {
this.hoverHelper = new ScrcpyHoverHelper(); this.hoverHelper = new ScrcpyHoverHelper();
this.running = true; this.running = true;
}); });
if (GLOBAL_STATE.backend instanceof AdbWebUsbBackend) {
this.keyboard = await AoaKeyboardInjector.register(
GLOBAL_STATE.backend.device
);
} else {
this.keyboard = new ScrcpyKeyboardInjector(client);
}
} catch (e: any) { } catch (e: any) {
GLOBAL_STATE.showErrorDialog(e); GLOBAL_STATE.showErrorDialog(e);
} finally { } finally {
@ -376,7 +389,8 @@ export class ScrcpyPageState {
RECORD_STATE.recording = false; RECORD_STATE.recording = false;
} }
this.pressedKeys.clear(); this.keyboard?.dispose();
this.keyboard = undefined;
this.fps = "0"; this.fps = "0";
clearTimeout(this.fpsCounterIntervalId); clearTimeout(this.fpsCounterIntervalId);

1
apps/demo/src/components/scrcpy/video-container.tsx
View file

@ -111,7 +111,6 @@ function handlePointerLeave(e: PointerEvent<HTMLDivElement>) {
e.preventDefault(); e.preventDefault();
e.stopPropagation(); e.stopPropagation();
// Because pointer capture on pointer down, this event only happens for hovering mouse and pen. // Because pointer capture on pointer down, this event only happens for hovering mouse and pen.
// Release the injected pointer, otherwise it will stuck at the last position. // Release the injected pointer, otherwise it will stuck at the last position.
injectTouch(AndroidMotionEventAction.HoverExit, e); injectTouch(AndroidMotionEventAction.HoverExit, e);

79
apps/demo/src/pages/audio.tsx Normal file
View file

@ -0,0 +1,79 @@
import { DefaultButton, PrimaryButton } from "@fluentui/react";
import { AdbWebUsbBackend } from "@yume-chan/adb-backend-webusb";
import {
aoaGetProtocol,
aoaSetAudioMode,
aoaStartAccessory,
} from "@yume-chan/aoa";
import { observer } from "mobx-react-lite";
import { useCallback, useState } from "react";
import { GLOBAL_STATE } from "../state";
function AudioPage() {
const [supported, setSupported] = useState<boolean | undefined>(undefined);
const handleQuerySupportClick = useCallback(async () => {
const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
const device = backend.device;
const version = await aoaGetProtocol(device);
setSupported(version >= 2);
}, []);
const handleEnableClick = useCallback(async () => {
const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
const device = backend.device;
const version = await aoaGetProtocol(device);
if (version < 2) {
return;
}
await aoaSetAudioMode(device, 1);
await aoaStartAccessory(device);
}, []);
const handleDisableClick = useCallback(async () => {
const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
const device = backend.device;
const version = await aoaGetProtocol(device);
if (version < 2) {
return;
}
await aoaSetAudioMode(device, 0);
await aoaStartAccessory(device);
}, []);
if (
!GLOBAL_STATE.backend ||
!(GLOBAL_STATE.backend instanceof AdbWebUsbBackend)
) {
return <div>Audio forward can only be used with WebUSB backend.</div>;
}
return (
<div>
<div>
Supported:{" "}
{supported === undefined ? "Unknown" : supported ? "Yes" : "No"}
</div>
<div>
<PrimaryButton
disabled={!GLOBAL_STATE.backend}
onClick={handleQuerySupportClick}
>
Query Support
</PrimaryButton>
<DefaultButton
disabled={!supported}
onClick={handleEnableClick}
>
Enable
</DefaultButton>
<DefaultButton
disabled={!supported}
onClick={handleDisableClick}
>
Disable
</DefaultButton>
</div>
</div>
);
}
export default observer(AudioPage);

42
apps/demo/src/pages/scrcpy.tsx
View file

@ -1,11 +1,6 @@
import { Dialog, LayerHost, ProgressIndicator, Stack } from "@fluentui/react"; import { Dialog, LayerHost, ProgressIndicator, Stack } from "@fluentui/react";
import { useId } from "@fluentui/react-hooks"; import { useId } from "@fluentui/react-hooks";
import { makeStyles, shorthands } from "@griffel/react"; import { makeStyles, shorthands } from "@griffel/react";
import {
AndroidKeyCode,
AndroidKeyEventAction,
AndroidKeyEventMeta,
} from "@yume-chan/scrcpy";
import { WebCodecsDecoder } from "@yume-chan/scrcpy-decoder-webcodecs"; import { WebCodecsDecoder } from "@yume-chan/scrcpy-decoder-webcodecs";
import { action, runInAction } from "mobx"; import { action, runInAction } from "mobx";
import { observer } from "mobx-react-lite"; import { observer } from "mobx-react-lite";
@ -132,29 +127,7 @@ async function handleKeyEvent(e: KeyboardEvent<HTMLDivElement>) {
e.stopPropagation(); e.stopPropagation();
const { type, code } = e; const { type, code } = e;
const keyCode = AndroidKeyCode[code as keyof typeof AndroidKeyCode]; STATE.keyboard![type === "keydown" ? "down" : "up"](code);
if (keyCode) {
if (type === "keydown") {
STATE.pressedKeys.add(keyCode);
} else {
STATE.pressedKeys.delete(keyCode);
}
// TODO: workaround the missing keyup event on macOS https://crbug.com/1393524
STATE.client!.controlMessageSerializer!.injectKeyCode({
action:
type === "keydown"
? AndroidKeyEventAction.Down
: AndroidKeyEventAction.Up,
keyCode,
metaState:
(e.ctrlKey ? AndroidKeyEventMeta.CtrlOn : 0) |
(e.shiftKey ? AndroidKeyEventMeta.ShiftOn : 0) |
(e.altKey ? AndroidKeyEventMeta.AltOn : 0) |
(e.metaKey ? AndroidKeyEventMeta.MetaOn : 0),
repeat: 0,
});
}
} }
function handleBlur() { function handleBlur() {
@ -162,18 +135,7 @@ function handleBlur() {
return; return;
} }
// Release all pressed keys on window blur, STATE.keyboard?.reset();
// Because there will not be any keyup events when window is not focused.
for (const key of STATE.pressedKeys) {
STATE.client.controlMessageSerializer!.injectKeyCode({
action: AndroidKeyEventAction.Up,
keyCode: key,
metaState: 0,
repeat: 0,
});
}
STATE.pressedKeys.clear();
} }
const Scrcpy: NextPage = () => { const Scrcpy: NextPage = () => {

3
apps/demo/src/utils/icons.tsx
View file

@ -34,6 +34,7 @@ import {
PersonFeedbackRegular, PersonFeedbackRegular,
PhoneLaptopRegular, PhoneLaptopRegular,
PhoneRegular, PhoneRegular,
PhoneSpeakerRegular,
PlayRegular, PlayRegular,
PlugConnectedRegular, PlugConnectedRegular,
PlugDisconnectedRegular, PlugDisconnectedRegular,
@ -86,6 +87,7 @@ export function register() {
PersonFeedback: <PersonFeedbackRegular style={STYLE} />, PersonFeedback: <PersonFeedbackRegular style={STYLE} />,
Phone: <PhoneRegular style={STYLE} />, Phone: <PhoneRegular style={STYLE} />,
PhoneLaptop: <PhoneLaptopRegular style={STYLE} />, PhoneLaptop: <PhoneLaptopRegular style={STYLE} />,
PhoneSpeaker: <PhoneSpeakerRegular style={STYLE} />,
Play: <PlayRegular style={STYLE} />, Play: <PlayRegular style={STYLE} />,
PlugConnected: <PlugConnectedRegular style={STYLE} />, PlugConnected: <PlugConnectedRegular style={STYLE} />,
PlugDisconnected: <PlugDisconnectedRegular style={STYLE} />, PlugDisconnected: <PlugDisconnectedRegular style={STYLE} />,
@ -155,6 +157,7 @@ const Icons = {
PersonFeedback: "PersonFeedback", PersonFeedback: "PersonFeedback",
Phone: "Phone", Phone: "Phone",
PhoneLaptop: "PhoneLaptop", PhoneLaptop: "PhoneLaptop",
PhoneSpeaker: "PhoneSpeaker",
Play: "Play", Play: "Play",
PlugConnected: "PlugConnected", PlugConnected: "PlugConnected",
PlugDisconnected: "PlugDisconnected", PlugDisconnected: "PlugDisconnected",

1685
common/config/rush/pnpm-lock.yaml generated
View file

File diff suppressed because it is too large Load diff

2
common/config/rush/repo-state.json
View file

@ -1,5 +1,5 @@
// DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
{ {
"pnpmShrinkwrapHash": "3e5a716ba1e8bfb6c9b5bd7018741744d14f80d0", "pnpmShrinkwrapHash": "8b04c1bdf0fa11f10bb4abc856fd9d1c0903908e",
"preferredVersionsHash": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f" "preferredVersionsHash": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f"
} }

11
libraries/aoa/.eslintrc.cjs Normal file
View file

@ -0,0 +1,11 @@
module.exports = {
"extends": [
"@yume-chan"
],
parserOptions: {
tsconfigRootDir: __dirname,
project: [
"./tsconfig.test.json"
],
},
}

16
libraries/aoa/.npmignore Normal file
View file

@ -0,0 +1,16 @@
.rush
# Test
coverage
**/*.spec.ts
**/*.spec.js
**/*.spec.js.map
**/__helpers__
jest.config.js
.eslintrc.cjs
tsconfig.json
tsconfig.test.json
# Logs
*.log

21
libraries/aoa/LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020-2023 Simon Chan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

3
libraries/aoa/README.md Normal file
View file

@ -0,0 +1,3 @@
# @yume-chan/aoa
A TypeScript implementation of [Android Open Accessory](https://source.android.com/docs/core/interaction/accessories/protocol) protocol using WebUSB API.

42
libraries/aoa/package.json Normal file
View file

@ -0,0 +1,42 @@
{
"name": "@yume-chan/aoa",
"version": "0.0.18",
"description": "TypeScript implementation of Android Open Accessory protocol.",
"keywords": [
"adb",
"android-phone"
],
"license": "MIT",
"author": {
"name": "Simon Chan",
"email": "cnsimonchan@live.com",
"url": "https://chensi.moe/blog"
},
"homepage": "https://github.com/yume-chan/ya-webadb/tree/main/packages/aoa#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/yume-chan/ya-webadb.git",
"directory": "packages/aoa"
},
"bugs": {
"url": "https://github.com/yume-chan/ya-webadb/issues"
},
"type": "module",
"main": "esm/index.js",
"types": "esm/index.d.ts",
"scripts": {
"build": "tsc -b tsconfig.build.json",
"build:watch": "tsc -b tsconfig.build.json",
"lint": "eslint src/**/*.ts --fix",
"prepublishOnly": "npm run build"
},
"dependencies": {
"@types/w3c-web-usb": "^1.0.6"
},
"devDependencies": {
"@yume-chan/eslint-config": "workspace:^1.0.0",
"@yume-chan/tsconfig": "workspace:^1.0.0",
"eslint": "^8.31.0",
"typescript": "^4.9.4"
}
}

125
libraries/aoa/src/audio.ts Normal file
View file

@ -0,0 +1,125 @@
import { AoaRequestType } from "./type.js";
// The original plan is to add more audio modes,
// but AOA audio accessory mode is soon deprecated in Android 8.
export enum AoaAudioMode {
Off,
/**
* 2 channel, 16 bit, 44.1KHz PCM
*/
On,
}
/**
* Sets the audio mode. This method must be called before {@link aoaStartAccessory}.
*
* AOA audio accessory mode turns the Android device into a USB microphone,
* all system audio will be directed to the microphone, to be capture by the USB host.
*
* It's like connecting a audio cable between the Android headphone jack and PC microphone jack,
* except all signals are digital.
*
* Audio mode is deprecated in Android 8. On Android 9 and later, this call still switches the device
* to audio accessory mode, and the device will be recognized as a USB microphone, but the
* required USB endpoint is not presented anymore.
* @param device The Android device.
* @param mode The audio mode.
*/
export async function aoaSetAudioMode(device: USBDevice, mode: AoaAudioMode) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.SetAudioMode,
value: mode,
index: 0,
},
new ArrayBuffer(0)
);
}
function findAudioStreamingInterface(device: USBDevice) {
for (const configuration of device.configurations) {
for (const interface_ of configuration.interfaces) {
for (const alternate of interface_.alternates) {
// Audio
if (alternate.interfaceClass !== 0x01) {
continue;
}
// AudioStreaming
if (alternate.interfaceSubclass !== 0x02) {
continue;
}
if (alternate.endpoints.length === 0) {
continue;
}
return { configuration, interface_, alternate };
}
}
}
throw new Error("No matched alternate interface found");
}
/**
* It doesn't work on Web, because Chrome blocked audio devices from WebUSB API.
* @param device The Android device.
* @returns A readable stream of raw audio data.
*/
export function aoaGetAudioStream(device: USBDevice) {
let endpointNumber!: number;
return new ReadableStream<Uint8Array>({
async start() {
const { configuration, interface_, alternate } =
findAudioStreamingInterface(device);
if (
device.configuration?.configurationValue !==
configuration.configurationValue
) {
await device.selectConfiguration(
configuration.configurationValue
);
}
if (!interface_.claimed) {
await device.claimInterface(interface_.interfaceNumber);
}
if (
interface_.alternate.alternateSetting !==
alternate.alternateSetting
) {
await device.selectAlternateInterface(
interface_.interfaceNumber,
alternate.alternateSetting
);
}
const endpoint = alternate.endpoints.find(
(endpoint) =>
endpoint.type === "isochronous" &&
endpoint.direction === "in"
);
if (!endpoint) {
throw new Error("No matched endpoint found");
}
endpointNumber = endpoint.endpointNumber;
},
async pull(controller) {
const result = await device.isochronousTransferIn(endpointNumber, [
1024,
]);
for (const packet of result.packets) {
const data = packet.data!;
const array = new Uint8Array(
data.buffer,
data.byteOffset,
data.byteLength
);
controller.enqueue(array);
}
},
});
}

12
libraries/aoa/src/filter.ts Normal file
View file

@ -0,0 +1,12 @@
export const AOA_DEFAULT_DEVICE_FILTERS = [
{
vendorId: 0x18d1,
// accessory
productId: 0x2d00,
},
{
vendorId: 0x18d1,
// accessory + adb
productId: 0x2d01,
},
] as const satisfies readonly USBDeviceFilter[];

105
libraries/aoa/src/hid.ts Normal file
View file

@ -0,0 +1,105 @@
import { AoaRequestType } from "./type.js";
export async function aoaHidRegister(
device: USBDevice,
accessoryId: number,
reportDescriptorSize: number
) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.RegisterHid,
value: accessoryId,
index: reportDescriptorSize,
},
new ArrayBuffer(0)
);
}
export async function aoaHidSetReportDescriptor(
device: USBDevice,
accessoryId: number,
reportDescriptor: Uint8Array
) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.SetHidReportDescriptor,
value: accessoryId,
index: 0,
},
reportDescriptor
);
}
export async function aoaHidUnregister(device: USBDevice, accessoryId: number) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.UnregisterHid,
value: accessoryId,
index: 0,
},
new ArrayBuffer(0)
);
}
export async function aoaHidSendInputReport(
device: USBDevice,
accessoryId: number,
event: Uint8Array
) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.SendHidEvent,
value: accessoryId,
index: 0,
},
event
);
}
/**
* Emulate a HID device over AOA protocol.
*
* It can only send input reports, but not send feature reports nor receive output reports.
*/
export class AoaHidDevice {
/**
* Register a HID device.
* @param device The Android device.
* @param accessoryId An arbitrary number to uniquely identify the HID device.
* @param reportDescriptor The HID report descriptor.
* @returns An instance of AoaHidDevice to send events.
*/
public static async register(
device: USBDevice,
accessoryId: number,
reportDescriptor: Uint8Array
) {
await aoaHidRegister(device, accessoryId, reportDescriptor.length);
await aoaHidSetReportDescriptor(device, accessoryId, reportDescriptor);
return new AoaHidDevice(device, accessoryId);
}
private _device: USBDevice;
private _accessoryId: number;
private constructor(device: USBDevice, accessoryId: number) {
this._device = device;
this._accessoryId = accessoryId;
}
public async sendInputReport(event: Uint8Array) {
await aoaHidSendInputReport(this._device, this._accessoryId, event);
}
public async unregister() {
await aoaHidUnregister(this._device, this._accessoryId);
}
}

7
libraries/aoa/src/index.ts Normal file
View file

@ -0,0 +1,7 @@
export * from "./audio.js";
export * from "./filter.js";
export * from "./hid.js";
export * from "./initialize.js";
export * from "./keyboard.js";
export * from "./mouse.js";
export * from "./type.js";

33
libraries/aoa/src/initialize.ts Normal file
View file

@ -0,0 +1,33 @@
import { AoaRequestType } from "./type.js";
export async function aoaGetProtocol(device: USBDevice) {
const result = await device.controlTransferIn(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.GetProtocol,
value: 0,
index: 0,
},
2
);
const version = result.data!.getUint16(0, true);
return version;
}
/**
* The device will reset (disconnect) after this call.
* @param device The Android device.
*/
export async function aoaStartAccessory(device: USBDevice) {
await device.controlTransferOut(
{
recipient: "device",
requestType: "vendor",
request: AoaRequestType.Start,
value: 0,
index: 0,
},
new ArrayBuffer(0)
);
}

311
libraries/aoa/src/keyboard.ts Normal file
View file

@ -0,0 +1,311 @@
// cspell:ignore Oper
// Most names follow Web API `KeyboardEvent.code`,
export enum HidKeyCode {
KeyA = 4,
KeyB,
KeyC,
KeyD,
KeyE,
KeyF,
KeyG,
KeyH,
KeyI,
KeyJ,
KeyK,
KeyL,
KeyM,
KeyN,
KeyO,
KeyP,
KeyQ,
KeyR,
KeyS,
KeyT,
KeyU,
KeyV,
KeyW,
KeyX,
KeyY,
KeyZ,
Digit1,
Digit2,
Digit3,
Digit4,
Digit5,
Digit6,
Digit7,
Digit8,
Digit9,
Digit0,
Enter,
Escape,
Backspace,
Tab,
Space,
Minus,
Equal,
BracketLeft,
BracketRight,
Backslash,
NonUsHash,
Semicolon,
Quote,
Backquote,
Comma,
Period,
Slash,
CapsLock,
F1,
F2,
F3,
F4,
F5,
F6,
F7,
F8,
F9,
F10,
F11,
F12,
PrintScreen,
ScrollLock,
Pause,
Insert,
Home,
PageUp,
Delete,
End,
PageDown,
ArrowRight,
ArrowLeft,
ArrowDown,
ArrowUp,
NumLock,
NumpadDivide,
NumpadMultiply,
NumpadSubtract,
NumpadAdd,
NumpadEnter,
Numpad1,
Numpad2,
Numpad3,
Numpad4,
Numpad5,
Numpad6,
Numpad7,
Numpad8,
Numpad9,
Numpad0,
NumpadDecimal,
NonUsBackslash,
ContextMenu,
Power,
NumpadEqual,
F13,
F14,
F15,
F16,
F17,
F18,
F19,
F20,
F21,
F22,
F23,
F24,
Execute,
Help,
Menu,
Select,
Stop,
Again,
Undo,
Cut,
Copy,
Paste,
Find,
Mute,
VolumeUp,
VolumeDown,
LockingCapsLock,
LockingNumLock,
LockingScrollLock,
NumpadComma,
KeypadEqualSign,
International1,
International2,
International3,
International4,
International5,
International6,
International7,
International8,
International9,
Lang1,
Lang2,
Lang3,
Lang4,
Lang5,
Lang6,
Lang7,
Lang8,
Lang9,
AlternateErase,
SysReq,
Cancel,
Clear,
Prior,
Return2,
Separator,
Out,
Oper,
ClearAgain,
CrSel,
ExSel,
Keypad00 = 0xb0,
Keypad000,
ThousandsSeparator,
DecimalSeparator,
CurrencyUnit,
CurrencySubUnit,
KeypadLeftParen,
KeypadRightParen,
KeypadLeftBrace,
KeypadRightBrace,
KeypadTab,
KeypadBackspace,
KeypadA,
KeypadB,
KeypadC,
KeypadD,
KeypadE,
KeypadF,
KeypadXor,
KeypadPower,
KeypadPercent,
KeypadLess,
KeypadGreater,
KeypadAmpersand,
KeypadDblAmpersand,
KeypadVerticalBar,
KeypadDblVerticalBar,
KeypadColon,
KeypadHash,
KeypadSpace,
KeypadAt,
KeypadExclamation,
KeypadMemStore,
KeypadMemRecall,
KeypadMemClear,
KeypadMemAdd,
KeypadMemSubtract,
KeypadMemMultiply,
KeypadMemDivide,
KeypadPlusMinus,
KeypadClear,
KeypadClearEntry,
KeypadBinary,
KeypadOctal,
KeypadDecimal,
KeypadHexadecimal,
ControlLeft = 0xe0,
ShiftLeft,
AltLeft,
MetaLeft,
ControlRight,
ShiftRight,
AltRight,
MetaRight,
}
export class HidKeyboard {
/**
* A HID Keyboard Report Descriptor.
*
* It's compatible with the legacy boot protocol. (1 byte modifier, 1 byte reserved, 6 bytes key codes).
* Technically it doesn't need to be compatible with the legacy boot protocol, but it's the most common implementation.
*/
public static readonly DESCRIPTOR = new Uint8Array(
// prettier-ignore
[
0x05, 0x01, // Usage Page (Generic Desktop)
0x09, 0x06, // Usage (Keyboard)
0xa1, 0x01, // Collection (Application)
0x05, 0x07, // Usage Page (Keyboard)
0x19, 0xe0, // Usage Minimum (Keyboard Left Control)
0x29, 0xe7, // Usage Maximum (Keyboard Right GUI)
0x15, 0x00, // Logical Minimum (0)
0x25, 0x01, // Logical Maximum (1)
0x75, 0x01, // Report Size (1)
0x95, 0x08, // Report Count (8)
0x81, 0x02, // Input (Data, Variable, Absolute)
0x75, 0x08, // Report Size (8)
0x95, 0x01, // Report Count (1)
0x81, 0x01, // Input (Constant)
0x05, 0x08, // Usage Page (LEDs)
0x19, 0x01, // Usage Minimum (Num Lock)
0x29, 0x05, // Usage Maximum (Kana)
0x75, 0x01, // Report Size (1)
0x95, 0x05, // Report Count (5)
0x91, 0x02, // Output (Data, Variable, Absolute)
0x75, 0x03, // Report Size (3)
0x95, 0x01, // Report Count (1)
0x91, 0x01, // Output (Constant)
0x05, 0x07, // Usage Page (Keyboard)
0x19, 0x00, // Usage Minimum (Reserved (no event indicated))
0x29, 0xdd, // Usage Maximum (Keyboard Application)
0x15, 0x00, // Logical Minimum (0)
0x25, 0xdd, // Logical Maximum (221)
0x75, 0x08, // Report Size (8)
0x95, 0x06, // Report Count (6)
0x81, 0x00, // Input (Data, Array)
0xc0 // End Collection
]
);
private _modifiers = 0;
private _keys: Set<HidKeyCode> = new Set();
public down(key: HidKeyCode) {
if (key >= HidKeyCode.ControlLeft && key <= HidKeyCode.MetaRight) {
this._modifiers |= 1 << (key - HidKeyCode.ControlLeft);
} else {
this._keys.add(key);
}
}
public up(key: HidKeyCode) {
if (key >= HidKeyCode.ControlLeft && key <= HidKeyCode.MetaRight) {
this._modifiers &= ~(1 << (key - HidKeyCode.ControlLeft));
} else {
this._keys.delete(key);
}
}
public reset() {
this._modifiers = 0;
this._keys.clear();
}
public serializeInputReport() {
const buffer = new Uint8Array(8);
buffer[0] = this._modifiers;
let i = 2;
for (const key of this._keys) {
buffer[i] = key;
i += 1;
if (i >= 8) {
break;
}
}
return buffer;
}
}

60
libraries/aoa/src/mouse.ts Normal file
View file

@ -0,0 +1,60 @@
export class HidMouse {
public static readonly descriptor = new Uint8Array(
// prettier-ignore
[
0x05, 0x01, // Usage Page (Generic Desktop)
0x09, 0x02, // Usage (Mouse)
0xa1, 0x01, // Collection (Application)
0x09, 0x01, // Usage (Pointer)
0xa1, 0x00, // Collection (Physical)
0x05, 0x09, // Usage Page (Button)
0x19, 0x01, // Usage Minimum (Button 1)
0x29, 0x05, // Usage Maximum (Button 5)
0x15, 0x00, // Logical Minimum (0)
0x25, 0x01, // Logical Maximum (1)
0x75, 0x01, // Report Size (1)
0x95, 0x05, // Report Count (5)
0x81, 0x02, // Input (Data, Variable, Absolute)
0x75, 0x03, // Report Size (3)
0x95, 0x01, // Report Count (1)
0x81, 0x01, // Input (Constant)
0x05, 0x01, // Usage Page (Generic Desktop)
0x09, 0x30, // Usage (X)
0x09, 0x31, // Usage (Y)
0x09, 0x38, // Usage (Wheel)
0x15, 0x81, // Logical Minimum (-127)
0x25, 0x7f, // Logical Maximum (127)
0x75, 0x08, // Report Size (8)
0x95, 0x03, // Report Count (3)
0x81, 0x06, // Input (Data, Variable, Relative)
0x05, 0x0C, // Usage Page (Consumer)
0x0A, 0x38, 0x02, // Usage (AC Pan)
0x15, 0x81, // Logical Minimum (-127)
0x25, 0x7f, // Logical Maximum (127)
0x75, 0x08, // Report Size (8)
0x95, 0x01, // Report Count (1)
0x81, 0x06, // Input (Data, Variable, Relative)
0xc0, // End Collection
0xc0, // End Collection
]
);
public static serializeInputReport(
movementX: number,
movementY: number,
buttons: number,
scrollX: number,
scrollY: number
): Uint8Array {
return new Uint8Array([
buttons,
movementX,
movementY,
scrollY,
scrollX,
]);
}
}

132
libraries/aoa/src/touchscreen.ts Normal file
View file

@ -0,0 +1,132 @@
const FINGER_DESCRIPTOR = new Uint8Array(
// prettier-ignore
[
0x09, 0x22, // Usage (Finger)
0xa1, 0x02, // Collection (Logical)
0x09, 0x51, // Usage (Contact Identifier)
0x15, 0x00, // Logical Minimum (0)
0x25, 0x7f, // Logical Maximum (127)
0x75, 0x08, // Report Size (8)
0x95, 0x01, // Report Count (1)
0x81, 0x02, // Input (Data, Variable, Absolute)
0x09, 0x42, // Usage (Tip Switch)
0x15, 0x00, // Logical Minimum (0)
0x25, 0x01, // Logical Maximum (1)
0x75, 0x01, // Report Size (1)
0x95, 0x01, // Report Count (1)
0x81, 0x02, // Input (Data, Variable, Absolute)
0x95, 0x07, // Report Count (7)
0x81, 0x03, // Input (Constant)
0x05, 0x01, // Usage Page (Generic Desktop)
0x09, 0x30, // Usage (X)
0x09, 0x31, // Usage (Y)
0x15, 0x00, // Logical Minimum (0)
0x26, 0xff, 0x7f, // Logical Maximum (32767)
0x35, 0x00, // Physical Minimum (0)
0x46, 0xff, 0x7f, // Physical Maximum (32767)
0x66, 0x00, 0x00, // Unit (None)
0x75, 0x10, // Report Size (16)
0x95, 0x02, // Report Count (2)
0x81, 0x02, // Input (Data, Variable, Absolute)
0xc0, // End Collection
]
);
const DESCRIPTOR_HEAD = new Uint8Array(
// prettier-ignore
[
0x05, 0x0d, // Usage Page (Digitizers)
0x09, 0x04, // Usage (Touch Screen)
0xa1, 0x01, // Collection (Application)
0x09, 0x55, // Usage (Contact Count Maximum)
0x25, 0x0a, // Logical Maximum (10)
0xB1, 0x02, // Feature (Data, Variable, Absolute)
0x09, 0x54, // Usage (Contact Count)
0x15, 0x00, // Logical Minimum (0)
0x25, 0x0a, // Logical Maximum (10)
0x75, 0x08, // Report Size (8)
0x95, 0x01, // Report Count (1)
0x81, 0x02, // Input (Data, Variable, Absolute)
]
);
const DESCRIPTOR_TAIL = new Uint8Array([
0xc0, // End Collection
]);
const DESCRIPTOR = new Uint8Array(
DESCRIPTOR_HEAD.length +
FINGER_DESCRIPTOR.length * 10 +
DESCRIPTOR_TAIL.length
);
let offset = 0;
DESCRIPTOR.set(DESCRIPTOR_HEAD, offset);
offset += DESCRIPTOR_HEAD.length;
for (let i = 0; i < 10; i += 1) {
DESCRIPTOR.set(FINGER_DESCRIPTOR, offset);
offset += FINGER_DESCRIPTOR.length;
}
DESCRIPTOR.set(DESCRIPTOR_TAIL, offset);
interface Finger {
x: number;
y: number;
}
/**
* A ten-point touch screen.
*/
export class HidTouchScreen {
public static readonly FINGER_DESCRIPTOR = FINGER_DESCRIPTOR;
public static readonly DESCRIPTOR = DESCRIPTOR;
private fingers: Map<number, Finger> = new Map();
public down(id: number, x: number, y: number) {
if (this.fingers.size >= 10) {
return;
}
this.fingers.set(id, {
x,
y,
});
}
public move(id: number, x: number, y: number) {
const finger = this.fingers.get(id);
if (finger) {
finger.x = x;
finger.y = y;
}
}
public up(id: number) {
this.fingers.delete(id);
}
public serializeInputReport(): Uint8Array {
const report = new Uint8Array(1 + 6 * 10);
report[0] = this.fingers.size;
let offset = 1;
for (const [id, finger] of this.fingers) {
report[offset] = id;
offset += 1;
report[offset] = 1;
offset += 1;
report[offset] = finger.x & 0xff;
report[offset + 1] = (finger.x >> 8) & 0xff;
report[offset + 2] = finger.y & 0xff;
report[offset + 3] = (finger.y >> 8) & 0xff;
offset += 4;
}
return report;
}
}

10
libraries/aoa/src/type.ts Normal file
View file

@ -0,0 +1,10 @@
export enum AoaRequestType {
GetProtocol = 51,
SendString,
Start,
RegisterHid,
UnregisterHid,
SetHidReportDescriptor,
SendHidEvent,
SetAudioMode,
}

12
libraries/aoa/tsconfig.build.json Normal file
View file

@ -0,0 +1,12 @@
{
"extends": "./node_modules/@yume-chan/tsconfig/tsconfig.base.json",
"compilerOptions": {
"lib": [
"ESNext",
"DOM"
],
"types": [
"w3c-web-usb",
]
}
}

10
libraries/aoa/tsconfig.json Normal file
View file

@ -0,0 +1,10 @@
{
"references": [
{
"path": "./tsconfig.test.json"
},
{
"path": "./tsconfig.build.json"
},
]
}

9
libraries/aoa/tsconfig.test.json Normal file
View file

@ -0,0 +1,9 @@
{
"extends": "./tsconfig.build.json",
"compilerOptions": {
"types": [
"w3c-web-usb",
],
},
"exclude": []
}

18
libraries/scrcpy/src/control/inject-keycode.ts
View file

@ -7,15 +7,27 @@ export enum AndroidKeyEventAction {
Up = 1, Up = 1,
} }
// https://cs.android.com/android/platform/superproject/+/master:frameworks/base/core/java/android/view/KeyEvent.java;l=993;drc=95c1165bb895dd844e1793460710f7163dd330a3
export enum AndroidKeyEventMeta { export enum AndroidKeyEventMeta {
AltOn = 0x02, AltOn = 0x02,
CtrlOn = 0x1000, AltLeftOn = 0x10,
AltRightOn = 0x20,
ShiftOn = 0x01, ShiftOn = 0x01,
ShiftLeftOn = 0x40,
ShiftRightOn = 0x80,
CtrlOn = 0x1000,
CtrlLeftOn = 0x2000,
CtrlRightOn = 0x4000,
MetaOn = 0x10000, MetaOn = 0x10000,
MetaLeftOn = 0x20000,
MetaRightOn = 0x40000,
CapsLockOn = 0x100000,
NumLockOn = 0x200000,
ScrollLockOn = 0x400000,
} }
// https://cs.android.com/android/platform/superproject/+/master:frameworks/base/core/java/android/view/KeyEvent.java;l=97;drc=95c1165bb895dd844e1793460710f7163dd330a3 // https://cs.android.com/android/platform/superproject/+/master:frameworks/base/core/java/android/view/KeyEvent.java;l=97;drc=95c1165bb895dd844e1793460710f7163dd330a3
// Most name follows Web API `KeyboardEvent.code`, // Most names follow Web API `KeyboardEvent.code`,
// Android-only (not exist in HID keyboard standard) keys are prefixed by `Android`. // Android-only (not exist in HID keyboard standard) keys are prefixed by `Android`.
export enum AndroidKeyCode { export enum AndroidKeyCode {
AndroidHome = 3, AndroidHome = 3,
@ -127,7 +139,7 @@ export enum AndroidKeyCode {
AndroidFocus, AndroidFocus,
Plus, // Name not verified Plus, // Name not verified
Menu, // Name not verified ContextMenu,
AndroidNotification, AndroidNotification,
AndroidSearch, AndroidSearch,

4
rush.json
View file

@ -383,6 +383,10 @@
"shouldPublish": true, "shouldPublish": true,
"versionPolicyName": "adb" "versionPolicyName": "adb"
}, },
{
"packageName": "@yume-chan/aoa",
"projectFolder": "libraries/aoa"
},
{ {
"packageName": "@yume-chan/b-tree", "packageName": "@yume-chan/b-tree",
"projectFolder": "libraries/b-tree" "projectFolder": "libraries/b-tree"