yetangitu/ya-webadb
1
0
Fork 0
mirror of https://github.com/yume-chan/ya-webadb.git synced 2025-10-03 09:49:24 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(webcodecs): split codec decoders

Browse source
This commit is contained in:
Simon Chan 2024-06-14 15:26:38 +08:00
parent dad1308cc4
commit 5620716a4f
No known key found for this signature in database
GPG key ID: A8B69F750B9BCEDD
14 changed files with 323 additions and 237 deletions
Download patch file Download diff file Expand all files Collapse all files

1
libraries/scrcpy-decoder-webcodecs/src/index.ts Normal file
View file

@ -0,0 +1 @@
export * from "./video/index.js";

101
libraries/scrcpy-decoder-webcodecs/src/video/codec/av1.ts Normal file
View file

@ -0,0 +1,101 @@
import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
import { Av1 } from "@yume-chan/scrcpy";
import type { CodecDecoder } from "./type.js";
import { decimalTwoDigits } from "./utils.js";
export class Av1Codec implements CodecDecoder {
#decoder: VideoDecoder;
#updateSize: (width: number, height: number) => void;
constructor(
decoder: VideoDecoder,
updateSize: (width: number, height: number) => void,
) {
this.#decoder = decoder;
this.#updateSize = updateSize;
}
#configure(data: Uint8Array) {
const parser = new Av1(data);
const sequenceHeader = parser.searchSequenceHeaderObu();
if (!sequenceHeader) {
return;
}
const {
seq_profile: seqProfile,
seq_level_idx: [seqLevelIdx = 0],
max_frame_width_minus_1,
max_frame_height_minus_1,
color_config: {
BitDepth,
mono_chrome: monoChrome,
subsampling_x: subsamplingX,
subsampling_y: subsamplingY,
chroma_sample_position: chromaSamplePosition,
color_description_present_flag,
},
} = sequenceHeader;
let colorPrimaries: Av1.ColorPrimaries;
let transferCharacteristics: Av1.TransferCharacteristics;
let matrixCoefficients: Av1.MatrixCoefficients;
let colorRange: boolean;
if (color_description_present_flag) {
({
color_primaries: colorPrimaries,
transfer_characteristics: transferCharacteristics,
matrix_coefficients: matrixCoefficients,
color_range: colorRange,
} = sequenceHeader.color_config);
} else {
colorPrimaries = Av1.ColorPrimaries.Bt709;
transferCharacteristics = Av1.TransferCharacteristics.Bt709;
matrixCoefficients = Av1.MatrixCoefficients.Bt709;
colorRange = false;
}
const width = max_frame_width_minus_1 + 1;
const height = max_frame_height_minus_1 + 1;
this.#updateSize(width, height);
const codec = [
"av01",
seqProfile.toString(16),
decimalTwoDigits(seqLevelIdx) +
(sequenceHeader.seq_tier[0] ? "H" : "M"),
decimalTwoDigits(BitDepth),
monoChrome ? "1" : "0",
(subsamplingX ? "1" : "0") +
(subsamplingY ? "1" : "0") +
chromaSamplePosition.toString(),
decimalTwoDigits(colorPrimaries),
decimalTwoDigits(transferCharacteristics),
decimalTwoDigits(matrixCoefficients),
colorRange ? "1" : "0",
].join(".");
this.#decoder.configure({
codec,
optimizeForLatency: true,
});
}
decode(packet: ScrcpyMediaStreamPacket): void {
if (packet.type === "configuration") {
return;
}
this.#configure(packet.data);
this.#decoder.decode(
new EncodedVideoChunk({
// Treat `undefined` as `key`, otherwise it won't decode.
type: packet.keyframe === false ? "delta" : "key",
timestamp: 0,
data: packet.data,
}),
);
}
}

42
libraries/scrcpy-decoder-webcodecs/src/video/codec/h264.ts Normal file
View file

@ -0,0 +1,42 @@
import { h264ParseConfiguration } from "@yume-chan/scrcpy";
import { H26xDecoder } from "./h26x.js";
import { hexTwoDigits } from "./utils.js";
export class H264Decoder extends H26xDecoder {
#decoder: VideoDecoder;
#updateSize: (width: number, height: number) => void;
constructor(
decoder: VideoDecoder,
updateSize: (width: number, height: number) => void,
) {
super(decoder);
this.#decoder = decoder;
this.#updateSize = updateSize;
}
override configure(data: Uint8Array): void {
const {
profileIndex,
constraintSet,
levelIndex,
croppedWidth,
croppedHeight,
} = h264ParseConfiguration(data);
this.#updateSize(croppedWidth, croppedHeight);
// https://www.rfc-editor.org/rfc/rfc6381#section-3.3
// ISO Base Media File Format Name Space
const codec =
"avc1." +
hexTwoDigits(profileIndex) +
hexTwoDigits(constraintSet) +
hexTwoDigits(levelIndex);
this.#decoder.configure({
codec: codec,
optimizeForLatency: true,
});
}
}

47
libraries/scrcpy-decoder-webcodecs/src/video/codec/h265.ts Normal file
View file

@ -0,0 +1,47 @@
import { getUint32LittleEndian } from "@yume-chan/no-data-view";
import { h265ParseConfiguration } from "@yume-chan/scrcpy";
import { H26xDecoder } from "./h26x.js";
import { hexDigits } from "./utils.js";
export class H265Decoder extends H26xDecoder {
#decoder: VideoDecoder;
#updateSize: (width: number, height: number) => void;
constructor(
decoder: VideoDecoder,
updateSize: (width: number, height: number) => void,
) {
super(decoder);
this.#decoder = decoder;
this.#updateSize = updateSize;
}
override configure(data: Uint8Array): void {
const {
generalProfileSpace,
generalProfileIndex,
generalProfileCompatibilitySet,
generalTierFlag,
generalLevelIndex,
generalConstraintSet,
croppedWidth,
croppedHeight,
} = h265ParseConfiguration(data);
this.#updateSize(croppedWidth, croppedHeight);
const codec = [
"hev1",
["", "A", "B", "C"][generalProfileSpace]! +
generalProfileIndex.toString(),
hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
(generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
...Array.from(generalConstraintSet, hexDigits),
].join(".");
this.#decoder.configure({
codec,
optimizeForLatency: true,
});
}
}

45
libraries/scrcpy-decoder-webcodecs/src/video/codec/h26x.ts Normal file
View file

@ -0,0 +1,45 @@
import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
import type { CodecDecoder } from "./type.js";
export abstract class H26xDecoder implements CodecDecoder {
#config: Uint8Array | undefined;
#decoder: VideoDecoder;
constructor(decoder: VideoDecoder) {
this.#decoder = decoder;
}
abstract configure(data: Uint8Array): void;
decode(packet: ScrcpyMediaStreamPacket): void {
if (packet.type === "configuration") {
this.#config = packet.data;
this.configure(packet.data);
return;
}
// For H.264 and H.265, when the stream is in Annex B format
// (which Scrcpy uses, as Android MediaCodec produces),
// configuration data needs to be combined with the first frame data.
// https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
let data: Uint8Array;
if (this.#config !== undefined) {
data = new Uint8Array(this.#config.length + packet.data.length);
data.set(this.#config, 0);
data.set(packet.data, this.#config.length);
this.#config = undefined;
} else {
data = packet.data;
}
this.#decoder.decode(
new EncodedVideoChunk({
// Treat `undefined` as `key`, otherwise won't decode.
type: packet.keyframe === false ? "delta" : "key",
timestamp: 0,
data,
}),
);
}
}

6
libraries/scrcpy-decoder-webcodecs/src/video/codec/index.ts Normal file
View file

@ -0,0 +1,6 @@
export * from "./av1.js";
export * from "./h264.js";
export * from "./h265.js";
export * from "./h26x.js";
export * from "./type.js";
export * from "./utils.js";

12
libraries/scrcpy-decoder-webcodecs/src/video/codec/type.ts Normal file
View file

@ -0,0 +1,12 @@
import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
export interface CodecDecoder {
decode(packet: ScrcpyMediaStreamPacket): void;
}
export interface CodecDecoderConstructor {
new (
decoder: VideoDecoder,
updateSize: (width: number, height: number) => void,
): CodecDecoder;
}

11
libraries/scrcpy-decoder-webcodecs/src/video/codec/utils.ts Normal file
View file

@ -0,0 +1,11 @@
export function hexDigits(value: number) {
return value.toString(16).toUpperCase();
}
export function hexTwoDigits(value: number) {
return value.toString(16).toUpperCase().padStart(2, "0");
}
export function decimalTwoDigits(value: number) {
return value.toString(10).padStart(2, "0");
}

257
libraries/scrcpy-decoder-webcodecs/src/video/decoder.ts
View file

@ -1,15 +1,6 @@
import { EventEmitter } from "@yume-chan/event";
import { getUint32LittleEndian } from "@yume-chan/no-data-view";
import type {
ScrcpyMediaStreamDataPacket,
ScrcpyMediaStreamPacket,
} from "@yume-chan/scrcpy";
import {
Av1,
ScrcpyVideoCodecId,
h264ParseConfiguration,
h265ParseConfiguration,
} from "@yume-chan/scrcpy";
import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
import { ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
import type {
ScrcpyVideoDecoder,
ScrcpyVideoDecoderCapability,
@ -17,21 +8,10 @@ import type {
import type { WritableStreamDefaultController } from "@yume-chan/stream-extra";
import { WritableStream } from "@yume-chan/stream-extra";
import { BitmapFrameRenderer } from "./bitmap.js";
import type { FrameRenderer } from "./renderer.js";
import { WebGLFrameRenderer } from "./webgl.js";
function hexDigits(value: number) {
return value.toString(16).toUpperCase();
}
function hexTwoDigits(value: number) {
return value.toString(16).toUpperCase().padStart(2, "0");
}
function decimalTwoDigits(value: number) {
return value.toString(10).padStart(2, "0");
}
import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
import type { CodecDecoder } from "./codec/type.js";
import type { FrameRenderer } from "./render/index.js";
import { BitmapFrameRenderer, WebGLFrameRenderer } from "./render/index.js";
export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
static isSupported() {
@ -49,6 +29,8 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
return this.#codec;
}
#codecDecoder: CodecDecoder;
#writable: WritableStream<ScrcpyMediaStreamPacket>;
get writable() {
return this.#writable;
@ -60,12 +42,12 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
}
#frameRendered = 0;
get frameRendered() {
get framesRendered() {
return this.#frameRendered;
}
#frameSkipped = 0;
get frameSkipped() {
get framesSkipped() {
return this.#frameSkipped;
}
@ -75,7 +57,6 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
}
#decoder: VideoDecoder;
#config: Uint8Array | undefined;
#renderer: FrameRenderer;
#currentFrameRendered = false;
@ -111,7 +92,7 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
}
this.#currentFrameRendered = false;
// PERF: H.264 renderer may draw multiple frames in one vertical sync interval to minimize latency.
// PERF: Draw every frame to minimize latency at cost of performance.
// When multiple frames are drawn in one vertical sync interval,
// only the last one is visible to users.
// But this ensures users can always see the most up-to-date screen.
@ -134,6 +115,27 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
},
});
switch (this.#codec) {
case ScrcpyVideoCodecId.H264:
this.#codecDecoder = new H264Decoder(
this.#decoder,
this.#updateSize,
);
break;
case ScrcpyVideoCodecId.H265:
this.#codecDecoder = new H265Decoder(
this.#decoder,
this.#updateSize,
);
break;
case ScrcpyVideoCodecId.AV1:
this.#codecDecoder = new Av1Codec(
this.#decoder,
this.#updateSize,
);
break;
}
let error: Error | undefined;
let controller: WritableStreamDefaultController | undefined;
this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
@ -145,38 +147,14 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
}
},
write: (packet) => {
if (this.#codec === ScrcpyVideoCodecId.AV1) {
if (packet.type === "configuration") {
return;
}
this.#configureAv1(packet.data);
this.#decoder.decode(
new EncodedVideoChunk({
// Treat `undefined` as `key`, otherwise it won't decode.
type: packet.keyframe === false ? "delta" : "key",
timestamp: 0,
data: packet.data,
}),
);
return;
}
switch (packet.type) {
case "configuration":
this.#configure(packet.data);
break;
case "data":
this.#decode(packet);
break;
}
this.#codecDecoder.decode(packet);
},
});
this.#onFramePresented();
}
#updateSize(width: number, height: number) {
#updateSize = (width: number, height: number) => {
if (width !== this.#canvas.width || height !== this.#canvas.height) {
this.#canvas.width = width;
this.#canvas.height = height;
@ -185,178 +163,13 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
height: height,
});
}
}
};
#onFramePresented = () => {
this.#currentFrameRendered = true;
this.#animationFrameId = requestAnimationFrame(this.#onFramePresented);
};
#configureH264(data: Uint8Array) {
const {
profileIndex,
constraintSet,
levelIndex,
croppedWidth,
croppedHeight,
} = h264ParseConfiguration(data);
this.#updateSize(croppedWidth, croppedHeight);
// https://www.rfc-editor.org/rfc/rfc6381#section-3.3
// ISO Base Media File Format Name Space
const codec =
"avc1." +
hexTwoDigits(profileIndex) +
hexTwoDigits(constraintSet) +
hexTwoDigits(levelIndex);
this.#decoder.configure({
codec: codec,
optimizeForLatency: true,
});
}
#configureH265(data: Uint8Array) {
const {
generalProfileSpace,
generalProfileIndex,
generalProfileCompatibilitySet,
generalTierFlag,
generalLevelIndex,
generalConstraintSet,
croppedWidth,
croppedHeight,
} = h265ParseConfiguration(data);
this.#updateSize(croppedWidth, croppedHeight);
const codec = [
"hev1",
["", "A", "B", "C"][generalProfileSpace]! +
generalProfileIndex.toString(),
hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
(generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
...Array.from(generalConstraintSet, hexDigits),
].join(".");
this.#decoder.configure({
codec,
optimizeForLatency: true,
});
}
#configureAv1(data: Uint8Array) {
const parser = new Av1(data);
const sequenceHeader = parser.searchSequenceHeaderObu();
if (!sequenceHeader) {
return;
}
const {
seq_profile: seqProfile,
seq_level_idx: [seqLevelIdx = 0],
max_frame_width_minus_1,
max_frame_height_minus_1,
color_config: {
BitDepth,
mono_chrome: monoChrome,
subsampling_x: subsamplingX,
subsampling_y: subsamplingY,
chroma_sample_position: chromaSamplePosition,
color_description_present_flag,
},
} = sequenceHeader;
let colorPrimaries: Av1.ColorPrimaries;
let transferCharacteristics: Av1.TransferCharacteristics;
let matrixCoefficients: Av1.MatrixCoefficients;
let colorRange: boolean;
if (color_description_present_flag) {
({
color_primaries: colorPrimaries,
transfer_characteristics: transferCharacteristics,
matrix_coefficients: matrixCoefficients,
color_range: colorRange,
} = sequenceHeader.color_config);
} else {
colorPrimaries = Av1.ColorPrimaries.Bt709;
transferCharacteristics = Av1.TransferCharacteristics.Bt709;
matrixCoefficients = Av1.MatrixCoefficients.Bt709;
colorRange = false;
}
const width = max_frame_width_minus_1 + 1;
const height = max_frame_height_minus_1 + 1;
this.#updateSize(width, height);
const codec = [
"av01",
seqProfile.toString(16),
decimalTwoDigits(seqLevelIdx) +
(sequenceHeader.seq_tier[0] ? "H" : "M"),
decimalTwoDigits(BitDepth),
monoChrome ? "1" : "0",
(subsamplingX ? "1" : "0") +
(subsamplingY ? "1" : "0") +
chromaSamplePosition.toString(),
decimalTwoDigits(colorPrimaries),
decimalTwoDigits(transferCharacteristics),
decimalTwoDigits(matrixCoefficients),
colorRange ? "1" : "0",
].join(".");
this.#decoder.configure({
codec,
optimizeForLatency: true,
});
}
#configure(data: Uint8Array) {
switch (this.#codec) {
case ScrcpyVideoCodecId.H264:
this.#configureH264(data);
this.#config = data;
break;
case ScrcpyVideoCodecId.H265:
this.#configureH265(data);
this.#config = data;
break;
case ScrcpyVideoCodecId.AV1:
// AV1 configuration is in normal stream
break;
}
}
#decode(packet: ScrcpyMediaStreamDataPacket) {
if (this.#decoder.state !== "configured") {
return;
}
// For H.264 and H.265, when the stream is in Annex B format
// (which Scrcpy uses, as Android MediaCodec produces),
// configuration data needs to be combined with the first frame data.
// https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
// AV1 doesn't need to do this, the handling code also doesn't set `#config`.
let data: Uint8Array;
if (this.#config !== undefined) {
data = new Uint8Array(this.#config.length + packet.data.length);
data.set(this.#config, 0);
data.set(packet.data, this.#config.length);
this.#config = undefined;
} else {
data = packet.data;
}
this.#decoder.decode(
new EncodedVideoChunk({
// Treat `undefined` as `key`, otherwise won't decode.
type: packet.keyframe === false ? "delta" : "key",
timestamp: 0,
data,
}),
);
}
dispose() {
cancelAnimationFrame(this.#animationFrameId);
if (this.#decoder.state !== "closed") {

3
libraries/scrcpy-decoder-webcodecs/src/video/index.ts Normal file
View file

@ -0,0 +1,3 @@
export * from "./codec/index.js";
export * from "./decoder.js";
export * from "./render/index.js";

2
libraries/scrcpy-decoder-webcodecs/src/video/render/bitmap.ts
View file

@ -1,4 +1,4 @@
import type { FrameRenderer } from "./renderer.js";
import type { FrameRenderer } from "./type.js";
export class BitmapFrameRenderer implements FrameRenderer {
#context: ImageBitmapRenderingContext;

3
libraries/scrcpy-decoder-webcodecs/src/video/render/index.ts Normal file
View file

@ -0,0 +1,3 @@
export * from "./bitmap.js";
export * from "./type.js";
export * from "./webgl.js";

28
libraries/scrcpy-decoder-webcodecs/src/video/render/webgl.ts
View file

@ -1,27 +1,27 @@
import type { FrameRenderer } from "./renderer.js";
import type { FrameRenderer } from "./type.js";
export class WebGLFrameRenderer implements FrameRenderer {
static vertexShaderSource = `
attribute vec2 xy;
attribute vec2 xy;
varying highp vec2 uv;
varying highp vec2 uv;
void main(void) {
gl_Position = vec4(xy, 0.0, 1.0);
// Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
// UV coordinates are Y-flipped relative to vertex coordinates.
uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
}
void main(void) {
gl_Position = vec4(xy, 0.0, 1.0);
// Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
// UV coordinates are Y-flipped relative to vertex coordinates.
uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
}
`;
static fragmentShaderSource = `
varying highp vec2 uv;
varying highp vec2 uv;
uniform sampler2D texture;
uniform sampler2D texture;
void main(void) {
gl_FragColor = texture2D(texture, uv);
}
void main(void) {
gl_FragColor = texture2D(texture, uv);
}
`;
#context: WebGLRenderingContext;

2
toolchain/eslint-config/run-eslint.js
View file

@ -16,6 +16,8 @@ const child = spawn(
eslint,
["--config", resolve(__dirname, "eslint.config.js"), "--fix", "."],
{
// https://github.com/nodejs/node/issues/52554
shell: true,
stdio: "inherit",
},
);