diff --git a/.vscode/settings.json b/.vscode/settings.json index 825f9309..2d5826a1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,6 +8,7 @@ "autorun", "Backquote", "Bframes", + "Bitstream", "bootloader", "brotli", "Callout", @@ -26,6 +27,7 @@ "Embedder", "entrypoints", "fflate", + "flac", "fluentui", "genymobile", "Genymobile's", @@ -56,6 +58,7 @@ "streamsaver", "struct", "struct's", + "subsampling", "tcpip", "tinyh", "transferables", diff --git a/libraries/scrcpy-decoder-webcodecs/src/index.ts b/libraries/scrcpy-decoder-webcodecs/src/index.ts index 3732585e..0f0d2d48 100644 --- a/libraries/scrcpy-decoder-webcodecs/src/index.ts +++ b/libraries/scrcpy-decoder-webcodecs/src/index.ts @@ -1,14 +1,13 @@ import { EventEmitter } from "@yume-chan/event"; -import type { - ScrcpyMediaStreamDataPacket, - ScrcpyMediaStreamPacket, -} from "@yume-chan/scrcpy"; +import { getUint32LittleEndian } from "@yume-chan/no-data-view"; import { + Av1, ScrcpyVideoCodecId, h264ParseConfiguration, h265ParseConfiguration, + type ScrcpyMediaStreamDataPacket, + type ScrcpyMediaStreamPacket, } from "@yume-chan/scrcpy"; -import { getUint32LittleEndian } from "@yume-chan/no-data-view"; import type { ScrcpyVideoDecoder, ScrcpyVideoDecoderCapability, @@ -19,11 +18,19 @@ import { BitmapFrameRenderer } from "./bitmap.js"; import type { FrameRenderer } from "./renderer.js"; import { WebGLFrameRenderer } from "./webgl.js"; -function toHex(value: number) { - return value.toString(16).padStart(2, "0").toUpperCase(); +function hexDigits(value: number) { + return value.toString(16).toUpperCase(); } -export class WebCodecsDecoder implements ScrcpyVideoDecoder { +function hexTwoDigits(value: number) { + return value.toString(16).toUpperCase().padStart(2, "0"); +} + +function decimalTwoDigits(value: number) { + return value.toString(10).padStart(2, "0"); +} + +export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder { static isSupported() { return typeof globalThis.VideoDecoder !== "undefined"; } @@ -149,9 +156,9 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder { // ISO Base Media File Format Name Space const codec = "avc1." + - toHex(profileIndex) + - toHex(constraintSet) + - toHex(levelIndex); + hexTwoDigits(profileIndex) + + hexTwoDigits(constraintSet) + + hexTwoDigits(levelIndex); this.#decoder.configure({ codec: codec, optimizeForLatency: true, @@ -181,16 +188,57 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder { "hev1", ["", "A", "B", "C"][generalProfileSpace]! + generalProfileIndex.toString(), - getUint32LittleEndian(generalProfileCompatibilitySet, 0).toString( - 16, - ), + hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)), (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(), - getUint32LittleEndian(generalConstraintSet, 0) - .toString(16) - .toUpperCase(), - getUint32LittleEndian(generalConstraintSet, 4) - .toString(16) - .toUpperCase(), + ...Array.from(generalConstraintSet, hexDigits), + ].join("."); + this.#decoder.configure({ + codec, + optimizeForLatency: true, + }); + } + + #configureAv1(data: Uint8Array) { + let sequenceHeader: Av1.SequenceHeaderObu | undefined; + const av1 = new Av1(data); + for (const obu of av1.bitstream()) { + if (obu.sequence_header_obu) { + sequenceHeader = obu.sequence_header_obu; + } + } + if (!sequenceHeader) { + throw new Error("No sequence header found"); + } + + const { + seq_profile: seqProfile, + seq_level_idx: [seqLevelIdx = 0], + color_config: { + BitDepth, + mono_chrome: monoChrome, + subsampling_x: subsamplingX, + subsampling_y: subsamplingY, + chroma_sample_position: chromaSamplePosition, + color_primaries: colorPrimaries, + transfer_characteristics: transferCharacteristics, + matrix_coefficients: matrixCoefficients, + color_range: colorRange, + }, + } = sequenceHeader; + const codec = [ + "av01", + seqProfile.toString(16), + decimalTwoDigits(seqLevelIdx) + + (sequenceHeader.seq_tier[0] ? "H" : "M"), + decimalTwoDigits(BitDepth), + monoChrome ? "1" : "0", + (subsamplingX ? "1" : "0") + + (subsamplingY ? "1" : "0") + + chromaSamplePosition.toString(), + decimalTwoDigits(colorPrimaries), + decimalTwoDigits(transferCharacteristics), + decimalTwoDigits(matrixCoefficients), + colorRange ? "1" : "0", ].join("."); this.#decoder.configure({ codec, @@ -206,6 +254,9 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder { case ScrcpyVideoCodecId.H265: this.#configureH265(data); break; + case ScrcpyVideoCodecId.AV1: + this.#configureAv1(data); + break; } this.#config = data; } diff --git a/libraries/scrcpy/src/codec/av1.ts b/libraries/scrcpy/src/codec/av1.ts index e69f9b22..a1558cbf 100644 --- a/libraries/scrcpy/src/codec/av1.ts +++ b/libraries/scrcpy/src/codec/av1.ts @@ -1,3 +1,11 @@ +// cspell: ignore uvlc +// cspell: ignore interintra +// cspell: ignore superres +// cspell: ignore cdef +// cspell: ignore bitdepth +// cspell: ignore Smpte +// cspell: ignore Chromat + export enum AndroidAv1Profile { Main8 = 1 << 0, Main10 = 1 << 1, @@ -31,3 +39,643 @@ export enum AndroidAv1Level { Level72 = 1 << 22, Level73 = 1 << 23, } + +class BitReader { + #data: Uint8Array; + #byte: number; + + #bytePosition: number = 0; + #bitPosition: number = 7; + + get byteAligned() { + return this.#bitPosition === 7; + } + + get ended() { + return this.#bytePosition >= this.#data.length; + } + + constructor(data: Uint8Array) { + this.#data = data; + this.#byte = data[0]!; + } + + f1() { + const value = this.#byte >> this.#bitPosition; + this.#bitPosition -= 1; + if (this.#bitPosition < 0) { + this.#bytePosition += 1; + this.#bitPosition = 7; + this.#byte = this.#data[this.#bytePosition]!; + } + return value & 1; + } + + f(n: number) { + let value = 0; + for (; n > 0; n -= 1) { + value <<= 1; + value |= this.f1(); + } + return value; + } + + skip(n: number) { + const bytes = (n / 8) | 0; + if (bytes > 0) { + this.#bytePosition += bytes; + n -= bytes * 8; + } + + n -= this.#bitPosition + 1; + this.#bytePosition += 1; + this.#bitPosition = 7 - n; + this.#byte = this.#data[this.#bytePosition]!; + } + + readBytes(n: number) { + if (!this.byteAligned) { + throw new Error("Bytes must be byte-aligned"); + } + + const value = this.#data.subarray( + this.#bytePosition, + this.#bytePosition + n, + ); + this.#bytePosition += n; + this.#byte = this.#data[this.#bytePosition]!; + return value; + } + + getPosition() { + return [this.#bytePosition, this.#bitPosition] as const; + } + + setPosition([bytePosition, bitPosition]: readonly [number, number]) { + this.#bytePosition = bytePosition; + this.#bitPosition = bitPosition; + this.#byte = this.#data[bytePosition]!; + } +} + +export class Av1 extends BitReader { + #Leb128Bytes: number = 0; + + uvlc() { + let leadingZeros = 0; + while (!this.f1()) { + leadingZeros += 1; + } + if (leadingZeros >= 32) { + return 2 ** 32 - 1; + } + const value = this.f(leadingZeros); + return value + ((1 << leadingZeros) >>> 0) - 1; + } + + leb128() { + if (!this.byteAligned) { + throw new Error("LEB128 must be byte-aligned"); + } + + let value = 0n; + this.#Leb128Bytes = 0; + for (let i = 0n; i < 8n; i += 1n) { + const leb128_byte = this.f(8); + value |= BigInt(leb128_byte & 0x7f) << (7n * i); + this.#Leb128Bytes += 1; + if ((leb128_byte & 0x80) == 0) { + break; + } + } + return value; + } + + *bitstream(): Generator { + while (!this.ended) { + const temporal_unit_size = this.leb128(); + yield* this.temporalUnit(temporal_unit_size); + } + } + + *temporalUnit(sz: bigint): Generator { + while (sz > 0) { + const frame_unit_size = this.leb128(); + sz -= BigInt(this.#Leb128Bytes); + yield* this.frameUnit(frame_unit_size); + sz -= frame_unit_size; + } + } + + *frameUnit(sz: bigint): Generator { + while (sz > 0) { + const obu_length = this.leb128(); + sz -= BigInt(this.#Leb128Bytes); + const obu = this.openBitstreamUnit(obu_length); + if (obu) { + yield obu; + } + sz -= obu_length; + } + } + + #OperatingPointIdc = 0; + + openBitstreamUnit(sz: bigint) { + const obu_header = this.obuHeader(); + let obu_size: bigint; + if (obu_header.obu_has_size_field) { + obu_size = this.leb128(); + } else { + obu_size = sz - 1n - (obu_header.obu_extension_flag ? 1n : 0n); + } + + const startPosition = this.getPosition(); + + if ( + obu_header.obu_type !== Av1.ObuType.SequenceHeader && + obu_header.obu_type !== Av1.ObuType.TemporalDelimiter && + this.#OperatingPointIdc !== 0 && + obu_header.obu_extension_header + ) { + const inTemporalLayer = !!( + this.#OperatingPointIdc & + (1 << obu_header.obu_extension_header.temporal_id) + ); + const inSpatialLayer = !!( + this.#OperatingPointIdc & + (1 << (obu_header.obu_extension_header.spatial_id + 8)) + ); + if (!inTemporalLayer || !inSpatialLayer) { + this.skip(Number(obu_size)); + return; + } + } + + let sequence_header_obu: + | ReturnType + | undefined; + switch (obu_header.obu_type) { + case Av1.ObuType.SequenceHeader: + sequence_header_obu = this.sequenceHeaderObu(); + break; + } + + const currentPosition = this.getPosition(); + const payloadBits = + (currentPosition[0] - startPosition[0]) * 8 + + (startPosition[1] - currentPosition[1]); + + if ( + obu_size > 0 && + obu_header.obu_type !== Av1.ObuType.TileGroup && + obu_header.obu_type !== Av1.ObuType.TileList && + obu_header.obu_type !== Av1.ObuType.Frame + ) { + this.skip(Number(obu_size) * 8 - payloadBits); + } + + return { + obu_header, + obu_size, + sequence_header_obu, + }; + } + + obuHeader() { + const obu_forbidden_bit = !!this.f1(); + if (obu_forbidden_bit) { + throw new Error("Invalid data"); + } + + const obu_type = this.f(4); + const obu_extension_flag = !!this.f1(); + const obu_has_size_field = !!this.f1(); + this.f1(); + + let obu_extension_header: + | ReturnType + | undefined; + if (obu_extension_flag) { + obu_extension_header = this.obuExtensionHeader(); + } + + return { + obu_type, + obu_extension_flag, + obu_has_size_field, + obu_extension_header, + }; + } + + obuExtensionHeader() { + const temporal_id = this.f(3); + const spatial_id = this.f(2); + this.skip(3); + return { temporal_id, spatial_id }; + } + + static readonly SelectScreenContentTools = 2; + static readonly SelectIntegerMv = 2; + + sequenceHeaderObu() { + const seq_profile = this.f(3); + const still_picture = !!this.f1(); + const reduced_still_picture_header = !!this.f1(); + + let timing_info_present_flag = false; + let timing_info: ReturnType | undefined; + let decoder_model_info_present_flag = false; + let decoder_model_info: ReturnType | undefined; + let initial_display_delay_present_flag = false; + let operating_points_cnt_minus_1 = 0; + const operating_point_idc: number[] = []; + const seq_level_idx: number[] = []; + const seq_tier: number[] = []; + const decoder_model_present_for_this_op: boolean[] = []; + const initial_display_delay_present_for_this_op: boolean[] = []; + let operating_parameters_info: + | ReturnType[] + | undefined; + let initial_display_delay_minus_1: number[] | undefined; + if (reduced_still_picture_header) { + operating_point_idc[0] = 0; + seq_level_idx[0] = this.f(5); + seq_tier[0] = 0; + decoder_model_present_for_this_op[0] = false; + initial_display_delay_present_for_this_op[0] = false; + } else { + timing_info_present_flag = !!this.f1(); + if (timing_info_present_flag) { + timing_info = this.timingInfo(); + decoder_model_info_present_flag = !!this.f1(); + if (decoder_model_info_present_flag) { + decoder_model_info = this.decoderModelInfo(); + operating_parameters_info = []; + } + } + initial_display_delay_present_flag = !!this.f1(); + if (initial_display_delay_present_flag) { + initial_display_delay_minus_1 = []; + } + operating_points_cnt_minus_1 = this.f(5); + for (let i = 0; i <= operating_points_cnt_minus_1; i += 1) { + operating_point_idc[i] = this.f(12); + seq_level_idx[i] = this.f(5); + if (seq_level_idx[i]! > 7) { + seq_tier[i] = this.f1(); + } else { + seq_tier[i] = 0; + } + if (decoder_model_info_present_flag) { + decoder_model_present_for_this_op[i] = !!this.f1(); + if (decoder_model_present_for_this_op[i]) { + operating_parameters_info![i] = + this.operatingParametersInfo(decoder_model_info!); + } + } else { + decoder_model_present_for_this_op[i] = false; + } + if (initial_display_delay_present_flag) { + initial_display_delay_present_for_this_op[i] = !!this.f1(); + if (initial_display_delay_present_for_this_op[i]) { + initial_display_delay_minus_1![i] = this.f(4); + } + } + } + } + + const operatingPoint = this.chooseOperatingPoint(); + this.#OperatingPointIdc = operating_point_idc[operatingPoint]!; + + const frame_width_bits_minus_1 = this.f(4); + const frame_height_bits_minus_1 = this.f(4); + const max_frame_width_minus_1 = this.f(frame_width_bits_minus_1 + 1); + const max_frame_height_minus_1 = this.f(frame_height_bits_minus_1 + 1); + + let frame_id_numbers_present_flag = false; + let delta_frame_id_length_minus_2: number | undefined; + let additional_frame_id_length_minus_1: number | undefined; + if (!reduced_still_picture_header) { + frame_id_numbers_present_flag = !!this.f1(); + if (frame_id_numbers_present_flag) { + delta_frame_id_length_minus_2 = this.f(4); + additional_frame_id_length_minus_1 = this.f(3); + } + } + + const use_128x128_superblock = !!this.f1(); + const enable_filter_intra = !!this.f1(); + const enable_intra_edge_filter = !!this.f1(); + + let enable_interintra_compound = false; + let enable_masked_compound = false; + let enable_warped_motion = false; + let enable_dual_filter = false; + let enable_order_hint = false; + let enable_jnt_comp = false; + let enable_ref_frame_mvs = false; + let seq_choose_screen_content_tools = false; + let seq_force_screen_content_tools = Av1.SelectScreenContentTools; + let seq_choose_integer_mv = false; + let seq_force_integer_mv = Av1.SelectIntegerMv; + let order_hint_bits_minus_1: number | undefined; + // let OrderHintBits = 0; + if (!reduced_still_picture_header) { + enable_interintra_compound = !!this.f1(); + enable_masked_compound = !!this.f1(); + enable_warped_motion = !!this.f1(); + enable_dual_filter = !!this.f1(); + + enable_order_hint = !!this.f1(); + if (enable_order_hint) { + enable_jnt_comp = !!this.f1(); + enable_ref_frame_mvs = !!this.f1(); + } + + seq_choose_screen_content_tools = !!this.f1(); + if (!seq_choose_screen_content_tools) { + seq_force_screen_content_tools = this.f1(); + } + + if (seq_force_screen_content_tools > 0) { + seq_choose_integer_mv = !!this.f1(); + if (!seq_choose_integer_mv) { + seq_force_integer_mv = this.f1(); + } + } + + if (enable_order_hint) { + order_hint_bits_minus_1 = this.f(3); + // OrderHintBits = order_hint_bits_minus_1 + 1; + } + } + + const enable_superres = !!this.f1(); + const enable_cdef = !!this.f1(); + const enable_restoration = !!this.f1(); + const color_config = this.colorConfig(seq_profile); + const film_grain_params_present = !!this.f1(); + + return { + seq_profile, + still_picture, + reduced_still_picture_header, + timing_info_present_flag, + timing_info, + decoder_model_info_present_flag, + decoder_model_info, + initial_display_delay_present_flag, + initial_display_delay_minus_1, + operating_points_cnt_minus_1, + operating_point_idc, + seq_level_idx, + seq_tier, + decoder_model_present_for_this_op, + operating_parameters_info, + initial_display_delay_present_for_this_op, + frame_width_bits_minus_1, + frame_height_bits_minus_1, + max_frame_width_minus_1, + max_frame_height_minus_1, + frame_id_numbers_present_flag, + delta_frame_id_length_minus_2, + additional_frame_id_length_minus_1, + use_128x128_superblock, + enable_filter_intra, + enable_intra_edge_filter, + enable_interintra_compound, + enable_masked_compound, + enable_warped_motion, + enable_dual_filter, + enable_order_hint, + enable_jnt_comp, + enable_ref_frame_mvs, + seq_choose_screen_content_tools, + seq_force_screen_content_tools, + seq_choose_integer_mv, + seq_force_integer_mv, + order_hint_bits_minus_1, + enable_superres, + enable_cdef, + enable_restoration, + color_config, + film_grain_params_present, + }; + } + + timingInfo() { + const num_units_in_display_tick = this.f(32); + const time_scale = this.f(32); + const equal_picture_interval = !!this.f1(); + let num_ticks_per_picture_minus_1: number | undefined; + if (equal_picture_interval) { + num_ticks_per_picture_minus_1 = this.uvlc(); + } + return { + num_units_in_display_tick, + time_scale, + equal_picture_interval, + num_ticks_per_picture_minus_1, + }; + } + + decoderModelInfo() { + const buffer_delay_length_minus_1 = this.f(5); + const num_units_in_decoding_tick = this.f(32); + const buffer_removal_time_length_minus_1 = this.f(5); + const frame_presentation_time_length_minus_1 = this.f(5); + return { + buffer_delay_length_minus_1, + num_units_in_decoding_tick, + buffer_removal_time_length_minus_1, + frame_presentation_time_length_minus_1, + }; + } + + operatingParametersInfo( + decoderModelInfo: ReturnType, + ) { + const n = decoderModelInfo.buffer_delay_length_minus_1 + 1; + const decoder_buffer_delay = this.f(n); + const encoder_buffer_delay = this.f(n); + const low_delay_mode_flag = !!this.f1(); + return { + decoder_buffer_delay, + encoder_buffer_delay, + low_delay_mode_flag, + }; + } + + chooseOperatingPoint() { + return 0; + } + + colorConfig(seq_profile: number) { + const high_bitdepth = !!this.f1(); + let twelve_bit = false; + let BitDepth = 8; + if (seq_profile === 2 && high_bitdepth) { + twelve_bit = !!this.f1(); + BitDepth = twelve_bit ? 12 : 10; + } else if (seq_profile <= 2) { + BitDepth = high_bitdepth ? 10 : 8; + } + + let mono_chrome = false; + if (seq_profile === 1) { + mono_chrome = !!this.f1(); + } + + // const NumPlanes = mono_chrome ? 1 : 3; + + const color_description_present_flag = !!this.f1(); + let color_primaries = Av1.ColorPrimaries.Unspecified; + let transfer_characteristics = Av1.TransferCharacteristics.Unspecified; + let matrix_coefficients = Av1.MatrixCoefficients.Unspecified; + if (color_description_present_flag) { + color_primaries = this.f(8); + transfer_characteristics = this.f(8); + matrix_coefficients = this.f(8); + } + + let color_range = false; + let subsampling_x: boolean; + let subsampling_y: boolean; + let chroma_sample_position = 0; + let separate_uv_delta_q = false; + if (mono_chrome) { + color_range = !!this.f1(); + subsampling_x = true; + subsampling_y = true; + } else { + if ( + color_primaries === Av1.ColorPrimaries.Bt709 && + transfer_characteristics === Av1.TransferCharacteristics.Srgb && + matrix_coefficients === Av1.MatrixCoefficients.Identity + ) { + color_range = true; + subsampling_x = false; + subsampling_y = false; + } else { + color_range = !!this.f1(); + switch (seq_profile) { + case 0: + subsampling_x = true; + subsampling_y = true; + break; + case 1: + subsampling_x = false; + subsampling_y = false; + break; + default: + if (BitDepth == 12) { + subsampling_x = !!this.f1(); + if (subsampling_x) { + subsampling_y = !!this.f1(); + } else { + subsampling_y = false; + } + } else { + subsampling_x = true; + subsampling_y = false; + } + break; + } + if (subsampling_x && subsampling_y) { + chroma_sample_position = this.f(2); + } + } + separate_uv_delta_q = !!this.f1(); + } + + return { + high_bitdepth, + twelve_bit, + BitDepth, + mono_chrome, + color_description_present_flag, + color_primaries, + transfer_characteristics, + matrix_coefficients, + color_range, + subsampling_x, + subsampling_y, + chroma_sample_position, + separate_uv_delta_q, + }; + } +} + +export namespace Av1 { + export type OpenBitstreamUnit = Exclude< + ReturnType, + undefined + >; + + export type SequenceHeaderObu = ReturnType; + + export enum ObuType { + SequenceHeader = 1, + TemporalDelimiter, + FrameHeader, + TileGroup, + Metadata, + Frame, + RedundantFrameHeader, + TileList, + Padding = 15, + } + + export enum ColorPrimaries { + Bt709 = 1, + Unspecified, + Bt470M = 4, + Bt470BG, + Bt601, + Smpte240, + GenericFilm, + Bt2020, + Xyz, + Smpte431, + Smpte432, + Ebu3213 = 22, + } + + export enum TransferCharacteristics { + Bt709 = 1, + Unspecified, + Bt470M = 4, + Bt470BG, + Bt601, + Smpte240, + Linear, + Log100, + Log100Sqrt10, + Iec61966, + Bt1361, + Srgb, + Bt2020Ten, + Bt2020Twelve, + Smpte2084, + Smpte428, + Hlg, + } + + export enum MatrixCoefficients { + Identity = 0, + Bt709, + Unspecified, + Fcc = 4, + Bt470BG, + Bt601, + Smpte240, + YCgCo, + Bt2020Ncl, + Bt2020Cl, + Smpte2085, + ChromatNcl, + ChromatCl, + ICtCp, + } +} diff --git a/libraries/scrcpy/src/codec/nalu.spec.ts b/libraries/scrcpy/src/codec/nalu.spec.ts index f366944a..2f49c095 100644 --- a/libraries/scrcpy/src/codec/nalu.spec.ts +++ b/libraries/scrcpy/src/codec/nalu.spec.ts @@ -3,84 +3,198 @@ import { describe, expect, it } from "@jest/globals"; import { NaluSodbBitReader } from "./nalu.js"; describe("nalu", () => { - describe.only("NaluSodbReader", () => { - it("should throw error if no end bit found", () => { - expect( - () => new NaluSodbBitReader(new Uint8Array([0b00000000])), - ).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`); - expect( - () => - new NaluSodbBitReader( - new Uint8Array([0b00000000, 0b00000000]), - ), - ).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`); + describe("NaluSodbReader", () => { + describe("constructor", () => { + it("should set `ended` if stream is effectively empty", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b10000000]), + ); + expect(reader).toHaveProperty("ended", true); + }); + + it("should throw error if stream is empty", () => { + expect( + () => new NaluSodbBitReader(new Uint8Array(0)), + ).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`); + }); + + it("should throw error if no end bit found (single byte)", () => { + expect( + () => new NaluSodbBitReader(new Uint8Array(1)), + ).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`); + }); + + it("should throw error if no end bit found (multiple bytes)", () => { + expect( + () => new NaluSodbBitReader(new Uint8Array(10)), + ).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`); + }); }); - it("should throw error if read after end bit", () => { - let reader = new NaluSodbBitReader(new Uint8Array([0b10000000])); - expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( - `"Bit index out of bounds"`, - ); - - reader = new NaluSodbBitReader( - new Uint8Array([0b11111111, 0b10000000]), - ); - for (let i = 0; i < 8; i += 1) { + describe("next", () => { + it("should read bits in Big Endian (single byte)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b10110111]), + ); expect(reader.next()).toBe(1); - } + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(1); + }); + + it("should read bits in Big Endian (multiple bytes)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b01001000, 0b10000100, 0b00010001]), + ); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + expect(reader.next()).toBe(0); + }); + + it("should throw error if read after end bit (single byte, middle)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b11111000]), + ); + for (let i = 0; i < 4; i += 1) { + expect(reader.next()).toBe(1); + } + expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( + `"Bit index out of bounds"`, + ); + }); + + it("should throw error if read after end bit (single byte, end)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b11111111]), + ); + for (let i = 0; i < 7; i += 1) { + expect(reader.next()).toBe(1); + } + expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( + `"Bit index out of bounds"`, + ); + }); + + it("should throw error if read after end bit (multiple bytes, start)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b11111111, 0b10000000]), + ); + for (let i = 0; i < 8; i += 1) { + expect(reader.next()).toBe(1); + } + expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( + `"Bit index out of bounds"`, + ); + }); + + it("should throw error if read after end bit (multiple bytes, middle)", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0b11111111, 0b11111000]), + ); + for (let i = 0; i < 12; i += 1) { + expect(reader.next()).toBe(1); + } + expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( + `"Bit index out of bounds"`, + ); + }); + + it("should skip emulation prevent byte", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([0xff, 0x00, 0x00, 0x03, 0xff, 0x80]), + ); + for (let i = 0; i < 8; i += 1) { + expect(reader.next()).toBe(1); + } + for (let i = 0; i < 16; i += 1) { + expect(reader.next()).toBe(0); + } + for (let i = 0; i < 8; i += 1) { + expect(reader.next()).toBe(1); + } + }); + + it("should skip successive emulation prevent bytes", () => { + const reader = new NaluSodbBitReader( + new Uint8Array([ + 0xff, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0xff, 0x80, + ]), + ); + for (let i = 0; i < 8; i += 1) { + expect(reader.next()).toBe(1); + } + for (let i = 0; i < 32; i += 1) { + expect(reader.next()).toBe(0); + } + for (let i = 0; i < 8; i += 1) { + expect(reader.next()).toBe(1); + } + }); + }); + }); + + describe("skip", () => { + it("should skip <8 bits in single byte", () => { + const reader = new NaluSodbBitReader(new Uint8Array([0b01000011])); + + reader.skip(1); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + + reader.skip(3); + expect(reader.next()).toBe(1); expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( `"Bit index out of bounds"`, ); }); - it("should skip emulation prevent byte", () => { + it("should skip <8 bits in multiple bytes", () => { const reader = new NaluSodbBitReader( - new Uint8Array([0xff, 0x00, 0x00, 0x03, 0xff, 0x80]), + new Uint8Array([0b00000100, 0b00101000]), + ); + + reader.skip(5); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + + reader.skip(3); + expect(reader.next()).toBe(1); + expect(reader.next()).toBe(0); + expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot( + `"Bit index out of bounds"`, ); - for (let i = 0; i < 8; i += 1) { - expect(reader.next()).toBe(1); - } - for (let i = 0; i < 16; i += 1) { - expect(reader.next()).toBe(0); - } - for (let i = 0; i < 8; i += 1) { - expect(reader.next()).toBe(1); - } }); - it("should skip successive emulation prevent bytes", () => { + it("should skip >8 bits without emulation prevention byte", () => { const reader = new NaluSodbBitReader( - new Uint8Array([ - 0xff, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0xff, 0x80, - ]), + new Uint8Array([0b00000000, 0b00100001]), ); - for (let i = 0; i < 8; i += 1) { - expect(reader.next()).toBe(1); - } - for (let i = 0; i < 32; i += 1) { - expect(reader.next()).toBe(0); - } - for (let i = 0; i < 8; i += 1) { - expect(reader.next()).toBe(1); - } - }); - - it("should read bits in Big Endian", () => { - let reader = new NaluSodbBitReader(new Uint8Array([0b10110011])); + reader.skip(10); expect(reader.next()).toBe(1); expect(reader.next()).toBe(0); - expect(reader.next()).toBe(1); - expect(reader.next()).toBe(1); - expect(reader.next()).toBe(0); - expect(reader.next()).toBe(0); - expect(reader.next()).toBe(1); - - reader = new NaluSodbBitReader(new Uint8Array([0b01001100])); - expect(reader.next()).toBe(0); - expect(reader.next()).toBe(1); - expect(reader.next()).toBe(0); - expect(reader.next()).toBe(0); - expect(reader.next()).toBe(1); }); }); }); diff --git a/libraries/scrcpy/src/codec/nalu.ts b/libraries/scrcpy/src/codec/nalu.ts index 64a2719b..73db07d4 100644 --- a/libraries/scrcpy/src/codec/nalu.ts +++ b/libraries/scrcpy/src/codec/nalu.ts @@ -232,12 +232,15 @@ export function naluRemoveEmulation(buffer: Uint8Array) { export class NaluSodbBitReader { readonly #nalu: Uint8Array; + // logical length is `#byteLength * 8 + (7 - #stopBitIndex)` readonly #byteLength: number; readonly #stopBitIndex: number; #zeroCount = 0; - #bytePosition = -1; - #bitPosition = -1; + + // logical position is `#bytePosition * 8 + (7 - #bitPosition)` + #bytePosition = 0; + #bitPosition = 7; #byte = 0; get byteLength() { @@ -258,8 +261,8 @@ export class NaluSodbBitReader { get ended() { return ( - this.#bytePosition === this.#byteLength && - this.#bitPosition === this.#stopBitIndex + this.#bytePosition >= this.#byteLength && + this.#bitPosition <= this.#stopBitIndex ); } @@ -276,7 +279,7 @@ export class NaluSodbBitReader { if (((byte >> j) & 1) === 1) { this.#byteLength = i; this.#stopBitIndex = j; - this.#readByte(); + this.#loadByte(); return; } } @@ -285,15 +288,20 @@ export class NaluSodbBitReader { throw new Error("Stop bit not found"); } - #readByte() { + #loadByte() { this.#byte = this.#nalu[this.#bytePosition]!; + + // If the current sequence is `0x000003`, skip to the next byte. + // `annexBSplitNalu` had validated the input, so don't need to check here. if (this.#zeroCount === 2 && this.#byte === 3) { this.#zeroCount = 0; this.#bytePosition += 1; - this.#readByte(); + this.#loadByte(); return; } + // `0x00000301` becomes `0x000001`, so only the `0x03` byte needs to be skipped + // The `0x00` bytes are still returned as-is if (this.#byte === 0) { this.#zeroCount += 1; } else { @@ -302,18 +310,19 @@ export class NaluSodbBitReader { } next() { - if (this.#bitPosition === -1) { - this.#bitPosition = 7; - this.#bytePosition += 1; - this.#readByte(); - } - if (this.ended) { throw new Error("Bit index out of bounds"); } const value = (this.#byte >> this.#bitPosition) & 1; + this.#bitPosition -= 1; + if (this.#bitPosition < 0) { + this.#bytePosition += 1; + this.#bitPosition = 7; + this.#loadByte(); + } + return value; } @@ -329,12 +338,38 @@ export class NaluSodbBitReader { return result; } - skip(length: number) { - for (let i = 0; i < length; i += 1) { - this.next(); + #ensurePositionValid() { + if ( + this.#bytePosition >= this.#byteLength && + this.#bitPosition < this.#stopBitIndex + ) { + throw new Error("Bit index out of bounds"); } } + skip(length: number) { + if (length <= this.#bitPosition + 1) { + this.#bitPosition -= length; + this.#ensurePositionValid(); + return; + } + + length -= this.#bitPosition + 1; + this.#bytePosition += 1; + this.#bitPosition = 7; + this.#loadByte(); + this.#ensurePositionValid(); + + for (; length >= 8; length -= 8) { + this.#bytePosition += 1; + this.#loadByte(); + this.#ensurePositionValid(); + } + + this.#bitPosition = 7 - length; + this.#ensurePositionValid(); + } + decodeExponentialGolombNumber(): number { let length = 0; while (this.next() === 0) { diff --git a/libraries/scrcpy/src/options/2_0.ts b/libraries/scrcpy/src/options/2_0.ts index 47b57a63..d1559065 100644 --- a/libraries/scrcpy/src/options/2_0.ts +++ b/libraries/scrcpy/src/options/2_0.ts @@ -80,7 +80,7 @@ export interface ScrcpyOptionsInit2_0 videoEncoder?: string | undefined; audio?: boolean; - audioCodec?: "opus" | "aac" | "raw"; + audioCodec?: "raw" | "opus" | "aac"; audioBitRate?: number; audioCodecOptions?: CodecOptions; audioEncoder?: string | undefined; @@ -90,7 +90,7 @@ export interface ScrcpyOptionsInit2_0 sendCodecMeta?: boolean; } -function omit( +export function omit( obj: T, keys: K[], ): Omit { diff --git a/libraries/scrcpy/src/options/2_3.ts b/libraries/scrcpy/src/options/2_3.ts new file mode 100644 index 00000000..6d6c2e5a --- /dev/null +++ b/libraries/scrcpy/src/options/2_3.ts @@ -0,0 +1,33 @@ +import { ScrcpyOptions1_21 } from "./1_21.js"; +import { omit } from "./2_0.js"; +import { ScrcpyOptions2_2, type ScrcpyOptionsInit2_2 } from "./2_2.js"; +import { ScrcpyOptionsBase } from "./types.js"; + +export interface ScrcpyOptionsInit2_3 + extends Omit { + audioCodec?: "raw" | "opus" | "aac" | "flac" | undefined; +} + +export class ScrcpyOptions2_3 extends ScrcpyOptionsBase< + ScrcpyOptionsInit2_3, + ScrcpyOptions2_2 +> { + static readonly DEFAULTS = { + ...ScrcpyOptions2_2.DEFAULTS, + } as const satisfies Required; + + override get defaults(): Required { + return ScrcpyOptions2_3.DEFAULTS; + } + + constructor(init: ScrcpyOptionsInit2_3) { + super(new ScrcpyOptions2_2(omit(init, ["audioCodec"])), { + ...ScrcpyOptions2_3.DEFAULTS, + ...init, + }); + } + + override serialize(): string[] { + return ScrcpyOptions1_21.serialize(this.value, this.defaults); + } +} diff --git a/libraries/scrcpy/src/options/codec.ts b/libraries/scrcpy/src/options/codec.ts index 0b4d50fa..02d07d56 100644 --- a/libraries/scrcpy/src/options/codec.ts +++ b/libraries/scrcpy/src/options/codec.ts @@ -33,6 +33,12 @@ export class ScrcpyAudioCodec implements ScrcpyOptionValue { "audio/aac", "mp4a.66", ); + static readonly FLAC = new ScrcpyAudioCodec( + "flac", + 0x66_6c_61_63, + "audio/flac", + "flac", + ); static readonly RAW = new ScrcpyAudioCodec( "raw", 0x00_72_61_77, diff --git a/libraries/scrcpy/src/options/index.ts b/libraries/scrcpy/src/options/index.ts index c710cd32..d7a64c7a 100644 --- a/libraries/scrcpy/src/options/index.ts +++ b/libraries/scrcpy/src/options/index.ts @@ -9,6 +9,7 @@ export * from "./1_25/index.js"; export * from "./2_0.js"; export * from "./2_1.js"; export * from "./2_2.js"; +export * from "./2_3.js"; export * from "./codec.js"; export * from "./latest.js"; export * from "./types.js";