diff --git a/README.md b/README.md index bf61067..5441028 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,29 @@ const zippedArrayBuffer = await zipper.start( true /* isLastFile */); ``` +### bitjs.codecs + +This package includes code for dealing with media files (audio/video). It is useful for deriving +ISO RFC6381 MIME type strings, including the codec information. Currently supports a limited subset +of MP4 and WEBM. + +How to use: + +```javascript + +import { getFullMIMEString } from 'bitjs/codecs/codecs.js'; +/** + * @typedef {import('bitjs/codecs/codecs.js').ProbeInfo} ProbeInfo + */ + +const cmd = 'ffprobe -show_format -show_streams -print_format json -v quiet foo.mp4'; +exec(cmd, (error, stdout) => { + /** @type {ProbeInfo} */ + const info = JSON.parse(stdout); + // 'video/mp4; codecs="avc1.4D4028, mp4a.40.2"' + const contentType = getFullMIMEString(info); +``` + ### bitjs.file This package includes code for dealing with files. It includes a sniffer which detects the type of file, given an ArrayBuffer. diff --git a/codecs/codecs.js b/codecs/codecs.js new file mode 100644 index 0000000..7c8a1d4 --- /dev/null +++ b/codecs/codecs.js @@ -0,0 +1,217 @@ +/* + * codecs.js + * + * Licensed under the MIT License + * + * Copyright(c) 2022 Google Inc. + */ + +/** + * This module helps interpret ffprobe -print_format json output. + * Its coverage is pretty sparse right now, so send me pull requests! + */ + +/** + * @typdef ProbeStream ffprobe -show_streams -print_format json. Only the fields we care about. + * @property {number} index + * @property {string} codec_name + * @property {string} codec_long_name + * @property {string} profile + * @property {string} codec_type Either 'audio' or 'video'. + * @property {string} codec_tag_string + * @property {string} id + * @property {number?} level + * @property {number?} width + * @property {number?} height + * @property {string} r_frame_rate Like "60000/1001" + */ + +/** + * @typedef ProbeFormat ffprobe -show_format -print_format json. Only the fields we care about. + * @property {string} filename + * @property {string} format_name + * @property {string} duration Number of seconds, as a string like "473.506367". + * @property {string} size Number of bytes, as a string. + * @property {string} bit_rate Bit rate, as a string. + */ + +/** + * @typedef ProbeInfo ffprobe -show_format -show_streams -print_format json + * @property {ProbeStream[]} streams + * @property {ProbeFormat} format + */ + +/** + * TODO: Reconcile this with file/sniffer.js findMimeType() which does signature matching. + * @param {ProbeInfo} info + * @returns {string} + */ +export function getShortMIMEString(info) { + if (!info) throw `Invalid ProbeInfo`; + if (!info.streams || info.streams.length === 0) throw `No streams in ProbeInfo`; + + const type = info.streams.some(s => s.codec_type === 'video') ? + 'video' : + info.streams.some(s => s.codec_type === 'audio') ? 'audio' : undefined; + if (!type) { + throw `Cannot handle media file type (no video/audio streams for ${info.format.format_name}). ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + + /** @type {string} */ + let subType; + switch (info.format.format_name) { + case 'avi': + subType = 'x-msvideo'; + break; + case 'mpeg': + subType = 'mpeg'; + break; + case 'mov,mp4,m4a,3gp,3g2,mj2': + subType = 'mp4'; + break; + case 'ogg': + subType = 'ogg'; + break; + // Should we detect .mkv files as x-matroska? + case 'matroska,webm': + subType = 'webm'; + break; + default: + throw `Cannot handle format ${info.format.format_name} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + + return `${type}/${subType}`; +} + +/** + * Accepts the ffprobe JSON output and returns an ISO MIME string with parameters (RFC6381), such + * as 'video/mp4; codecs="avc1.4D4028, mp4a.40.2"'. This string should be suitable to be used on + * the server as the Content-Type header of a media stream which can subsequently be used on the + * client as the type value of a SourceBuffer object `mediaSource.addSourceBuffer(contentType)`. + * NOTE: For now, this method fails hard (throws an error) when it encounters a format/codec it + * does not recognize. Please file a bug or send a PR. + * @param {ProbeInfo} info + * @returns {string} + */ +export function getFullMIMEString(info) { + /** A string like 'video/mp4' */ + let contentType = `${getShortMIMEString(info)}`; + let codecFrags = new Set(); + + for (const stream of info.streams) { + if (stream.codec_type === 'audio') { + // TODO! At least mp4a! + } + else if (stream.codec_type === 'video') { + switch (stream.codec_tag_string) { + case 'avc1': codecFrags.add(getAVC1CodecString(stream)); break; + case 'vp09': codecFrags.add(getVP09CodecString(stream)); break; + default: + throw `Could not handle codec_tag_string ${stream.codec_tag_string} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + } + } + + if (codecFrags.length === 0) return contentType; + + return contentType + '; codecs="' + Array.from(codecFrags).join(',') + '"'; +} + +// TODO: Consider whether any of these should be exported. + +/** + * https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#iso_base_media_file_format_mp4_quicktime_and_3gp + * @param {ProbeStream} stream + * @returns {string} + */ +function getAVC1CodecString(stream) { + if (!stream.profile) throw `No profile found in AVC1 stream`; + + let frag = 'avc1'; + + // Add PP and CC hex digits. + switch (stream.profile) { + case 'Constrained Baseline': + frag += '.4240'; + break; + case 'Baseline': + frag += '.4200'; + break; + case 'Extended': + frag += '.5800'; + break; + case 'Main': + frag += '.4D00'; + break; + case 'High': + frag += '.6400'; + break; + default: + throw `Cannot handle AVC1 stream with profile ${stream.profile} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + + // Add LL hex digits. + const levelAsHex = Number(stream.level).toString(16).toUpperCase().padStart(2, '0'); + if (levelAsHex.length !== 2) { + throw `Cannot handle AVC1 level ${stream.level} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + frag += levelAsHex; + + return frag; +} + +/** + * https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#webm + * @param {ProbeStream} stream + * @returns {string} + */ +function getVP09CodecString(stream) { + // TODO: Consider just returning 'vp9' here instead since I have so much guesswork. + // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#webm + + // The ISO format is cccc.PP.LL.DD + let frag = 'vp09'; + + // Add PP hex digits. + switch (stream.profile) { + case 'Profile 0': + frag += '.00'; + break; + case 'Profile 1': + frag += '.01'; + break; + case 'Profile 2': + frag += '.02'; + break; + case 'Profile 3': + frag += '.03'; + break; + default: + throw `Cannot handle VP09 stream with profile ${stream.profile} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + + // Add LL hex digits. + // TODO: ffprobe is spitting out -99 as level... I'm guessing on LL here. + if (stream.level === -99) { frag += '.FF'; } + else { + const levelAsHex = Number(stream.level).toString(16).toUpperCase().padStart(2, '0'); + if (levelAsHex.length !== 2) { + throw `Cannot handle VP09 level ${stream.level} yet. ` + + `Please file a bug https://github.com/codedread/bitjs/issues/new`; + } + frag += `.${levelAsHex}`; + } + + // Add DD hex digits. + // TODO: This is just a guess at DD (16?), need to try and extract this info from + // ffprobe JSON output instead. + frag += '.10'; + + return frag; +} diff --git a/tests/codecs.spec.js b/tests/codecs.spec.js new file mode 100644 index 0000000..42b7cf7 --- /dev/null +++ b/tests/codecs.spec.js @@ -0,0 +1,255 @@ +/* + * codecs.spec.js + * + * Licensed under the MIT License + * + * Copyright(c) 2022 Google Inc. + */ + +import 'mocha'; +import { expect } from 'chai'; +import { getFullMIMEString, getShortMIMEString } from '../codecs/codecs.js'; + +/** + * @typedef {import('../codecs/codecs.js').ProbeStream} ProbeStream + */ +/** + * @typedef {import('../codecs/codecs.js').ProbeFormat} ProbeFormat + */ +/** + * @typedef {import('../codecs/codecs.js').ProbeInfo} ProbeInfo + */ + +describe('codecs test suite', () => { + + describe('getShortMIMEString()', () => { + it('throws when unknown', () => { + expect(() => getShortMIMEString()).to.throw(); + expect(() => getShortMIMEString(null)).to.throw(); + expect(() => getShortMIMEString({})).to.throw(); + expect(() => getShortMIMEString({ + streams: [], + })).to.throw(); + expect(() => getShortMIMEString({ + format: { format_name: 'mp4' }, + streams: [], + })).to.throw(); + expect(() => getShortMIMEString({ + format: { format_name: 'invalid-video-format' }, + streams: [ { codec_type: 'video' } ], + })).to.throw(); + }); + + it('detects AVI video', () => { + expect(getShortMIMEString({ + format: { format_name: 'avi' }, + streams: [ { codec_type: 'video' } ], + })).equals('video/x-msvideo'); + }); + + it('detects MPEG video', () => { + expect(getShortMIMEString({ + format: { format_name: 'mpeg' }, + streams: [ { codec_type: 'video' } ], + })).equals('video/mpeg'); + }); + + it('detects MPEG audio', () => { + expect(getShortMIMEString({ + format: { format_name: 'mpeg' }, + streams: [ { codec_type: 'audio' } ], + })).equals('audio/mpeg'); + }); + + it('detects MP4 video', () => { + expect(getShortMIMEString({ + format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' }, + streams: [ { codec_type: 'video' } ], + })).equals('video/mp4'); + }); + + it('detects MP4 audio', () => { + expect(getShortMIMEString({ + format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' }, + streams: [ { codec_type: 'audio' } ], + })).equals('audio/mp4'); + }); + + it('detects OGG video', () => { + expect(getShortMIMEString({ + format: { format_name: 'ogg' }, + streams: [ { codec_type: 'video' } ], + })).equals('video/ogg'); + }); + + it('detects OGG audio', () => { + expect(getShortMIMEString({ + format: { format_name: 'ogg' }, + streams: [ { codec_type: 'audio' } ], + })).equals('audio/ogg'); + }); + + it('detects WEBM video', () => { + expect(getShortMIMEString({ + format: { format_name: 'matroska,webm' }, + streams: [ { codec_type: 'video' } ], + })).equals('video/webm'); + }); + + it('detects WEBM audio', () => { + expect(getShortMIMEString({ + format: { format_name: 'matroska,webm' }, + streams: [ { codec_type: 'audio' } ], + })).equals('audio/webm'); + }); + }); + + describe('getFullMIMEString()', () => { + it('throws when unknown', () => { + expect(() => getFullMIMEString()).to.throw(); + expect(() => getFullMIMEString(null)).to.throw(); + expect(() => getFullMIMEString({})).to.throw(); + expect(() => getFullMIMEString({ + streams: [], + })).to.throw(); + expect(() => getFullMIMEString({ + format: { format_name: 'invalid-video-format' }, + streams: [ { codec_type: 'video' } ], + })).to.throw(); + }); + + describe('AVC1', () => { + /** @type {ProbeInfo} */ + let info; + + beforeEach(() => { + info = { + format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' }, + streams: [{ + codec_type: 'video', + codec_tag_string: 'avc1', + }], + }; + }); + + describe('Profile tests', () => { + beforeEach(() => { + info.streams[0].level = 20; + }); + + it('detects Constrained Baseline Profile', () => { + info.streams[0].profile = 'Constrained Baseline'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4240')); + }); + + it('detects Baseline Profile', () => { + info.streams[0].profile = 'Baseline'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4200')); + }); + + it('detects Extended Profile', () => { + info.streams[0].profile = 'Extended'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.5800')); + }); + + it('detects Main Profile', () => { + info.streams[0].profile = 'Main'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4D00')); + }); + + it('detects High Profile', () => { + info.streams[0].profile = 'High'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.6400')); + }); + }); + + describe('Level tests', () => { + beforeEach(() => { + info.streams[0].profile = 'Main'; + }); + + it('detects 2-digit hex level', () => { + info.streams[0].level = 21; // 0x15 + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.')) + .and.satisfy(s => s.endsWith('15"')); + }); + + it('detects 1-digit hex level', () => { + info.streams[0].level = 10; // 0x0A + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.')) + .and.satisfy(s => s.endsWith('0A"')); + }); + }); + }); + }); + + describe('VP09', () => { + /** @type {ProbeInfo} */ + let info; + + beforeEach(() => { + info = { + format: { format_name: 'matroska,webm' }, + streams: [{ + codec_type: 'video', + codec_tag_string: 'vp09', + }], + }; + }); + + describe('Profile tests', () => { + beforeEach(() => { + info.streams[0].level = 20; + }); + + it('detects Profile 0', () => { + info.streams[0].profile = 'Profile 0'; + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.00.')); + }); + }); + + describe('Level tests', () => { + beforeEach(() => { + info.streams[0].profile = 'Profile 0'; + }); + + it('detects 2-digit hex level', () => { + info.streams[0].level = 21; // 0x15 + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.')) + .and.satisfy(s => { + const matches = s.match(/vp09\.[0-9]{2}\.([0-9A-F]{2})\.[0-9A-F]{2}/); + return matches && matches.length === 2 && matches[1] === '15'; + }); + }); + + it('detects level = -99', () => { + info.streams[0].level = -99; // I'm not sure what ffprobe means by this. + expect(getFullMIMEString(info)) + .to.be.a('string') + .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.')) + .and.satisfy(s => { + const matches = s.match(/vp09\.[0-9]{2}\.([0-9A-F]{2})\.[0-9A-F]{2}/); + return matches && matches.length === 2 && matches[1] === 'FF'; + }); + }); + }); + }); +});