Add a package for detect audio/video codec information from ffprobe JSON output

README.md

@@ -96,6 +96,29 @@ const zippedArrayBuffer = await zipper.start(
   true /* isLastFile */);
 ```
 
+### bitjs.codecs
+
+This package includes code for dealing with media files (audio/video). It is useful for deriving
+ISO RFC6381 MIME type strings, including the codec information. Currently supports a limited subset
+of MP4 and WEBM.
+
+How to use:
+
+```javascript
+
+import { getFullMIMEString } from 'bitjs/codecs/codecs.js';
+/**
+ * @typedef {import('bitjs/codecs/codecs.js').ProbeInfo} ProbeInfo
+ */
+
+const cmd = 'ffprobe -show_format -show_streams -print_format json -v quiet foo.mp4';
+exec(cmd, (error, stdout) => {
+  /** @type {ProbeInfo} */
+  const info = JSON.parse(stdout);
+  // 'video/mp4; codecs="avc1.4D4028, mp4a.40.2"'
+  const contentType = getFullMIMEString(info);
+```
+
 ### bitjs.file
 
 This package includes code for dealing with files.  It includes a sniffer which detects the type of file, given an ArrayBuffer.

codecs/codecs.js

@@ -0,0 +1,217 @@
+/*
+ * codecs.js
+ *
+ * Licensed under the MIT License
+ *
+ * Copyright(c) 2022 Google Inc.
+ */
+
+/**
+ * This module helps interpret ffprobe -print_format json output.
+ * Its coverage is pretty sparse right now, so send me pull requests!
+ */
+
+/**
+ * @typdef ProbeStream ffprobe -show_streams -print_format json. Only the fields we care about.
+ * @property {number} index
+ * @property {string} codec_name
+ * @property {string} codec_long_name
+ * @property {string} profile
+ * @property {string} codec_type Either 'audio' or 'video'.
+ * @property {string} codec_tag_string
+ * @property {string} id
+ * @property {number?} level
+ * @property {number?} width
+ * @property {number?} height
+ * @property {string} r_frame_rate Like "60000/1001"
+ */
+
+/**
+ * @typedef ProbeFormat ffprobe -show_format -print_format json. Only the fields we care about.
+ * @property {string} filename
+ * @property {string} format_name
+ * @property {string} duration Number of seconds, as a string like "473.506367".
+ * @property {string} size Number of bytes, as a string.
+ * @property {string} bit_rate Bit rate, as a string.
+ */
+
+/**
+ * @typedef ProbeInfo ffprobe -show_format -show_streams -print_format json
+ * @property {ProbeStream[]} streams
+ * @property {ProbeFormat} format
+ */
+
+/**
+ * TODO: Reconcile this with file/sniffer.js findMimeType() which does signature matching.
+ * @param {ProbeInfo} info
+ * @returns {string}
+ */
+export function getShortMIMEString(info) {
+  if (!info) throw `Invalid ProbeInfo`;
+  if (!info.streams || info.streams.length === 0) throw `No streams in ProbeInfo`;
+
+  const type = info.streams.some(s => s.codec_type === 'video') ?
+      'video' :
+      info.streams.some(s => s.codec_type === 'audio') ? 'audio' : undefined;
+  if (!type) {
+    throw `Cannot handle media file type (no video/audio streams for ${info.format.format_name}). ` +
+        `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+  }
+
+  /** @type {string} */
+  let subType;
+  switch (info.format.format_name) {
+    case 'avi':
+      subType = 'x-msvideo';
+      break;
+    case 'mpeg':
+      subType = 'mpeg';
+      break;
+    case 'mov,mp4,m4a,3gp,3g2,mj2':
+      subType = 'mp4';
+      break;
+    case 'ogg':
+      subType = 'ogg';
+      break;
+    // Should we detect .mkv files as x-matroska?
+    case 'matroska,webm':
+      subType = 'webm';
+      break;
+    default:
+      throw `Cannot handle format ${info.format.format_name} yet. ` +
+          `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+  }
+
+  return `${type}/${subType}`;
+}
+
+/**
+ * Accepts the ffprobe JSON output and returns an ISO MIME string with parameters (RFC6381), such
+ * as 'video/mp4; codecs="avc1.4D4028, mp4a.40.2"'. This string should be suitable to be used on
+ * the server as the Content-Type header of a media stream which can subsequently be used on the
+ * client as the type value of a SourceBuffer object `mediaSource.addSourceBuffer(contentType)`.
+ * NOTE: For now, this method fails hard (throws an error) when it encounters a format/codec it
+ *     does not recognize. Please file a bug or send a PR.
+ * @param {ProbeInfo} info
+ * @returns {string}
+ */
+export function getFullMIMEString(info) {
+  /** A string like 'video/mp4' */
+  let contentType = `${getShortMIMEString(info)}`;
+  let codecFrags = new Set();
+
+  for (const stream of info.streams) {
+    if (stream.codec_type === 'audio') {
+      // TODO! At least mp4a!
+    }
+    else if (stream.codec_type === 'video') {
+      switch (stream.codec_tag_string) {
+        case 'avc1': codecFrags.add(getAVC1CodecString(stream)); break;
+        case 'vp09': codecFrags.add(getVP09CodecString(stream)); break;
+        default:
+          throw `Could not handle codec_tag_string ${stream.codec_tag_string} yet. ` +
+              `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+      }
+    }
+  }
+
+  if (codecFrags.length === 0) return contentType;
+
+  return contentType + '; codecs="' + Array.from(codecFrags).join(',') + '"';
+}
+
+// TODO: Consider whether any of these should be exported.
+
+/**
+ * https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#iso_base_media_file_format_mp4_quicktime_and_3gp
+ * @param {ProbeStream} stream
+ * @returns {string}
+ */
+function getAVC1CodecString(stream) {
+  if (!stream.profile) throw `No profile found in AVC1 stream`;
+
+  let frag = 'avc1';
+
+  // Add PP and CC hex digits.
+  switch (stream.profile) {
+    case 'Constrained Baseline':
+      frag += '.4240';
+      break;
+    case 'Baseline':
+      frag += '.4200';
+      break;
+    case 'Extended':
+      frag += '.5800';
+      break;
+    case 'Main':
+      frag += '.4D00';
+      break;
+    case 'High':
+      frag += '.6400';
+      break;
+    default:
+      throw `Cannot handle AVC1 stream with profile ${stream.profile} yet. ` +
+          `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+  }
+
+  // Add LL hex digits.
+  const levelAsHex = Number(stream.level).toString(16).toUpperCase().padStart(2, '0');
+  if (levelAsHex.length !== 2) {
+    throw `Cannot handle AVC1 level ${stream.level} yet. ` +
+        `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+  }
+  frag += levelAsHex;
+
+  return frag;
+}
+
+/**
+ * https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#webm
+ * @param {ProbeStream} stream
+ * @returns {string}
+ */
+function getVP09CodecString(stream) {
+  // TODO: Consider just returning 'vp9' here instead since I have so much guesswork.
+  // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter#webm
+
+  // The ISO format is cccc.PP.LL.DD
+  let frag = 'vp09';
+
+  // Add PP hex digits.
+  switch (stream.profile) {
+    case 'Profile 0':
+      frag += '.00';
+      break;
+    case 'Profile 1':
+      frag += '.01';
+      break;
+    case 'Profile 2':
+      frag += '.02';
+      break;
+    case 'Profile 3':
+      frag += '.03';
+      break;
+    default:
+      throw `Cannot handle VP09 stream with profile ${stream.profile} yet. ` +
+          `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+  }
+
+  // Add LL hex digits.
+  // TODO: ffprobe is spitting out -99 as level... I'm guessing on LL here.
+  if (stream.level === -99) { frag += '.FF'; }
+  else {
+    const levelAsHex = Number(stream.level).toString(16).toUpperCase().padStart(2, '0');
+    if (levelAsHex.length !== 2) {
+      throw `Cannot handle VP09 level ${stream.level} yet. ` + 
+          `Please file a bug https://github.com/codedread/bitjs/issues/new`;
+    }
+    frag += `.${levelAsHex}`;
+  }
+
+  // Add DD hex digits.
+  // TODO: This is just a guess at DD (16?), need to try and extract this info from
+  //     ffprobe JSON output instead.
+  frag += '.10';
+
+  return frag;
+}

tests/codecs.spec.js

@@ -0,0 +1,255 @@
+/*
+ * codecs.spec.js
+ *
+ * Licensed under the MIT License
+ *
+ * Copyright(c) 2022 Google Inc.
+ */
+
+import 'mocha';
+import { expect } from 'chai';
+import { getFullMIMEString, getShortMIMEString } from '../codecs/codecs.js';
+
+/**
+ * @typedef {import('../codecs/codecs.js').ProbeStream} ProbeStream
+ */
+/**
+ * @typedef {import('../codecs/codecs.js').ProbeFormat} ProbeFormat
+ */
+/**
+ * @typedef {import('../codecs/codecs.js').ProbeInfo} ProbeInfo
+ */
+
+describe('codecs test suite', () => {
+
+  describe('getShortMIMEString()', () => {
+    it('throws when unknown', () => {
+      expect(() => getShortMIMEString()).to.throw();
+      expect(() => getShortMIMEString(null)).to.throw();
+      expect(() => getShortMIMEString({})).to.throw();
+      expect(() => getShortMIMEString({
+        streams: [],
+      })).to.throw();
+      expect(() => getShortMIMEString({
+        format: { format_name: 'mp4' },
+        streams: [],
+      })).to.throw();
+      expect(() => getShortMIMEString({
+        format: { format_name: 'invalid-video-format' },
+        streams: [ { codec_type: 'video' } ],
+      })).to.throw();
+    });
+
+    it('detects AVI video', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'avi' },
+        streams: [ { codec_type: 'video' } ],
+      })).equals('video/x-msvideo');
+    });
+
+    it('detects MPEG video', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'mpeg' },
+        streams: [ { codec_type: 'video' } ],
+      })).equals('video/mpeg');
+    });
+
+    it('detects MPEG audio', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'mpeg' },
+        streams: [ { codec_type: 'audio' } ],
+      })).equals('audio/mpeg');
+    });
+
+    it('detects MP4 video', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' },
+        streams: [ { codec_type: 'video' } ],
+      })).equals('video/mp4');
+    });
+
+    it('detects MP4 audio', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' },
+        streams: [ { codec_type: 'audio' } ],
+      })).equals('audio/mp4');
+    });
+
+    it('detects OGG video', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'ogg' },
+        streams: [ { codec_type: 'video' } ],
+      })).equals('video/ogg');
+    });
+
+    it('detects OGG audio', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'ogg' },
+        streams: [ { codec_type: 'audio' } ],
+      })).equals('audio/ogg');
+    });
+
+    it('detects WEBM video', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'matroska,webm' },
+        streams: [ { codec_type: 'video' } ],
+      })).equals('video/webm');
+    });
+
+    it('detects WEBM audio', () => {
+      expect(getShortMIMEString({
+        format: { format_name: 'matroska,webm' },
+        streams: [ { codec_type: 'audio' } ],
+      })).equals('audio/webm');
+    });
+  });
+
+  describe('getFullMIMEString()', () => {
+    it('throws when unknown', () => {
+      expect(() => getFullMIMEString()).to.throw();
+      expect(() => getFullMIMEString(null)).to.throw();
+      expect(() => getFullMIMEString({})).to.throw();
+      expect(() => getFullMIMEString({
+        streams: [],
+      })).to.throw();
+      expect(() => getFullMIMEString({
+        format: { format_name: 'invalid-video-format' },
+        streams: [ { codec_type: 'video' } ],
+      })).to.throw();
+    });
+
+    describe('AVC1', () => {
+      /** @type {ProbeInfo} */
+      let info;
+
+      beforeEach(() => {
+        info = {
+          format: { format_name: 'mov,mp4,m4a,3gp,3g2,mj2' },
+          streams: [{
+            codec_type: 'video',
+            codec_tag_string: 'avc1',
+          }],
+        };
+      });
+
+      describe('Profile tests', () => {
+        beforeEach(() => {
+          info.streams[0].level = 20;
+        });
+
+        it('detects Constrained Baseline Profile', () => {
+          info.streams[0].profile = 'Constrained Baseline';
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4240'));
+        });
+
+        it('detects Baseline Profile', () => {
+          info.streams[0].profile = 'Baseline';
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4200'));
+        });
+
+        it('detects Extended Profile', () => {
+          info.streams[0].profile = 'Extended';
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.5800'));
+        });
+
+        it('detects Main Profile', () => {
+          info.streams[0].profile = 'Main';
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.4D00'));
+        });
+
+        it('detects High Profile', () => {
+          info.streams[0].profile = 'High';
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.6400'));
+        });
+      });
+
+      describe('Level tests', () => {
+        beforeEach(() => {
+          info.streams[0].profile = 'Main';
+        });
+
+        it('detects 2-digit hex level', () => {
+          info.streams[0].level = 21; // 0x15
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.'))
+              .and.satisfy(s => s.endsWith('15"'));
+        });
+
+        it('detects 1-digit hex level', () => {
+          info.streams[0].level = 10; // 0x0A
+          expect(getFullMIMEString(info))
+              .to.be.a('string')
+              .and.satisfy(s => s.startsWith('video/mp4; codecs="avc1.'))
+              .and.satisfy(s => s.endsWith('0A"'));
+        });
+      });
+    });
+  });
+
+  describe('VP09', () => {
+    /** @type {ProbeInfo} */
+    let info;
+
+    beforeEach(() => {
+      info = {
+        format: { format_name: 'matroska,webm' },
+        streams: [{
+          codec_type: 'video',
+          codec_tag_string: 'vp09',
+        }],
+      };
+    });
+
+    describe('Profile tests', () => {
+      beforeEach(() => {
+        info.streams[0].level = 20;
+      });
+
+      it('detects Profile 0', () => {
+        info.streams[0].profile = 'Profile 0';
+        expect(getFullMIMEString(info))
+            .to.be.a('string')
+            .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.00.'));
+      });
+    });
+
+    describe('Level tests', () => {
+      beforeEach(() => {
+        info.streams[0].profile = 'Profile 0';
+      });
+
+      it('detects 2-digit hex level', () => {
+        info.streams[0].level = 21; // 0x15
+        expect(getFullMIMEString(info))
+            .to.be.a('string')
+            .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.'))
+            .and.satisfy(s => {
+              const matches = s.match(/vp09\.[0-9]{2}\.([0-9A-F]{2})\.[0-9A-F]{2}/);
+              return matches && matches.length === 2 && matches[1] === '15';
+            });
+        });
+
+      it('detects level = -99', () => {
+        info.streams[0].level = -99; // I'm not sure what ffprobe means by this.
+        expect(getFullMIMEString(info))
+            .to.be.a('string')
+            .and.satisfy(s => s.startsWith('video/webm; codecs="vp09.'))
+            .and.satisfy(s => {
+              const matches = s.match(/vp09\.[0-9]{2}\.([0-9A-F]{2})\.[0-9A-F]{2}/);
+              return matches && matches.length === 2 && matches[1] === 'FF';
+            });
+      });
+    });
+  });
+});