diff --git a/CHANGELOG.md b/CHANGELOG.md index 2558074..1294b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [1.2.2] - 2024-01-?? + +### Added + +- archive: Support DEFLATE in Zipper where JS implementations support it in CompressionStream. +- io: Added a skip() method to BitStream to match ByteStream. + ## [1.2.1] - 2024-01-19 ### Added diff --git a/archive/compress.js b/archive/compress.js index 73ef1fe..1afe8c8 100644 --- a/archive/compress.js +++ b/archive/compress.js @@ -76,11 +76,24 @@ export class Zipper { */ constructor(options) { /** - * @type {ZipCompressionMethod} + * @type {CompressorOptions} * @private */ + this.zipOptions = options; this.zipCompressionMethod = options.zipCompressionMethod || ZipCompressionMethod.STORE; - if (this.zipCompressionMethod === ZipCompressionMethod.DEFLATE) throw `DEFLATE not supported.`; + if (!Object.values(ZipCompressionMethod).includes(this.zipCompressionMethod)) { + throw `Compression method ${this.zipCompressionMethod} not supported`; + } + + if (this.zipCompressionMethod === ZipCompressionMethod.DEFLATE) { + // As per https://developer.mozilla.org/en-US/docs/Web/API/CompressionStream, NodeJS only + // supports deflate-raw from 21.2.0+ (Nov 2023). https://nodejs.org/en/blog/release/v21.2.0. + try { + new CompressionStream('deflate-raw'); + } catch (err) { + throw `CompressionStream with deflate-raw not supported by JS runtime: ${err}`; + } + } /** * @type {CompressStatus} @@ -155,7 +168,7 @@ export class Zipper { }; this.compressState = CompressStatus.READY; - this.appendFiles(files, isLastFile); + this.port_.postMessage({ files, isLastFile, compressionMethod: this.zipCompressionMethod}); }); } @@ -170,4 +183,4 @@ export class Zipper { this.byteArray.set(oldArray); this.byteArray.set(newBytes, oldArray.byteLength); } -} \ No newline at end of file +} diff --git a/archive/zip.js b/archive/zip.js index 5be7038..cbcc46c 100644 --- a/archive/zip.js +++ b/archive/zip.js @@ -13,7 +13,7 @@ import { ByteBuffer } from '../io/bytebuffer.js'; import { CENTRAL_FILE_HEADER_SIG, CRC32_MAGIC_NUMBER, END_OF_CENTRAL_DIR_SIG, - LOCAL_FILE_HEADER_SIG } from './common.js'; + LOCAL_FILE_HEADER_SIG, ZipCompressionMethod } from './common.js'; /** @typedef {import('./common.js').FileInfo} FileInfo */ @@ -37,9 +37,10 @@ let hostPort; * @typedef CompressFilesMessage A message the client sends to the implementation. * @property {FileInfo[]} files A set of files to add to the zip file. * @property {boolean} isLastFile Indicates this is the last set of files to add to the zip file. + * @property {ZipCompressionMethod=} compressionMethod The compression method to use. Ignored except + * for the first message sent. */ -// TODO: Support DEFLATE. // TODO: Support options that can let client choose levels of compression/performance. /** @@ -54,6 +55,9 @@ let hostPort; * @property {number} byteOffset (4 bytes) */ +/** @type {ZipCompressionMethod} */ +let compressionMethod = ZipCompressionMethod.STORE; + /** @type {FileInfo[]} */ let filesCompressed = []; @@ -138,30 +142,39 @@ function dateToDosTime(jsDate) { /** * @param {FileInfo} file - * @returns {ByteBuffer} + * @returns {Promise} */ -function zipOneFile(file) { +async function zipOneFile(file) { + /** @type {Uint8Array} */ + let compressedBytes; + if (compressionMethod === ZipCompressionMethod.STORE) { + compressedBytes = file.fileData; + } else if (compressionMethod === ZipCompressionMethod.DEFLATE) { + const blob = new Blob([file.fileData.buffer]); + const compressedStream = blob.stream().pipeThrough(new CompressionStream('deflate-raw')); + compressedBytes = new Uint8Array(await new Response(compressedStream).arrayBuffer()); + } + // Zip Local File Header has 30 bytes and then the filename and extrafields. const fileHeaderSize = 30 + file.fileName.length; /** @type {ByteBuffer} */ - const buffer = new ByteBuffer(fileHeaderSize + file.fileData.byteLength); + const buffer = new ByteBuffer(fileHeaderSize + compressedBytes.byteLength); buffer.writeNumber(LOCAL_FILE_HEADER_SIG, 4); // Magic number. buffer.writeNumber(0x0A, 2); // Version. buffer.writeNumber(0, 2); // General Purpose Flags. - buffer.writeNumber(0, 2); // Compression Method. 0 = Store only. + buffer.writeNumber(compressionMethod, 2); // Compression Method. const jsDate = new Date(file.lastModTime); /** @type {CentralDirectoryFileHeaderInfo} */ const centralDirectoryInfo = { - compressionMethod: 0, + compressionMethod, lastModFileTime: dateToDosTime(jsDate), lastModFileDate: dateToDosDate(jsDate), crc32: calculateCRC32(0, file.fileData), - // TODO: For now, this is easy. Later when we do DEFLATE, we will have to calculate. - compressedSize: file.fileData.byteLength, + compressedSize: compressedBytes.byteLength, uncompressedSize: file.fileData.byteLength, fileName: file.fileName, byteOffset: numBytesWritten, @@ -176,7 +189,7 @@ function zipOneFile(file) { buffer.writeNumber(centralDirectoryInfo.fileName.length, 2); // Filename length. buffer.writeNumber(0, 2); // Extra field length. buffer.writeASCIIString(centralDirectoryInfo.fileName); // Filename. Assumes ASCII. - buffer.insertBytes(file.fileData); // File data. + buffer.insertBytes(compressedBytes); return buffer; } @@ -195,7 +208,7 @@ function writeCentralFileDirectory() { buffer.writeNumber(0, 2); // Version made by. // 0x31e buffer.writeNumber(0, 2); // Version needed to extract (minimum). // 0x14 buffer.writeNumber(0, 2); // General purpose bit flag - buffer.writeNumber(0, 2); // Compression method. + buffer.writeNumber(compressionMethod, 2); // Compression method. buffer.writeNumber(cdInfo.lastModFileTime, 2); // Last Mod File Time. buffer.writeNumber(cdInfo.lastModFileDate, 2); // Last Mod Date. buffer.writeNumber(cdInfo.crc32, 4); // crc32. @@ -228,7 +241,7 @@ function writeCentralFileDirectory() { * @param {{data: CompressFilesMessage}} evt The event for the implementation to process. It is an * error to send any more events after a previous event had isLastFile is set to true. */ -const onmessage = function(evt) { +const onmessage = async function(evt) { if (state === CompressorState.FINISHED) { throw `The zip implementation was sent a message after last file received.`; } @@ -239,11 +252,19 @@ const onmessage = function(evt) { state = CompressorState.COMPRESSING; + if (filesCompressed.length === 0 && evt.data.compressionMethod !== undefined) { + if (!Object.values(ZipCompressionMethod).includes(evt.data.compressionMethod)) { + throw `Do not support compression method ${evt.data.compressionMethod}`; + } + + compressionMethod = evt.data.compressionMethod; + } + const msg = evt.data; const filesToCompress = msg.files; while (filesToCompress.length > 0) { const fileInfo = filesToCompress.shift(); - const fileBuffer = zipOneFile(fileInfo); + const fileBuffer = await zipOneFile(fileInfo); filesCompressed.push(fileInfo); numBytesWritten += fileBuffer.data.byteLength; hostPort.postMessage({ type: 'compress', bytes: fileBuffer.data }, [ fileBuffer.data.buffer ]); diff --git a/tests/archive-compress.spec.js b/tests/archive-compress.spec.js index e6cbcd8..df37a67 100644 --- a/tests/archive-compress.spec.js +++ b/tests/archive-compress.spec.js @@ -36,7 +36,11 @@ describe('bitjs.archive.compress', () => { } }); - it('zipper works', (done) => { + it('zipper throws for invalid compression method', async () => { + expect(() => new Zipper({zipCompressionMethod: 42})).throws(); + }); + + it('zipper works for STORE', (done) => { const files = new Map(inputFileInfos); const zipper = new Zipper({zipCompressionMethod: ZipCompressionMethod.STORE}); zipper.start(Array.from(files.values()), true).then(byteArray => { @@ -57,4 +61,29 @@ describe('bitjs.archive.compress', () => { unarchiver.start(); }); }); + + it('zipper works for DEFLATE, where supported', async () => { + const files = new Map(inputFileInfos); + try { + const zipper = new Zipper({zipCompressionMethod: ZipCompressionMethod.DEFLATE}); + const byteArray = await zipper.start(Array.from(files.values()), true); + + expect(zipper.compressState).equals(CompressStatus.COMPLETE); + expect(byteArray.byteLength < decompressedFileSize).equals(true); + + const unarchiver = getUnarchiver(byteArray.buffer); + unarchiver.addEventListener('extract', evt => { + const {filename, fileData} = evt.unarchivedFile; + expect(files.has(filename)).equals(true); + const inputFile = files.get(filename).fileData; + expect(inputFile.byteLength).equals(fileData.byteLength); + for (let b = 0; b < inputFile.byteLength; ++b) { + expect(inputFile[b]).equals(fileData[b]); + } + }); + await unarchiver.start(); + } catch (err) { + // Do nothing. This runtime did not support DEFLATE. (Node < 21.2.0) + } + }); });