1
0
Fork 0
mirror of https://github.com/codedread/bitjs synced 2025-10-03 09:39:16 +02:00

Move actual archive functionality into archive-internal.js so we can eventually unit test it in nodejs.

This commit is contained in:
Jeff Schiller 2021-06-20 10:19:43 -07:00
parent 7f5cba6150
commit a5e3d2f0b5
2 changed files with 439 additions and 371 deletions

400
archive/archive-internal.js Normal file
View file

@ -0,0 +1,400 @@
/**
* archive-internal.js
*
* Provides base functionality for unarchiving, extracted here as an internal
* module for unit testing. Import archive.js instead.
*
* Licensed under the MIT License
*
* Copyright(c) 2021 Google Inc.
*/
import { findMimeType } from '../file/sniffer.js';
/**
* The UnarchiveEvent types.
*/
export const UnarchiveEventType = {
START: 'start',
PROGRESS: 'progress',
EXTRACT: 'extract',
FINISH: 'finish',
INFO: 'info',
ERROR: 'error'
};
/**
* An unarchive event.
*/
export class UnarchiveEvent {
/**
* @param {string} type The event type.
*/
constructor(type) {
/**
* The event type.
* @type {string}
*/
this.type = type;
}
}
/**
* Useful for passing info up to the client (for debugging).
*/
export class UnarchiveInfoEvent extends UnarchiveEvent {
/**
* @param {string} msg The info message.
*/
constructor(msg) {
super(UnarchiveEventType.INFO);
/**
* The information message.
* @type {string}
*/
this.msg = msg;
}
}
/**
* An unrecoverable error has occured.
*/
export class UnarchiveErrorEvent extends UnarchiveEvent {
/**
* @param {string} msg The error message.
*/
constructor(msg) {
super(UnarchiveEventType.ERROR);
/**
* The information message.
* @type {string}
*/
this.msg = msg;
}
}
/**
* Start event.
*/
export class UnarchiveStartEvent extends UnarchiveEvent {
constructor() {
super(UnarchiveEventType.START);
}
}
/**
* Finish event.
*/
export class UnarchiveFinishEvent extends UnarchiveEvent {
/**
* @param {Object} metadata A collection fo metadata about the archive file.
*/
constructor(metadata = {}) {
super(UnarchiveEventType.FINISH);
this.metadata = metadata;
}
}
/**
* Progress event.
*/
export class UnarchiveProgressEvent extends UnarchiveEvent {
/**
* @param {string} currentFilename
* @param {number} currentFileNumber
* @param {number} currentBytesUnarchivedInFile
* @param {number} currentBytesUnarchived
* @param {number} totalUncompressedBytesInArchive
* @param {number} totalFilesInArchive
* @param {number} totalCompressedBytesRead
*/
constructor(currentFilename, currentFileNumber, currentBytesUnarchivedInFile,
currentBytesUnarchived, totalUncompressedBytesInArchive, totalFilesInArchive,
totalCompressedBytesRead) {
super(UnarchiveEventType.PROGRESS);
this.currentFilename = currentFilename;
this.currentFileNumber = currentFileNumber;
this.currentBytesUnarchivedInFile = currentBytesUnarchivedInFile;
this.totalFilesInArchive = totalFilesInArchive;
this.currentBytesUnarchived = currentBytesUnarchived;
this.totalUncompressedBytesInArchive = totalUncompressedBytesInArchive;
this.totalCompressedBytesRead = totalCompressedBytesRead;
}
}
/**
* Extract event.
*/
export class UnarchiveExtractEvent extends UnarchiveEvent {
/**
* @param {UnarchivedFile} unarchivedFile
*/
constructor(unarchivedFile) {
super(UnarchiveEventType.EXTRACT);
/**
* @type {UnarchivedFile}
*/
this.unarchivedFile = unarchivedFile;
}
}
/**
* Base class for all Unarchivers.
* TODO: When EventTarget constructors are broadly supported, make this extend
* EventTarget and remove event listener code.
* https://caniuse.com/#feat=mdn-api_eventtarget_eventtarget
*/
export class Unarchiver {
/**
* @param {ArrayBuffer} arrayBuffer The Array Buffer.
* @param {Function(string):Worker} createWorkerFn A function that creates a Worker from a script file.
* @param {Object|string} options An optional object of options, or a string representing where
* the BitJS files are located. The string version of this argument is deprecated.
* Available options:
* 'pathToBitJS': A string indicating where the BitJS files are located.
* 'debug': A boolean where true indicates that the archivers should log debug output.
*/
constructor(arrayBuffer, createWorkerFn, options = {}) {
if (typeof options === 'string') {
console.warn(`Deprecated: Don't send a raw string to Unarchiver()`);
console.warn(` send {'pathToBitJS':'${options}'} instead`);
options = { 'pathToBitJS': options };
}
/**
* The ArrayBuffer object.
* @type {ArrayBuffer}
* @protected
*/
this.ab = arrayBuffer;
/**
* A factory method that creates a Worker that does the unarchive work.
* @type {Function(string): Worker}
* @private
*/
this.createWorkerFn_ = createWorkerFn;
/**
* The path to the BitJS files.
* @type {string}
* @private
*/
this.pathToBitJS_ = options.pathToBitJS || '/';
/** @orivate {boolean} */
this.debugMode_ = !!(options.debug);
/**
* A map from event type to an array of listeners.
* @type {Map.<string, Array>}
*/
this.listeners_ = {};
for (let type in UnarchiveEventType) {
this.listeners_[UnarchiveEventType[type]] = [];
}
/**
* Private web worker initialized during start().
* @type {Worker}
* @private
*/
this.worker_ = null;
}
/**
* This method must be overridden by the subclass to return the script filename.
* @return {string} The script filename.
* @protected.
*/
getScriptFileName() {
throw 'Subclasses of AbstractUnarchiver must overload getScriptFileName()';
}
/**
* Adds an event listener for UnarchiveEvents.
*
* @param {string} Event type.
* @param {function} An event handler function.
*/
addEventListener(type, listener) {
if (type in this.listeners_) {
if (this.listeners_[type].indexOf(listener) == -1) {
this.listeners_[type].push(listener);
}
}
}
/**
* Removes an event listener.
*
* @param {string} Event type.
* @param {EventListener|function} An event listener or handler function.
*/
removeEventListener(type, listener) {
if (type in this.listeners_) {
const index = this.listeners_[type].indexOf(listener);
if (index != -1) {
this.listeners_[type].splice(index, 1);
}
}
}
/**
* Create an UnarchiveEvent out of the object sent back from the Worker.
* @param {Object} obj
* @return {UnarchiveEvent}
* @private
*/
createUnarchiveEvent_(obj) {
switch (obj.type) {
case UnarchiveEventType.START:
return new UnarchiveStartEvent();
case UnarchiveEventType.PROGRESS:
return new UnarchiveProgressEvent(
obj.currentFilename,
obj.currentFileNumber,
obj.currentBytesUnarchivedInFile,
obj.currentBytesUnarchived,
obj.totalUncompressedBytesInArchive,
obj.totalFilesInArchive,
obj.totalCompressedBytesRead);
case UnarchiveEventType.EXTRACT:
return new UnarchiveExtractEvent(obj.unarchivedFile);
case UnarchiveEventType.FINISH:
return new UnarchiveFinishEvent(obj.metadata);
case UnarchiveEventType.INFO:
return new UnarchiveInfoEvent(obj.msg);
case UnarchiveEventType.ERROR:
return new UnarchiveErrorEvent(obj.msg);
}
}
/**
* Receive an event and pass it to the listener functions.
*
* @param {Object} obj
* @private
*/
handleWorkerEvent_(obj) {
const type = obj.type;
if (type && Object.values(UnarchiveEventType).includes(type) &&
this.listeners_[obj.type] instanceof Array) {
const evt = this.createUnarchiveEvent_(obj);
this.listeners_[evt.type].forEach(function (listener) { listener(evt) });
if (evt.type == UnarchiveEventType.FINISH) {
this.worker_.terminate();
}
} else {
console.log(`Unknown object received from worker: ${obj}`);
}
}
/**
* Starts the unarchive in a separate Web Worker thread and returns immediately.
*/
start() {
const me = this;
const scriptFileName = this.pathToBitJS_ + this.getScriptFileName();
if (scriptFileName) {
this.worker_ = this.createWorkerFn_(scriptFileName);
this.worker_.onerror = function (e) {
console.log('Worker error: message = ' + e.message);
throw e;
};
this.worker_.onmessage = function (e) {
if (typeof e.data == 'string') {
// Just log any strings the workers pump our way.
console.log(e.data);
} else {
me.handleWorkerEvent_(e.data);
}
};
const ab = this.ab;
this.worker_.postMessage({
file: ab,
logToConsole: this.debugMode_,
});
this.ab = null;
}
}
// TODO: Create a startSync() method that does not use a worker for Node.
/**
* Adds more bytes to the unarchiver's Worker thread.
* @param {ArrayBuffer} ab The ArrayBuffer with more bytes in it.
*/
update(ab) {
if (this.worker_) {
this.worker_.postMessage({ bytes: ab });
}
}
/**
* Terminates the Web Worker for this Unarchiver and returns immediately.
*/
stop() {
if (this.worker_) {
this.worker_.terminate();
}
}
}
export class UnzipperInternal extends Unarchiver {
constructor(arrayBuffer, createWorkerFn, options) {
super(arrayBuffer, createWorkerFn, options);
}
getScriptFileName() { return 'archive/unzip.js'; }
}
export class UnrarrerInternal extends Unarchiver {
constructor(arrayBuffer, createWorkerFn, options) {
super(arrayBuffer, createWorkerFn, options);
}
getScriptFileName() { return 'archive/unrar.js'; }
}
export class UntarrerInternal extends Unarchiver {
constructor(arrayBuffer, createWorkerFn, options) {
super(arrayBuffer, createWorkerFn, options);
}
getScriptFileName() { return 'archive/untar.js'; };
}
/**
* Factory method that creates an unarchiver based on the byte signature found
* in the arrayBuffer.
* @param {ArrayBuffer} ab
* @param {Function(string):Worker} createWorkerFn A function that creates a Worker from a script file.
* @param {Object|string} options An optional object of options, or a string representing where
* the path to the unarchiver script files.
* @return {Unarchiver}
*/
export function getUnarchiverInternal(ab, createWorkerFn, options = {}) {
if (ab.byteLength < 10) {
return null;
}
let unarchiver = null;
const mimeType = findMimeType(ab);
if (mimeType === 'application/x-rar-compressed') { // Rar!
unarchiver = new UnrarrerInternal(ab, createWorkerFn, options);
} else if (mimeType === 'application/zip') { // PK (Zip)
unarchiver = new UnzipperInternal(ab, createWorkerFn, options);
} else { // Try with tar
unarchiver = new UntarrerInternal(ab, createWorkerFn, options);
}
return unarchiver;
}

View file

@ -8,138 +8,23 @@
* Copyright(c) 2011 Google Inc. * Copyright(c) 2011 Google Inc.
*/ */
import { findMimeType } from '../file/sniffer.js'; import { UnarchiveErrorEvent, UnarchiveEvent, UnarchiveEventType,
UnarchiveExtractEvent, UnarchiveFinishEvent, UnarchiveInfoEvent,
UnarchiveProgressEvent, UnarchiveStartEvent, Unarchiver,
UnrarrerInternal, UntarrerInternal, UnzipperInternal,
getUnarchiverInternal } from './archive-internal.js';
/** export {
* An unarchive event. UnarchiveErrorEvent,
*/ UnarchiveEvent,
export class UnarchiveEvent { UnarchiveEventType,
/** UnarchiveExtractEvent,
* @param {string} type The event type. UnarchiveFinishEvent,
*/ UnarchiveInfoEvent,
constructor(type) { UnarchiveProgressEvent,
/** UnarchiveStartEvent,
* The event type. Unarchiver,
* @type {string} }
*/
this.type = type;
}
}
/**
* The UnarchiveEvent types.
*/
export const UnarchiveEventType = {
START: 'start',
PROGRESS: 'progress',
EXTRACT: 'extract',
FINISH: 'finish',
INFO: 'info',
ERROR: 'error'
};
/**
* Useful for passing info up to the client (for debugging).
*/
export class UnarchiveInfoEvent extends UnarchiveEvent {
/**
* @param {string} msg The info message.
*/
constructor(msg) {
super(UnarchiveEventType.INFO);
/**
* The information message.
* @type {string}
*/
this.msg = msg;
}
}
/**
* An unrecoverable error has occured.
*/
export class UnarchiveErrorEvent extends UnarchiveEvent {
/**
* @param {string} msg The error message.
*/
constructor(msg) {
super(UnarchiveEventType.ERROR);
/**
* The information message.
* @type {string}
*/
this.msg = msg;
}
}
/**
* Start event.
*/
export class UnarchiveStartEvent extends UnarchiveEvent {
constructor() {
super(UnarchiveEventType.START);
}
}
/**
* Finish event.
*/
export class UnarchiveFinishEvent extends UnarchiveEvent {
/**
* @param {Object} metadata A collection fo metadata about the archive file.
*/
constructor(metadata = {}) {
super(UnarchiveEventType.FINISH);
this.metadata = metadata;
}
}
/**
* Progress event.
*/
export class UnarchiveProgressEvent extends UnarchiveEvent {
/**
* @param {string} currentFilename
* @param {number} currentFileNumber
* @param {number} currentBytesUnarchivedInFile
* @param {number} currentBytesUnarchived
* @param {number} totalUncompressedBytesInArchive
* @param {number} totalFilesInArchive
* @param {number} totalCompressedBytesRead
*/
constructor(currentFilename, currentFileNumber, currentBytesUnarchivedInFile,
currentBytesUnarchived, totalUncompressedBytesInArchive, totalFilesInArchive,
totalCompressedBytesRead) {
super(UnarchiveEventType.PROGRESS);
this.currentFilename = currentFilename;
this.currentFileNumber = currentFileNumber;
this.currentBytesUnarchivedInFile = currentBytesUnarchivedInFile;
this.totalFilesInArchive = totalFilesInArchive;
this.currentBytesUnarchived = currentBytesUnarchived;
this.totalUncompressedBytesInArchive = totalUncompressedBytesInArchive;
this.totalCompressedBytesRead = totalCompressedBytesRead;
}
}
/**
* Extract event.
*/
export class UnarchiveExtractEvent extends UnarchiveEvent {
/**
* @param {UnarchivedFile} unarchivedFile
*/
constructor(unarchivedFile) {
super(UnarchiveEventType.EXTRACT);
/**
* @type {UnarchivedFile}
*/
this.unarchivedFile = unarchivedFile;
}
}
/** /**
* All extracted files returned by an Unarchiver will implement * All extracted files returned by an Unarchiver will implement
@ -153,260 +38,43 @@ export class UnarchiveExtractEvent extends UnarchiveEvent {
*/ */
/** /**
* Base class for all Unarchivers. * The goal is to make this testable - send getUnarchiver() an array buffer of
* TODO: When EventTarget constructors are broadly supported, make this extend * an archive, call start on the unarchiver, expect the returned result.
* EventTarget and remove event listener code. *
* https://caniuse.com/#feat=mdn-api_eventtarget_eventtarget * Problem: It relies on Web Workers, and that won't work in a nodejs context.
* Solution: Make archive.js very thin, have it feed web-specific things into
* an internal module that is isomorphic JavaScript.
*
* TODO:
* - write unit tests for archive-internal.js that use the nodejs Worker
* equivalent.
* - maybe use @pgriess/node-webworker or @audreyt/node-webworker-threads or
* just node's worker_threads ?
*/ */
export class Unarchiver {
/**
* @param {ArrayBuffer} arrayBuffer The Array Buffer.
* @param {Object|string} options An optional object of options, or a string representing where
* the BitJS files are located. Available options:
* 'pathToBitJS': A string indicating where the BitJS files are located.
* 'debug': A boolean where true indicates that the archivers should log debug output.
* @param {string} opt_pathToBitJS Optional string for where the BitJS files are located.
*/
constructor(arrayBuffer, options = {}) {
if (typeof options === 'string') {
options = { 'pathToBitJS': options };
}
/** const createWorkerFn = (scriptFilename) => new Worker(scriptFilename);
* The ArrayBuffer object.
* @type {ArrayBuffer}
* @protected
*/
this.ab = arrayBuffer;
/** // Thin wrappers of unarchivers for clients who want to construct a specific
* The path to the BitJS files. // unarchiver themselves rather than use getUnarchiver().
* @type {string} export class Unzipper extends UnzipperInternal {
* @private constructor(ab, options) { super(ab, createWorkerFn, options); }
*/
this.pathToBitJS_ = options.pathToBitJS || '/';
/** @orivate {boolean} */
this.debugMode_ = !!(options.debug);
/**
* A map from event type to an array of listeners.
* @type {Map.<string, Array>}
*/
this.listeners_ = {};
for (let type in UnarchiveEventType) {
this.listeners_[UnarchiveEventType[type]] = [];
}
/**
* Private web worker initialized during start().
* @type {Worker}
* @private
*/
this.worker_ = null;
}
/**
* This method must be overridden by the subclass to return the script filename.
* @return {string} The script filename.
* @protected.
*/
getScriptFileName() {
throw 'Subclasses of AbstractUnarchiver must overload getScriptFileName()';
}
/**
* Adds an event listener for UnarchiveEvents.
*
* @param {string} Event type.
* @param {function} An event handler function.
*/
addEventListener(type, listener) {
if (type in this.listeners_) {
if (this.listeners_[type].indexOf(listener) == -1) {
this.listeners_[type].push(listener);
}
}
}
/**
* Removes an event listener.
*
* @param {string} Event type.
* @param {EventListener|function} An event listener or handler function.
*/
removeEventListener(type, listener) {
if (type in this.listeners_) {
const index = this.listeners_[type].indexOf(listener);
if (index != -1) {
this.listeners_[type].splice(index, 1);
}
}
}
/**
* Create an UnarchiveEvent out of the object sent back from the Worker.
* @param {Object} obj
* @return {UnarchiveEvent}
* @private
*/
createUnarchiveEvent_(obj) {
switch (obj.type) {
case UnarchiveEventType.START:
return new UnarchiveStartEvent();
case UnarchiveEventType.PROGRESS:
return new UnarchiveProgressEvent(
obj.currentFilename,
obj.currentFileNumber,
obj.currentBytesUnarchivedInFile,
obj.currentBytesUnarchived,
obj.totalUncompressedBytesInArchive,
obj.totalFilesInArchive,
obj.totalCompressedBytesRead);
case UnarchiveEventType.EXTRACT:
return new UnarchiveExtractEvent(obj.unarchivedFile);
case UnarchiveEventType.FINISH:
return new UnarchiveFinishEvent(obj.metadata);
case UnarchiveEventType.INFO:
return new UnarchiveInfoEvent(obj.msg);
case UnarchiveEventType.ERROR:
return new UnarchiveErrorEvent(obj.msg);
}
}
/**
* Receive an event and pass it to the listener functions.
*
* @param {Object} obj
* @private
*/
handleWorkerEvent_(obj) {
const type = obj.type;
if (type && Object.values(UnarchiveEventType).includes(type) &&
this.listeners_[obj.type] instanceof Array) {
const evt = this.createUnarchiveEvent_(obj);
this.listeners_[evt.type].forEach(function (listener) { listener(evt) });
if (evt.type == UnarchiveEventType.FINISH) {
this.worker_.terminate();
}
} else {
console.log(`Unknown object received from worker: ${obj}`);
}
}
/**
* Starts the unarchive in a separate Web Worker thread and returns immediately.
*/
start() {
const me = this;
const scriptFileName = this.pathToBitJS_ + this.getScriptFileName();
if (scriptFileName) {
this.worker_ = new Worker(scriptFileName);
this.worker_.onerror = function (e) {
console.log('Worker error: message = ' + e.message);
throw e;
};
this.worker_.onmessage = function (e) {
if (typeof e.data == 'string') {
// Just log any strings the workers pump our way.
console.log(e.data);
} else {
me.handleWorkerEvent_(e.data);
}
};
const ab = this.ab;
this.worker_.postMessage({
file: ab,
logToConsole: this.debugMode_,
});
this.ab = null;
}
}
// TODO: Create a startSync() method that does not use a worker for Node.
/**
* Adds more bytes to the unarchiver's Worker thread.
* @param {ArrayBuffer} ab The ArrayBuffer with more bytes in it.
*/
update(ab) {
if (this.worker_) {
this.worker_.postMessage({ bytes: ab });
}
}
/**
* Terminates the Web Worker for this Unarchiver and returns immediately.
*/
stop() {
if (this.worker_) {
this.worker_.terminate();
}
}
} }
export class Unrarrer extends UnrarrerInternal {
/** constructor(ab, options) { super(ab, createWorkerFn, options); }
* Unzipper
*/
export class Unzipper extends Unarchiver {
constructor(arrayBuffer, options) {
super(arrayBuffer, options);
}
getScriptFileName() { return 'archive/unzip.js'; }
} }
export class Untarrer extends UntarrerInternal {
/** constructor(ab, options) { super(ab, createWorkerFn, options); }
* Unrarrer
*/
export class Unrarrer extends Unarchiver {
constructor(arrayBuffer, options) {
super(arrayBuffer, options);
}
getScriptFileName() { return 'archive/unrar.js'; }
}
/**
* Untarrer
* @extends {Unarchiver}
* @constructor
*/
export class Untarrer extends Unarchiver {
constructor(arrayBuffer, options) {
super(arrayBuffer, options);
}
getScriptFileName() { return 'archive/untar.js'; };
} }
/** /**
* Factory method that creates an unarchiver based on the byte signature found * Factory method that creates an unarchiver based on the byte signature found
* in the arrayBuffer. * in the arrayBuffer.
* @param {ArrayBuffer} ab * @param {ArrayBuffer} ab
* @param {Object|string} options An optional object of options, or a string representing where * @param {Object} options An optional object of options.
* the path to the unarchiver script files.
* @return {Unarchiver} * @return {Unarchiver}
*/ */
export function getUnarchiver(ab, options = {}) { export function getUnarchiver(ab, options = {}) {
if (ab.byteLength < 10) { return getUnarchiverInternal(ab, createWorkerFn, options);
return null;
}
let unarchiver = null;
const mimeType = findMimeType(ab);
if (mimeType === 'application/x-rar-compressed') { // Rar!
unarchiver = new Unrarrer(ab, options);
} else if (mimeType === 'application/zip') { // PK (Zip)
unarchiver = new Unzipper(ab, options);
} else { // Try with tar
unarchiver = new Untarrer(ab, options);
}
return unarchiver;
} }