From cf26e0a2de1c1b387d45b1846e915a3be02eab5f Mon Sep 17 00:00:00 2001 From: Jeff Schiller Date: Sat, 16 Dec 2023 15:28:37 -0800 Subject: [PATCH] Add some unit tests for unarchivers. Provide a way to disconnect the impl from the host (for unit tests). --- archive/common.js | 21 +++++-- archive/decompress.js | 23 ++++++- archive/unrar.js | 22 ++++++- archive/untar.js | 22 ++++++- archive/unzip.js | 24 +++++++- archive/webworker-wrapper.js | 8 ++- archive/zip.js | 16 ++++- io/bytestream.js | 1 - tests/archive-testfiles/archive-rar.rar | Bin 0 -> 506 bytes tests/archive-testfiles/archive-tar.tar | Bin 0 -> 4608 bytes .../archive-testfiles/archive-zip-faster.zip | Bin 0 -> 785 bytes .../archive-testfiles/archive-zip-smaller.zip | Bin 0 -> 782 bytes tests/archive-testfiles/archive-zip-store.zip | Bin 0 -> 1673 bytes tests/archive-testfiles/sample-1.txt | 20 ++++++ tests/archive-testfiles/sample-2.csv | 4 ++ tests/archive-testfiles/sample-3.json | 6 ++ tests/decompress.spec.js | 57 ++++++++++++++++++ 17 files changed, 211 insertions(+), 13 deletions(-) create mode 100644 tests/archive-testfiles/archive-rar.rar create mode 100644 tests/archive-testfiles/archive-tar.tar create mode 100644 tests/archive-testfiles/archive-zip-faster.zip create mode 100644 tests/archive-testfiles/archive-zip-smaller.zip create mode 100644 tests/archive-testfiles/archive-zip-store.zip create mode 100644 tests/archive-testfiles/sample-1.txt create mode 100644 tests/archive-testfiles/sample-2.csv create mode 100644 tests/archive-testfiles/sample-3.json create mode 100644 tests/decompress.spec.js diff --git a/archive/common.js b/archive/common.js index 4635c66..53e2fc3 100644 --- a/archive/common.js +++ b/archive/common.js @@ -10,6 +10,12 @@ // Requires the following JavaScript features: MessageChannel, MessagePort, and dynamic imports. +/** + * @typedef Implementation + * @property {MessagePort} hostPort The port the host uses to communicate with the implementation. + * @property {Function} disconnectFn A function to call when the port has been disconnected. + */ + /** * Connects a host to a compress/decompress implementation via MessagePorts. The implementation must * have an exported connect() function that accepts a MessagePort. If the runtime support Workers @@ -17,8 +23,9 @@ * dynamically imports the implementation inside the current JS context (node, bun). * @param {string} implFilename The compressor/decompressor implementation filename relative to this * path (e.g. './unzip.js'). - * @returns {Promise} The Promise resolves to the MessagePort connected to the - * implementation that the host should use. + * @param {Function} disconnectFn A function to run when the port is disconnected. + * @returns {Promise} The Promise resolves to the Implementation, which includes the + * MessagePort connected to the implementation that the host should use. */ export async function getConnectedPort(implFilename) { const messageChannel = new MessageChannel(); @@ -28,13 +35,19 @@ export async function getConnectedPort(implFilename) { if (typeof Worker === 'undefined') { const implModule = await import(`${implFilename}`); await implModule.connect(implPort); - return hostPort; + return { + hostPort, + disconnectFn: () => implModule.disconnect(), + }; } return new Promise((resolve, reject) => { const workerScriptPath = new URL(`./webworker-wrapper.js`, import.meta.url).href; const worker = new Worker(workerScriptPath, { type: 'module' }); worker.postMessage({ implSrc: implFilename }, [implPort]); - resolve(hostPort); + resolve({ + hostPort, + disconnectFn: () => worker.postMessage({ disconnect: true }), + }); }); } diff --git a/archive/decompress.js b/archive/decompress.js index 4bd0579..c777e88 100644 --- a/archive/decompress.js +++ b/archive/decompress.js @@ -56,6 +56,13 @@ export class Unarchiver extends EventTarget { */ port_; + /** + * A function to call to disconnect the implementation from the host. + * @type {Function} + * @private + */ + disconnectFn_; + /** * @param {ArrayBuffer} arrayBuffer The Array Buffer. Note that this ArrayBuffer must not be * referenced once it is sent to the Unarchiver, since it is marked as Transferable and sent @@ -87,6 +94,16 @@ export class Unarchiver extends EventTarget { this.debugMode_ = !!(options.debug); } + /** + * Overridden so that the type hints for eventType are specific. + * @param {'progress'|'extract'|'finish'} eventType + * @param {EventListenerOrEventListenerObject} listener + * @override + */ + addEventListener(eventType, listener) { + super.addEventListener(eventType, listener); + } + /** * This method must be overridden by the subclass to return the script filename. * @returns {string} The MIME type of the archive. @@ -164,7 +181,9 @@ export class Unarchiver extends EventTarget { * using the update() method. */ async start() { - this.port_ = await getConnectedPort(this.getScriptFileName()); + const impl = await getConnectedPort(this.getScriptFileName()); + this.port_ = impl.hostPort; + this.disconnectFn_ = impl.disconnectFn; return new Promise((resolve, reject) => { this.port_.onerror = (evt) => { console.log('Impl error: message = ' + evt.message); @@ -221,7 +240,9 @@ export class Unarchiver extends EventTarget { stop() { if (this.port_) { this.port_.close(); + this.disconnectFn_(); this.port_ = null; + this.disconnectFn_ = null; } } } diff --git a/archive/unrar.js b/archive/unrar.js index f12c84f..51b37b4 100644 --- a/archive/unrar.js +++ b/archive/unrar.js @@ -1466,8 +1466,28 @@ const onmessage = function (event) { */ export function connect(port) { if (hostPort) { - throw `hostPort already connected`; + throw `hostPort already connected in unrar.js`; } hostPort = port; port.onmessage = onmessage; } + +export function disconnect() { + if (!hostPort) { + throw `hostPort was not connected in unzip.js`; + } + + hostPort = null; + + unarchiveState = UnarchiveState.NOT_STARTED; + bytestream = null; + allLocalFiles = null; + logToConsole = false; + + currentFilename = ''; + currentFileNumber = 0; + currentBytesUnarchivedInFile = 0; + currentBytesUnarchived = 0; + totalUncompressedBytesInArchive = 0; + totalFilesInArchive = 0; +} diff --git a/archive/untar.js b/archive/untar.js index dea10e7..829fe80 100644 --- a/archive/untar.js +++ b/archive/untar.js @@ -221,8 +221,28 @@ const onmessage = function (event) { */ export function connect(port) { if (hostPort) { - throw `hostPort already connected`; + throw `hostPort already connected in untar.js`; } hostPort = port; port.onmessage = onmessage; } + +export function disconnect() { + if (!hostPort) { + throw `hostPort was not connected in unzip.js`; + } + + hostPort = null; + + unarchiveState = UnarchiveState.NOT_STARTED; + bytestream = null; + allLocalFiles = null; + logToConsole = false; + + currentFilename = ''; + currentFileNumber = 0; + currentBytesUnarchivedInFile = 0; + currentBytesUnarchived = 0; + totalUncompressedBytesInArchive = 0; + totalFilesInArchive = 0; +} diff --git a/archive/unzip.js b/archive/unzip.js index 3385772..c4b29f6 100644 --- a/archive/unzip.js +++ b/archive/unzip.js @@ -781,8 +781,30 @@ const onmessage = function (event) { */ export function connect(port) { if (hostPort) { - throw `hostPort already connected`; + throw `hostPort already connected in unzip.js`; } + hostPort = port; port.onmessage = onmessage; } + +export function disconnect() { + if (!hostPort) { + throw `hostPort was not connected in unzip.js`; + } + + hostPort = null; + + unarchiveState = UnarchiveState.NOT_STARTED; + bytestream = null; + allLocalFiles = null; + logToConsole = false; + + // Progress variables. + currentFilename = ''; + currentFileNumber = 0; + currentBytesUnarchivedInFile = 0; + currentBytesUnarchived = 0; + totalUncompressedBytesInArchive = 0; + totalFilesInArchive = 0; +} diff --git a/archive/webworker-wrapper.js b/archive/webworker-wrapper.js index b9763b3..e334bdc 100644 --- a/archive/webworker-wrapper.js +++ b/archive/webworker-wrapper.js @@ -16,6 +16,10 @@ let implPort; onmessage = async (evt) => { - const module = await import(evt.data.implSrc); - module.connect(evt.ports[0]); + if (evt.data.implSrc) { + const module = await import(evt.data.implSrc); + module.connect(evt.ports[0]); + } else if (evt.data.disconnect) { + module.disconnect(); + } }; diff --git a/archive/zip.js b/archive/zip.js index c6e7696..12f64b1 100644 --- a/archive/zip.js +++ b/archive/zip.js @@ -80,7 +80,6 @@ const CompressorState = { FINISHED: 3, }; let state = CompressorState.NOT_STARTED; -let lastFileReceived = false; const crc32Table = createCRC32Table(); /** Helper functions. */ @@ -280,8 +279,21 @@ const onmessage = function(evt) { */ export function connect(port) { if (hostPort) { - throw `hostPort already connected`; + throw `hostPort already connected in zip.js`; } hostPort = port; port.onmessage = onmessage; } + +export function disconnect() { + if (!hostPort) { + throw `hostPort was not connected in unzip.js`; + } + + hostPort = null; + + centralDirectoryInfos = []; + numBytesWritten = 0; + state = CompressorState.NOT_STARTED; + lastFileReceived = false; +} diff --git a/io/bytestream.js b/io/bytestream.js index a18d7b2..d1ae24f 100644 --- a/io/bytestream.js +++ b/io/bytestream.js @@ -22,7 +22,6 @@ export class ByteStream { */ constructor(ab, opt_offset, opt_length) { if (!(ab instanceof ArrayBuffer)) { - console.error(typeof ab); throw 'Error! ByteStream constructed with an invalid ArrayBuffer object'; } diff --git a/tests/archive-testfiles/archive-rar.rar b/tests/archive-testfiles/archive-rar.rar new file mode 100644 index 0000000000000000000000000000000000000000..4591ae5216ab72806ed5b08fd54935a7198dfcb0 GIT binary patch literal 506 zcmWGaEK-zWXJq*Nu<127BP%-t8zW;wLjyz2Q}HQUOw8QV7+GGhFKK34I&bQn24+T{ z;>6s7oK#&yy^@L&E@t5k-)d6TbEdN%<(X{2tnB}Io5Vzp7TA=raZ}nmuDV$^Lxb!72Mbo|L`g2+($M}+g=&5 z+*nb=X!pU?>*&49bM-hbT~F1L7CGki#`y8}O<{t)8BA9-cA;>rBhjg zGv?)J$w*vS996-7`1=OdoEDDxw~oG6@;_R0J@3=L28W#3C)z-J-!icL0NJ}co4XKb zFE2FkjP!3)moaxRDO6GcC%Dp z5=!`Z=i;=$&{OYDU38r$XM22)1H;C37bGR^jb=QORV r^2DaeU;S1Xa{s8;nby;_8?WY={@Ql3=0?_@s_p{W@~}W=Ru%>Tf_m3Z literal 0 HcmV?d00001 diff --git a/tests/archive-testfiles/archive-tar.tar b/tests/archive-testfiles/archive-tar.tar new file mode 100644 index 0000000000000000000000000000000000000000..4333926b826d0aba8da197c92b49a92676ae8877 GIT binary patch literal 4608 zcmeHJO>5jR5cS!=V$h`*$H?E%Yi>EThm>AQ4#JVwN^HqUauT=XzjwyoY+1;*fv_b9 z34^7lH_~XNd5ja|G{}oA^-n%JQ{$Z1WeHJ%t6-&Ia+phURZSHPIjGZdo>Kw9uf9~$*NJgTD8s#Hm-+JW$ zKLKui3E<_EhfWPLg>~*QCe|SAPJ+~C-u2)OjN)EGYo+i?@8X|oS}h%n#!67yODi2s zCE%29k!r>XA*YOnUeK;>FK7bSdNlSP5Et7nHr_i#zJTwAUoCFuX;7{3lK8jix(4Ob zI1c9s_YvNegH!o&N_(~>yU5elJ)Gja7e{IQoBs1G%MSX_n{w5EUIhJrZ`|KvPwMKI zNhT=!Qj;*p>>&9pVOSQhm|b0e{jy3sB6rScgf6eYzaaQ@Sp1gF0_Vd;V4wd*O4oWO z@BW$p&7bW~f3vyC*4fRSYN#CsAN9OVaiV3|&2E7Iv+m8g)X}z0NaWLz~dGoMEauPAG1eLZZT|W0=uIEyg?~}^X z_@3O744Pui-<&5A^k>PnJ82mnx*aL8wJOYMYu(>kr_Hg8-SwNxNcVzARa%a__ao1n zKWbX|K^Q`u{Gv^-%X)$gRz0@G&W#y$Rs_GTv6?gDoox#L4kN(&!m{43G z;iIy1hfgOr1Hu6Tz@Xio&0UxTw7dd{c@aTttd~`sp9gk4zy}UpVGZi zZTysF-bN8WOX7y+LR&Ejt;$%-=zyA8(cfgQhWRhdXl?o() z4rX9r1mY!)AQozxV1=X!G&dugh#C6GCf-7r2#pJ%iJ-W^VW~f3vyC*4fRSYNp{YXHa*1Jg+jq`7k-MAd{=?b>MHl*_V$m-TmHq`F6KU#mTl&lN?S> zj(_`=>Ecn#`lmaJVl4kXt6&a*dn>5DKYSw4XVyS(c>*z#w~X|Xi_5^?vcc{x@6%^I zbywjP4F)_e-)&t_ z9P{;m5PGClWVLASlFs=`_wKTU8-K3&Y{AfzDXSQClPxi4;o$@7RR>x%qfhDHs5XAe zGVi6)>x*kuJU`c)oeDa8Z}pieQ?vTBeJeMK1#z+`uU~(C?>k^fF*3<9<4OV&KnF80 zFaq(CMi2`%L9jv+1e%+XO~ed+WD~C=OoYY-&_qyN;4u+1IFU_^06G=wMp)nijl>eL U0p6@^AP+GE;dda-$OPg606JU!ssI20 literal 0 HcmV?d00001 diff --git a/tests/archive-testfiles/archive-zip-store.zip b/tests/archive-testfiles/archive-zip-store.zip new file mode 100644 index 0000000000000000000000000000000000000000..84c1e2f82c02691f2c1587e4380ee89a69ecec4a GIT binary patch literal 1673 zcmeH{zi-qq6vut!4ha_|Mg~+pdFc`&0UKgtf=Y1W3LQ{|tkicgm)IeG*WQ)uzyw>x zz=oLGorwkU|1dMqmG_eN?&uHxf=gmKd4BKvp6&S4U60yQ@OAU!!RFD|&tHB;LU6k- zZi}FcGBUeFwTz#8K8$k4-KT1(P-U3?%A@A3{d#WfOTjp6L=_wU|mYCz9`8Qtap%U z&YEVL2S{tzS3@AXZn9MJq#&*ZFw&#^rWy_VZ&T?GEGAMe=ds+6<$o36vh&fM z(c<}sgUwF_jK@R{ZVR-ym!x5*rp3dHX~Aa^C#8uA62|c{mTCqF6)S>`lIltC+0z;X zWO)<3Bh#ba%To)bP;y>Qm*Dkdxbk*PrEt43hu8gtqw9Lsh+N#x(CeK9P&FX>zA7{Z z^#EQe&JaVBntCp3D`rpJfspxK_TosgSX^sj3h!{jQRM_P@5oB;{d_|3;+n7I|FXE#u=vbO78TF$(4u0}o++N}7b)_7{YUZqA}=cb boGYF#IZ0ly+}-ugRi02p_c8U;4uAaxsK^gr literal 0 HcmV?d00001 diff --git a/tests/archive-testfiles/sample-1.txt b/tests/archive-testfiles/sample-1.txt new file mode 100644 index 0000000..b7bac92 --- /dev/null +++ b/tests/archive-testfiles/sample-1.txt @@ -0,0 +1,20 @@ +This is a sample text file. This text file is large enough to make creating +zip files more interesting, since compression has a chance to work on a larger +sample file that has duplicate words in it. + +This is a sample text file. This text file is large enough to make creating +zip files more interesting, since compression has a chance to work on a larger +sample file that has duplicate words in it. + +This is a sample text file. This text file is large enough to make creating +zip files more interesting, since compression has a chance to work on a larger +sample file that has duplicate words in it. + +This is a sample text file. This text file is large enough to make creating +zip files more interesting, since compression has a chance to work on a larger +sample file that has duplicate words in it. + +This is a sample text file. This text file is large enough to make creating +zip files more interesting, since compression has a chance to work on a larger +sample file that has duplicate words in it. + diff --git a/tests/archive-testfiles/sample-2.csv b/tests/archive-testfiles/sample-2.csv new file mode 100644 index 0000000..dd1a5c9 --- /dev/null +++ b/tests/archive-testfiles/sample-2.csv @@ -0,0 +1,4 @@ +filetype,extension +"text file","txt" +"JSON file","json" +"CSV file","csv" diff --git a/tests/archive-testfiles/sample-3.json b/tests/archive-testfiles/sample-3.json new file mode 100644 index 0000000..2259b5e --- /dev/null +++ b/tests/archive-testfiles/sample-3.json @@ -0,0 +1,6 @@ +{ + "file formats": ["csv", "json", "txt"], + "tv shows": { + "it's": ["monty", "python's", "flying", "circus"] + } +} diff --git a/tests/decompress.spec.js b/tests/decompress.spec.js new file mode 100644 index 0000000..4a0c936 --- /dev/null +++ b/tests/decompress.spec.js @@ -0,0 +1,57 @@ +import * as fs from 'node:fs'; +import 'mocha'; +import { expect } from 'chai'; + +import { Unarchiver, Unrarrer, Untarrer, Unzipper, getUnarchiver } from '../archive/decompress.js'; + +const PATH = `tests/archive-testfiles/`; + +const INPUT_FILES = [ + 'sample-1.txt', + 'sample-2.csv', + 'sample-3.json', +]; + +const ARCHIVE_FILES = [ + 'archive-rar.rar', + 'archive-tar.tar', + 'archive-zip-store.zip', + 'archive-zip-faster.zip', + 'archive-zip-smaller.zip', +]; + +describe('bitjs.archive.decompress', () => { + /** @type {Map} */ + let inputArrayBuffers = new Map(); + + before(() => { + for (const inputFile of INPUT_FILES) { + const nodeBuf = fs.readFileSync(`${PATH}${inputFile}`); + const ab = nodeBuf.buffer.slice(nodeBuf.byteOffset, nodeBuf.byteOffset + nodeBuf.length); + inputArrayBuffers.set(inputFile, ab); + } + }); + + for (const outFile of ARCHIVE_FILES) { + it(outFile, (done) => { + const bufs = new Map(inputArrayBuffers); + const nodeBuf = fs.readFileSync(`${PATH}${outFile}`); + const ab = nodeBuf.buffer.slice(nodeBuf.byteOffset, nodeBuf.byteOffset + nodeBuf.length); + let unarchiver = getUnarchiver(ab); + expect(unarchiver instanceof Unarchiver).equals(true); + + unarchiver.addEventListener('extract', evt => { + const {filename, fileData} = evt.unarchivedFile; + expect(bufs.has(filename)).equals(true); + const ab = bufs.get(filename); + expect(fileData.byteLength).equals(ab.byteLength); + for (let b = 0; b < fileData.byteLength; ++b) { + expect(fileData[b] === ab[b]); + } + // Remove the value from the map so that it is only used once. + bufs.delete(filename); + }); + unarchiver.start().then(() => { done() }); + }); + } +});