diff --git a/archive/unrar.js b/archive/unrar.js index f510385..40296e8 100644 --- a/archive/unrar.js +++ b/archive/unrar.js @@ -13,6 +13,7 @@ // This file expects to be invoked as a Worker (see onmessage below). importScripts('../io/bitstream.js'); +importScripts('../io/bytestream.js'); importScripts('../io/bytebuffer.js'); importScripts('archive.js'); importScripts('rarvm.js'); @@ -26,7 +27,7 @@ const UnarchiveState = { // State - consider putting these into a class. let unarchiveState = UnarchiveState.NOT_STARTED; -let bitstream = null; +let bytestream = null; let allLocalFiles = null; let logToConsole = false; @@ -53,7 +54,7 @@ const postProgress = function() { currentBytesUnarchived, totalUncompressedBytesInArchive, totalFilesInArchive, - parseInt(bitstream.getNumBitsRead() / 8, 10), + parseInt(bytestream.getNumBytesRead(), 10), )); }; @@ -85,88 +86,89 @@ const ENDARC_HEAD = 0x7b; */ class RarVolumeHeader { /** - * @param {bitjs.io.BitStream} bstream + * @param {bitjs.io.ByteStream} bstream */ constructor(bstream) { let headBytesRead = 0; // byte 1,2 - this.crc = bstream.readBits(16); + this.crc = bstream.readNumber(2); // byte 3 - this.headType = bstream.readBits(8); + this.headType = bstream.readNumber(1); // Get flags // bytes 4,5 this.flags = {}; - this.flags.value = bstream.peekBits(16); + this.flags.value = bstream.readNumber(2); + const flagsValue = this.flags.value; switch (this.headType) { case MAIN_HEAD: - this.flags.MHD_VOLUME = !!bstream.readBits(1); - this.flags.MHD_COMMENT = !!bstream.readBits(1); - this.flags.MHD_LOCK = !!bstream.readBits(1); - this.flags.MHD_SOLID = !!bstream.readBits(1); - this.flags.MHD_PACK_COMMENT = !!bstream.readBits(1); + this.flags.MHD_VOLUME = !!(flagsValue & 0x01); + this.flags.MHD_COMMENT = !!(flagsValue & 0x02); + this.flags.MHD_LOCK = !!(flagsValue & 0x04); + this.flags.MHD_SOLID = !!(flagsValue & 0x08); + this.flags.MHD_PACK_COMMENT = !!(flagsValue & 0x10); this.flags.MHD_NEWNUMBERING = this.flags.MHD_PACK_COMMENT; - this.flags.MHD_AV = !!bstream.readBits(1); - this.flags.MHD_PROTECT = !!bstream.readBits(1); - this.flags.MHD_PASSWORD = !!bstream.readBits(1); - this.flags.MHD_FIRSTVOLUME = !!bstream.readBits(1); - this.flags.MHD_ENCRYPTVER = !!bstream.readBits(1); - bstream.readBits(6); // unused + this.flags.MHD_AV = !!(flagsValue & 0x20); + this.flags.MHD_PROTECT = !!(flagsValue & 0x40); + this.flags.MHD_PASSWORD = !!(flagsValue & 0x80); + this.flags.MHD_FIRSTVOLUME = !!(flagsValue & 0x100); + this.flags.MHD_ENCRYPTVER = !!(flagsValue & 0x200); + //bstream.readBits(6); // unused break; case FILE_HEAD: - this.flags.LHD_SPLIT_BEFORE = !!bstream.readBits(1); // 0x0001 - this.flags.LHD_SPLIT_AFTER = !!bstream.readBits(1); // 0x0002 - this.flags.LHD_PASSWORD = !!bstream.readBits(1); // 0x0004 - this.flags.LHD_COMMENT = !!bstream.readBits(1); // 0x0008 - this.flags.LHD_SOLID = !!bstream.readBits(1); // 0x0010 - bstream.readBits(3); // unused - this.flags.LHD_LARGE = !!bstream.readBits(1); // 0x0100 - this.flags.LHD_UNICODE = !!bstream.readBits(1); // 0x0200 - this.flags.LHD_SALT = !!bstream.readBits(1); // 0x0400 - this.flags.LHD_VERSION = !!bstream.readBits(1); // 0x0800 - this.flags.LHD_EXTTIME = !!bstream.readBits(1); // 0x1000 - this.flags.LHD_EXTFLAGS = !!bstream.readBits(1); // 0x2000 - bstream.readBits(2); // unused + this.flags.LHD_SPLIT_BEFORE = !!(flagsValue & 0x01); + this.flags.LHD_SPLIT_AFTER = !!(flagsValue & 0x02); + this.flags.LHD_PASSWORD = !!(flagsValue & 0x04); + this.flags.LHD_COMMENT = !!(flagsValue & 0x08); + this.flags.LHD_SOLID = !!(flagsValue & 0x10); + // 3 bits unused + this.flags.LHD_LARGE = !!(flagsValue & 0x100); + this.flags.LHD_UNICODE = !!(flagsValue & 0x200); + this.flags.LHD_SALT = !!(flagsValue & 0x400); + this.flags.LHD_VERSION = !!(flagsValue & 0x800); + this.flags.LHD_EXTTIME = !!(flagsValue & 0x1000); + this.flags.LHD_EXTFLAGS = !!(flagsValue & 0x2000); + // 2 bits unused //info(" LHD_SPLIT_BEFORE = " + this.flags.LHD_SPLIT_BEFORE); break; default: - bstream.readBits(16); + break; } // byte 6,7 - this.headSize = bstream.readBits(16); + this.headSize = bstream.readNumber(2); headBytesRead += 7; switch (this.headType) { case MAIN_HEAD: - this.highPosAv = bstream.readBits(16); - this.posAv = bstream.readBits(32); + this.highPosAv = bstream.readNumber(2); + this.posAv = bstream.readNumber(4); headBytesRead += 6; if (this.flags.MHD_ENCRYPTVER) { - this.encryptVer = bstream.readBits(8); + this.encryptVer = bstream.readNumber(1); headBytesRead += 1; } //info("Found MAIN_HEAD with highPosAv=" + this.highPosAv + ", posAv=" + this.posAv); break; case FILE_HEAD: - this.packSize = bstream.readBits(32); - this.unpackedSize = bstream.readBits(32); - this.hostOS = bstream.readBits(8); - this.fileCRC = bstream.readBits(32); - this.fileTime = bstream.readBits(32); - this.unpVer = bstream.readBits(8); - this.method = bstream.readBits(8); - this.nameSize = bstream.readBits(16); - this.fileAttr = bstream.readBits(32); + this.packSize = bstream.readNumber(4); + this.unpackedSize = bstream.readNumber(4); + this.hostOS = bstream.readNumber(1); + this.fileCRC = bstream.readNumber(4); + this.fileTime = bstream.readNumber(4); + this.unpVer = bstream.readNumber(1); + this.method = bstream.readNumber(1); + this.nameSize = bstream.readNumber(2); + this.fileAttr = bstream.readNumber(4); headBytesRead += 25; if (this.flags.LHD_LARGE) { //info("Warning: Reading in LHD_LARGE 64-bit size values"); - this.HighPackSize = bstream.readBits(32); - this.HighUnpSize = bstream.readBits(32); + this.HighPackSize = bstream.readNumber(4); + this.HighUnpSize = bstream.readNumber(4); headBytesRead += 8; } else { this.HighPackSize = 0; @@ -184,6 +186,7 @@ class RarVolumeHeader { // read in filename + // TODO: Use readString? this.filename = bstream.readBytes(this.nameSize); headBytesRead += this.nameSize; let _s = ''; @@ -195,13 +198,13 @@ class RarVolumeHeader { if (this.flags.LHD_SALT) { //info("Warning: Reading in 64-bit salt value"); - this.salt = bstream.readBits(64); // 8 bytes + this.salt = bstream.readBytes(8); // 8 bytes headBytesRead += 8; } if (this.flags.LHD_EXTTIME) { // 16-bit flags - const extTimeFlags = bstream.readBits(16); + const extTimeFlags = bstream.readNumber(2); headBytesRead += 2; // this is adapted straight out of arcread.cpp, Archive::ReadHeader() @@ -211,12 +214,12 @@ class RarVolumeHeader { continue; } if (I != 0) { - bstream.readBits(16); + bstream.readBytes(2); headBytesRead += 2; } const count = (rmode & 3); for (let J = 0; J < count; ++J) { - bstream.readBits(8); + bstream.readNumber(1); headBytesRead += 1; } } @@ -232,7 +235,9 @@ class RarVolumeHeader { break; default: - info("Found a header of type 0x" + byteValueToHexString(this.headType)); + if (logToConsole) { + info("Found a header of type 0x" + byteValueToHexString(this.headType)); + } // skip the rest of the header bytes (for now) bstream.readBytes(this.headSize - 7); break; @@ -1256,7 +1261,9 @@ function unpack(v) { rBuffer = new bitjs.io.ByteBuffer(v.header.unpackedSize); - info("Unpacking " + v.filename + " RAR v" + Ver); + if (logToConsole) { + info("Unpacking " + v.filename + " RAR v" + Ver); + } switch (Ver) { case 15: // rar 1.5 compression @@ -1282,7 +1289,7 @@ function unpack(v) { */ class RarLocalFile { /** - * @param {bitjs.io.BitStream} bstream + * @param {bitjs.io.ByteStream} bstream */ constructor(bstream) { this.header = new RarVolumeHeader(bstream); @@ -1306,7 +1313,9 @@ class RarLocalFile { if (!this.header.flags.LHD_SPLIT_BEFORE) { // unstore file if (this.header.method == 0x30) { - info("Unstore "+this.filename); + if (logToConsole) { + info("Unstore " + this.filename); + } this.isValid = true; currentBytesUnarchivedInFile += this.fileData.length; @@ -1327,34 +1336,38 @@ class RarLocalFile { // Reads in the volume and main header. function unrar_start() { - let bstream = bitstream.tee(); + let bstream = bytestream.tee(); const header = new RarVolumeHeader(bstream); if (header.crc == 0x6152 && header.headType == 0x72 && header.flags.value == 0x1A21 && header.headSize == 7) { - info("Found RAR signature"); + if (logToConsole) { + info("Found RAR signature"); + } const mhead = new RarVolumeHeader(bstream); if (mhead.headType != MAIN_HEAD) { info("Error! RAR did not include a MAIN_HEAD header"); } else { - bitstream = bstream.tee(); + bytestream = bstream.tee(); } } } function unrar() { - let bstream = bitstream.tee(); + let bstream = bytestream.tee(); let localFile = null; do { localFile = new RarLocalFile(bstream); - info("RAR localFile isValid=" + localFile.isValid + ", volume packSize=" + localFile.header.packSize); - localFile.header.dump(); + if (logToConsole) { + info("RAR localFile isValid=" + localFile.isValid + ", volume packSize=" + localFile.header.packSize); + localFile.header.dump(); + } if (localFile && localFile.isValid && localFile.header.packSize > 0) { - bitstream = bstream.tee(); + bytestream = bstream.tee(); totalUncompressedBytesInArchive += localFile.header.unpackedSize; allLocalFiles.push(localFile); @@ -1376,7 +1389,7 @@ function unrar() { postProgress(); - bitstream = bstream.tee(); + bytestream = bstream.tee(); }; // event.data.file has the first ArrayBuffer. @@ -1386,8 +1399,8 @@ onmessage = function(event) { logToConsole = !!event.data.logToConsole; // This is the very first time we have been called. Initialize the bytestream. - if (!bitstream) { - bitstream = new bitjs.io.BitStream(bytes); + if (!bytestream) { + bytestream = new bitjs.io.ByteStream(bytes); currentFilename = ""; currentFileNumber = 0; @@ -1398,7 +1411,7 @@ onmessage = function(event) { allLocalFiles = []; postMessage(new bitjs.archive.UnarchiveStartEvent()); } else { - bitstream.push(bytes); + bytestream.push(bytes); } if (unarchiveState === UnarchiveState.NOT_STARTED) { diff --git a/io/bitstream.js b/io/bitstream.js index 22cde65..3ae1c3e 100644 --- a/io/bitstream.js +++ b/io/bitstream.js @@ -13,10 +13,11 @@ var bitjs = bitjs || {}; bitjs.io = bitjs.io || {}; -// TODO: Add method for tee-ing off the stream with tests. /** * This object allows you to peek and consume bits and bytes out of a stream. - * More bits can be pushed into the back of the stream via the push() method. + * Note that this stream is optimized, and thus, will *NOT* throw an error if + * the end of the stream is reached. Only use this in scenarios where you + * already have all the bits you need. */ bitjs.io.BitStream = class { /** @@ -35,21 +36,14 @@ bitjs.io.BitStream = class { const length = opt_length || ab.byteLength; /** - * The current page of bytes in the stream. + * The bytes in the stream. * @type {Uint8Array} * @private */ this.bytes = new Uint8Array(ab, offset, length); /** - * The next pages of bytes in the stream. - * @type {Array} - * @private - */ - this.pages_ = []; - - /** - * The byte in the current page that we are currently on. + * The byte in the stream that we are currently on. * @type {Number} * @private */ @@ -84,27 +78,7 @@ bitjs.io.BitStream = class { */ getNumBitsLeft() { const bitsLeftInByte = 8 - this.bitPtr; - const bitsLeftInCurrentPage = (this.bytes.byteLength - this.bytePtr - 1) * 8 + bitsLeftInByte; - return this.pages_.reduce((acc, arr) => acc + arr.length * 8, bitsLeftInCurrentPage); - } - - /** - * Move the pointer ahead n bits. The bytePtr and current page are updated as needed. - * This is a private method, no validation is done. - * @param {number} n Number of bits to increment. - * @private - */ - movePointer_(n) { - this.bitPtr += n; - this.bitsRead_ += n; - while (this.bitPtr >= 8) { - this.bitPtr -= 8; - this.bytePtr++; - while (this.bytePtr >= this.bytes.length && this.pages_.length > 0) { - this.bytePtr -= this.bytes.length; - this.bytes = this.pages_.shift(); - } - } + return (this.bytes.byteLength - this.bytePtr - 1) * 8 + bitsLeftInByte; } /** @@ -120,20 +94,13 @@ bitjs.io.BitStream = class { peekBits_ltr(n, opt_movePointers) { const NUM = parseInt(n, 10); let num = NUM; - if (n !== num || num < 0) { - throw 'Error! Called peekBits_ltr() with a non-positive integer'; - } else if (num === 0) { + if (n !== num || num <= 0) { return 0; } - if (num > this.getNumBitsLeft()) { - throw 'Error! Overflowed the bit stream! n=' + n + ', bytePtr=' + this.bytePtr + - ', bytes.length=' + this.bytes.length + ', bitPtr=' + this.bitPtr; - } - + const BITMASK = bitjs.io.BitStream.BITMASK; const movePointers = opt_movePointers || false; - let curPage = this.bytes; - let pageIndex = 0; + let bytes = this.bytes; let bytePtr = this.bytePtr; let bitPtr = this.bitPtr; let result = 0; @@ -141,32 +108,33 @@ bitjs.io.BitStream = class { // keep going until we have no more bits left to peek at while (num > 0) { - if (bytePtr >= curPage.length && this.pages_.length > 0) { - curPage = this.pages_[pageIndex++]; - bytePtr = 0; + // We overflowed the stream, so just return what we got. + if (bytePtr >= bytes.length) { + break; } const numBitsLeftInThisByte = (8 - bitPtr); if (num >= numBitsLeftInThisByte) { - const mask = (bitjs.io.BitStream.BITMASK[numBitsLeftInThisByte] << bitPtr); - result |= (((curPage[bytePtr] & mask) >> bitPtr) << bitsIn); + const mask = (BITMASK[numBitsLeftInThisByte] << bitPtr); + result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); bytePtr++; bitPtr = 0; bitsIn += numBitsLeftInThisByte; num -= numBitsLeftInThisByte; } else { - const mask = (bitjs.io.BitStream.BITMASK[num] << bitPtr); - result |= (((curPage[bytePtr] & mask) >> bitPtr) << bitsIn); + const mask = (BITMASK[num] << bitPtr); + result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); bitPtr += num; - bitsIn += num; - num = 0; + break; } } if (movePointers) { - this.movePointer_(NUM); + this.bitPtr = bitPtr; + this.bytePtr = bytePtr; + this.bitsRead_ += NUM; } return result; @@ -185,50 +153,45 @@ bitjs.io.BitStream = class { peekBits_rtl(n, opt_movePointers) { const NUM = parseInt(n, 10); let num = NUM; - if (n !== num || num < 0) { - throw 'Error! Called peekBits_rtl() with a non-positive integer'; - } else if (num === 0) { + if (n !== num || num <= 0) { return 0; } - if (num > this.getNumBitsLeft()) { - throw 'Error! Overflowed the bit stream! n=' + n + ', bytePtr=' + this.bytePtr + - ', bytes.length=' + this.bytes.length + ', bitPtr=' + this.bitPtr; - } - + const BITMASK = bitjs.io.BitStream.BITMASK; const movePointers = opt_movePointers || false; - let curPage = this.bytes; - let pageIndex = 0; + let bytes = this.bytes; let bytePtr = this.bytePtr; let bitPtr = this.bitPtr; let result = 0; // keep going until we have no more bits left to peek at while (num > 0) { - if (bytePtr >= curPage.length && this.pages_.length > 0) { - curPage = this.pages_[pageIndex++]; - bytePtr = 0; + // We overflowed the stream, so just return the bits we got. + if (bytePtr >= bytes.length) { + break; } const numBitsLeftInThisByte = (8 - bitPtr); if (num >= numBitsLeftInThisByte) { result <<= numBitsLeftInThisByte; - result |= (bitjs.io.BitStream.BITMASK[numBitsLeftInThisByte] & curPage[bytePtr]); + result |= (BITMASK[numBitsLeftInThisByte] & bytes[bytePtr]); bytePtr++; bitPtr = 0; num -= numBitsLeftInThisByte; } else { result <<= num; const numBits = 8 - num - bitPtr; - result |= ((curPage[bytePtr] & (bitjs.io.BitStream.BITMASK[num] << numBits)) >> numBits); + result |= ((bytes[bytePtr] & (BITMASK[num] << numBits)) >> numBits); bitPtr += num; - num = 0; + break; } } if (movePointers) { - this.movePointer_(NUM); + this.bitPtr = bitPtr; + this.bytePtr = bytePtr; + this.bitsRead_ += NUM; } return result; @@ -286,20 +249,19 @@ bitjs.io.BitStream = class { const movePointers = opt_movePointers || false; const result = new Uint8Array(num); - let curPage = this.bytes; + let bytes = this.bytes; let ptr = this.bytePtr; let bytesLeftToCopy = num; - let pageIndex = 0; while (bytesLeftToCopy > 0) { - const bytesLeftInPage = curPage.length - ptr; - const sourceLength = Math.min(bytesLeftToCopy, bytesLeftInPage); + const bytesLeftInStream = bytes.length - ptr; + const sourceLength = Math.min(bytesLeftToCopy, bytesLeftInStream); - result.set(curPage.subarray(ptr, ptr + sourceLength), num - bytesLeftToCopy); + result.set(bytes.subarray(ptr, ptr + sourceLength), num - bytesLeftToCopy); ptr += sourceLength; - if (ptr >= curPage.length) { - curPage = this.pages_[pageIndex++]; - ptr = 0; + // Overflowed the stream, just return what we got. + if (ptr >= bytes.length) { + break; } bytesLeftToCopy -= sourceLength; @@ -320,33 +282,6 @@ bitjs.io.BitStream = class { readBytes(n) { return this.peekBytes(n, true); } - - /** - * Feeds more bytes into the back of the stream. - * @param {ArrayBuffer} ab - */ - push(ab) { - if (!(ab instanceof ArrayBuffer)) { - throw 'Error! BitStream.push() called with an invalid ArrayBuffer object'; - } - - this.pages_.push(new Uint8Array(ab)); - } - - /** - * Creates a new BitStream from this BitStream that can be read / peeked. - * @return {bitjs.io.BitStream} A clone of this BitStream. - */ - tee() { - const clone = new bitjs.io.BitStream(this.bytes.buffer); - clone.bytes = this.bytes; - clone.pages_ = this.pages_.slice(); - clone.bytePtr = this.bytePtr; - clone.bitPtr = this.bitPtr; - clone.peekBits = this.peekBits; - clone.bitsRead_ = this.bitsRead_; - return clone; - } } // mask for getting N number of bits (0-8) diff --git a/io/bytestream.js b/io/bytestream.js index f1df759..cb5df36 100644 --- a/io/bytestream.js +++ b/io/bytestream.js @@ -13,7 +13,6 @@ var bitjs = bitjs || {}; bitjs.io = bitjs.io || {}; -// TODO: Add method for tee-ing off the stream with tests. /** * This object allows you to peek and consume bytes as numbers and strings out * of a stream. More bytes can be pushed into the back of the stream via the @@ -185,7 +184,7 @@ bitjs.io.ByteStream = class { */ peekBytes(n, movePointers) { const num = parseInt(n, 10); - if (n !== num || num <= 0) { + if (n !== num || num < 0) { throw 'Error! Called peekBytes() with a non-positive integer'; } else if (num === 0) { return new Uint8Array(); diff --git a/tests/io-bitstream-test.html b/tests/io-bitstream-test.html index d228cc7..909f11f 100644 --- a/tests/io-bitstream-test.html +++ b/tests/io-bitstream-test.html @@ -32,9 +32,8 @@ // 10010 = 2 + 16 = 18 assertEquals(18, stream.readBits(5)); - // Only 1 bit left in the buffer, make sure it throws an error if we try to read more. - assertThrows(() => stream.readBits(2), - 'Did not throw when trying to read bits past the end of the stream during RTL'); + // Ensure the last bit is read, even if we flow past the end of the stream. + assertEquals(1, stream.readBits(2)); }, testBitPeekAndRead_LTR() { @@ -50,9 +49,8 @@ // 11001 = 1 + 8 + 16 = 25 assertEquals(25, stream.readBits(5)); - // Only 1 bit left in the buffer, make sure it throws an error if we try to read more. - assertThrows(() => stream.readBits(2), - 'Did not throw when trying to read bits past the end of the stream during LTR'); + // Only 1 bit left in the buffer, make sure it reads in, even if we over-read. + assertEquals(0, stream.readBits(2)); }, testBitStreamReadBytes() { @@ -74,60 +72,6 @@ assertThrows(() => stream.readBytes(3), 'Did not throw when trying to read bytes past the end of the stream'); }, - - testBitStreamReadAfterPush_RTL() { - array = new Uint8Array(1); - array[0] = Number('0b01010110'); - const stream = new bitjs.io.BitStream(array.buffer, true /* rtl */); - const readBits = stream.readBits(8); - assertEquals(0x56, readBits, 'Could not read 8 bits: ' + readBits); - assertThrows(() => stream.readBits(4), - 'Did not throw when trying to read a bit past the end of the stream'); - - const anotherArray = new Uint8Array(1); - anotherArray[0] = Number('0b01010110'); - stream.push(anotherArray.buffer); - - assertEquals(0x5, stream.readBits(4), - 'Could not read in next 4 bits after pushing in RTL'); - assertEquals(0x6, stream.readBits(4), - 'Could not read in next 4 bits after pushing in RTL'); - }, - - testBitStreamReadAfterPush_LTR() { - array = new Uint8Array(1); - array[0] = Number('0b01010110'); - const stream = new bitjs.io.BitStream(array.buffer, false /* rtl */); - const readBits = stream.readBits(8); - assertEquals(0x56, readBits, 'Could not read 8 bits: ' + readBits); - assertThrows(() => stream.readBits(4), - 'Did not throw when trying to read a bit past the end of the stream'); - - const anotherArray = new Uint8Array(1); - anotherArray[0] = Number('0b01010110'); - stream.push(anotherArray.buffer); - - assertEquals(0x6, stream.readBits(4), - 'Could not read in next 4 bits after pushing in LTR'); - assertEquals(0x5, stream.readBits(4), - 'Could not read in next 4 bits after pushing in LTR'); - }, - - testTee() { - for (let i = 0; i < 4; ++i) array[i] = 65 + i; - const stream = new bitjs.io.BitStream(array.buffer); - stream.readBits(1); - - const anotherArray = new Uint8Array(4); - for (let i = 0; i < 4; ++i) anotherArray[i] = 69 + i; - stream.push(anotherArray.buffer); - - const teed = stream.tee(); - teed.readBytes(5); - assertEquals(8 * 8 - 1, stream.getNumBitsLeft()); - // readBytes() throws away any unused bits. - assertEquals(2 * 8, teed.getNumBitsLeft()); - }, }, });