diff --git a/Makefile b/Makefile index cf0a19c7..86bc6303 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ CARGO_FLAGS_SAFE=\ CARGO_FLAGS=$(CARGO_FLAGS_SAFE) -C target-feature=+bulk-memory -CORE_FILES=const.js config.js io.js main.js lib.js ide.js pci.js floppy.js \ +CORE_FILES=const.js config.js io.js main.js lib.js buffer.js ide.js pci.js floppy.js \ memory.js dma.js pit.js vga.js ps2.js pic.js rtc.js uart.js hpet.js \ acpi.js apic.js ioapic.js \ state.js ne2k.js sb16.js virtio.js bus.js log.js \ diff --git a/debug.html b/debug.html index 61d6c2ef..11f11627 100644 --- a/debug.html +++ b/debug.html @@ -8,7 +8,7 @@ "use strict"; var CORE_FILES = - "const.js config.js log.js lib.js cpu.js debug.js " + + "const.js config.js log.js lib.js buffer.js cpu.js debug.js " + "io.js main.js ide.js pci.js floppy.js " + "memory.js dma.js pit.js vga.js ps2.js pic.js rtc.js uart.js acpi.js apic.js ioapic.js hpet.js sb16.js " + "ne2k.js state.js virtio.js bus.js elf.js kernel.js"; diff --git a/src/browser/lib.js b/src/browser/lib.js index beae0a3d..f02092dd 100644 --- a/src/browser/lib.js +++ b/src/browser/lib.js @@ -1,8 +1,5 @@ "use strict"; -/** @const */ -var ASYNC_SAFE = false; - (function() { if(typeof XMLHttpRequest === "undefined") @@ -14,11 +11,6 @@ var ASYNC_SAFE = false; v86util.load_file = load_file; } - v86util.AsyncXHRBuffer = AsyncXHRBuffer; - v86util.AsyncXHRPartfileBuffer = AsyncXHRPartfileBuffer; - v86util.AsyncFileBuffer = AsyncFileBuffer; - v86util.SyncFileBuffer = SyncFileBuffer; - // Reads len characters at offset from Memory object mem as a JS string v86util.read_sized_string_from_mem = function read_sized_string_from_mem(mem, offset, len) { @@ -178,685 +170,4 @@ var ASYNC_SAFE = false; }); } } - - if(typeof XMLHttpRequest === "undefined") - { - var determine_size = function(path, cb) - { - require("fs")["stat"](path, (err, stats) => - { - if(err) - { - cb(err); - } - else - { - cb(null, stats.size); - } - }); - }; - } - else - { - var determine_size = function(url, cb) - { - v86util.load_file(url, { - done: (buffer, http) => - { - var header = http.getResponseHeader("Content-Range") || ""; - var match = header.match(/\/(\d+)\s*$/); - - if(match) - { - cb(null, +match[1]); - } - else - { - const error = "`Range: bytes=...` header not supported (Got `" + header + "`)"; - cb(error); - } - }, - headers: { - Range: "bytes=0-0", - } - }); - }; - } - - /** - * Asynchronous access to ArrayBuffer, loading blocks lazily as needed, - * using the `Range: bytes=...` header - * - * @constructor - * @param {string} filename Name of the file to download - * @param {number|undefined} size - */ - function AsyncXHRBuffer(filename, size) - { - this.filename = filename; - - /** @const */ - this.block_size = 256; - this.byteLength = size; - - this.block_cache = new Map(); - this.block_cache_is_write = new Set(); - - this.onload = undefined; - this.onprogress = undefined; - } - - AsyncXHRBuffer.prototype.load = function() - { - if(this.byteLength !== undefined) - { - this.onload && this.onload(Object.create(null)); - return; - } - - // Determine the size using a request - - determine_size(this.filename, (error, size) => - { - if(error) - { - throw new Error("Cannot use: " + this.filename + ". " + error); - } - else - { - dbg_assert(size >= 0); - this.byteLength = size; - this.onload && this.onload(Object.create(null)); - } - }); - }; - - /** - * @param {number} offset - * @param {number} len - * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - */ - AsyncXHRBuffer.prototype.get_from_cache = function(offset, len) - { - var number_of_blocks = len / this.block_size; - var block_index = offset / this.block_size; - - for(var i = 0; i < number_of_blocks; i++) - { - var block = this.block_cache.get(block_index + i); - - if(!block) - { - return; - } - } - - if(number_of_blocks === 1) - { - return this.block_cache.get(block_index); - } - else - { - var result = new Uint8Array(len); - for(var i = 0; i < number_of_blocks; i++) - { - result.set(this.block_cache.get(block_index + i), i * this.block_size); - } - return result; - } - }; - - /** - * @param {number} offset - * @param {number} len - * @param {function(!Uint8Array)} fn - */ - AsyncXHRBuffer.prototype.get = function(offset, len, fn) - { - console.assert(offset + len <= this.byteLength); - console.assert(offset % this.block_size === 0); - console.assert(len % this.block_size === 0); - console.assert(len); - - var block = this.get_from_cache(offset, len); - if(block) - { - if(ASYNC_SAFE) - { - setTimeout(fn.bind(this, block), 0); - } - else - { - fn(block); - } - return; - } - - v86util.load_file(this.filename, { - done: function done(buffer) - { - var block = new Uint8Array(buffer); - this.handle_read(offset, len, block); - fn(block); - }.bind(this), - range: { start: offset, length: len }, - }); - }; - - /** - * Relies on this.byteLength, this.block_cache and this.block_size - * - * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - * - * @param {number} start - * @param {!Uint8Array} data - * @param {function()} fn - */ - AsyncXHRBuffer.prototype.set = function(start, data, fn) - { - console.assert(start + data.byteLength <= this.byteLength); - - var len = data.length; - - console.assert(start % this.block_size === 0); - console.assert(len % this.block_size === 0); - console.assert(len); - - var start_block = start / this.block_size; - var block_count = len / this.block_size; - - for(var i = 0; i < block_count; i++) - { - var block = this.block_cache.get(start_block + i); - - if(block === undefined) - { - block = new Uint8Array(this.block_size); - this.block_cache.set(start_block + i, block); - } - - var data_slice = data.subarray(i * this.block_size, (i + 1) * this.block_size); - block.set(data_slice); - - console.assert(block.byteLength === data_slice.length); - - this.block_cache_is_write.add(start_block + i); - } - - fn(); - }; - - /** - * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - * @param {number} offset - * @param {number} len - * @param {!Uint8Array} block - */ - AsyncXHRBuffer.prototype.handle_read = function(offset, len, block) - { - // Used by AsyncXHRBuffer, AsyncXHRPartfileBuffer and AsyncFileBuffer - // Overwrites blocks from the original source that have been written since - - var start_block = offset / this.block_size; - var block_count = len / this.block_size; - - for(var i = 0; i < block_count; i++) - { - const cached_block = this.block_cache.get(start_block + i); - - if(cached_block) - { - block.set(cached_block, i * this.block_size); - } - else if(this.cache_reads) - { - const cached = new Uint8Array(this.block_size); - cached.set(block.subarray(i * this.block_size, (i + 1) * this.block_size)); - this.block_cache.set(start_block + i, cached); - } - } - }; - - AsyncXHRBuffer.prototype.get_buffer = function(fn) - { - // We must download all parts, unlikely a good idea for big files - fn(); - }; - - ///** - // * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - // */ - //AsyncXHRBuffer.prototype.get_block_cache = function() - //{ - // var count = Object.keys(this.block_cache).length; - - // var buffer = new Uint8Array(count * this.block_size); - // var indices = []; - - // var i = 0; - // for(var index of Object.keys(this.block_cache)) - // { - // var block = this.block_cache.get(index); - // dbg_assert(block.length === this.block_size); - // index = +index; - // indices.push(index); - // buffer.set( - // block, - // i * this.block_size - // ); - // i++; - // } - - // return { - // buffer, - // indices, - // block_size: this.block_size, - // }; - //}; - - /** - * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - */ - AsyncXHRBuffer.prototype.get_state = function() - { - const state = []; - const block_cache = []; - - for(let [index, block] of this.block_cache) - { - dbg_assert(isFinite(index)); - if(this.block_cache_is_write.has(index)) - { - block_cache.push([index, block]); - } - } - - state[0] = block_cache; - return state; - }; - - /** - * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} - */ - AsyncXHRBuffer.prototype.set_state = function(state) - { - const block_cache = state[0]; - this.block_cache.clear(); - this.block_cache_is_write.clear(); - - for(let [index, block] of block_cache) - { - dbg_assert(isFinite(index)); - this.block_cache.set(index, block); - this.block_cache_is_write.add(index); - } - }; - - /** - * Asynchronous access to ArrayBuffer, loading blocks lazily as needed, - * downloading files named filename-%d-%d.ext (where the %d are start and end offset). - * Or, if partfile_alt_format is set, filename-%08d.ext (where %d is the part number, compatible with gnu split). - * - * @constructor - * @param {string} filename Name of the file to download - * @param {number|undefined} size - * @param {number|undefined} fixed_chunk_size - * @param {boolean|undefined} partfile_alt_format - */ - function AsyncXHRPartfileBuffer(filename, size, fixed_chunk_size, partfile_alt_format) - { - const parts = filename.match(/(.*)(\..*)/); - - if(parts) - { - this.basename = parts[1]; - this.extension = parts[2]; - } - else - { - this.basename = filename; - this.extension = ""; - } - - /** @const */ - this.block_size = 256; // TODO: Could probably be set to fixed_chunk_size if present - this.block_cache = new Map(); - this.block_cache_is_write = new Set(); - - this.byteLength = size; - this.fixed_chunk_size = fixed_chunk_size; - this.partfile_alt_format = !!partfile_alt_format; - - this.cache_reads = !!fixed_chunk_size; // TODO: could also be useful in other cases (needs testing) - - this.onload = undefined; - this.onprogress = undefined; - } - - AsyncXHRPartfileBuffer.prototype.load = function() - { - if(this.byteLength !== undefined) - { - this.onload && this.onload(Object.create(null)); - return; - } - dbg_assert(false); - this.onload && this.onload(Object.create(null)); - }; - - /** - * @param {number} offset - * @param {number} len - * @param {function(!Uint8Array)} fn - */ - AsyncXHRPartfileBuffer.prototype.get = function(offset, len, fn) - { - console.assert(offset + len <= this.byteLength); - console.assert(offset % this.block_size === 0); - console.assert(len % this.block_size === 0); - console.assert(len); - - const block = this.get_from_cache(offset, len); - - if(block) - { - if(ASYNC_SAFE) - { - setTimeout(fn.bind(this, block), 0); - } - else - { - fn(block); - } - return; - } - - if(this.fixed_chunk_size) - { - const start_index = Math.floor(offset / this.fixed_chunk_size); - const m_offset = offset - start_index * this.fixed_chunk_size; - dbg_assert(m_offset >= 0); - const total_count = Math.ceil((m_offset + len) / this.fixed_chunk_size); - const blocks = new Uint8Array(total_count * this.fixed_chunk_size); - let finished = 0; - - for(let i = 0; i < total_count; i++) - { - const offset = (start_index + i) * this.fixed_chunk_size; - - const part_filename = - this.partfile_alt_format ? - // matches output of gnu split: - // split -b 512 -a8 -d --additional-suffix .img w95.img w95- - this.basename + "-" + (start_index + i + "").padStart(8, "0") + this.extension - : - this.basename + "-" + offset + "-" + (offset + this.fixed_chunk_size) + this.extension; - - // XXX: unnecessary allocation - const block = this.get_from_cache(offset, this.fixed_chunk_size); - - if(block) - { - const cur = i * this.fixed_chunk_size; - blocks.set(block, cur); - finished++; - if(finished === total_count) - { - const tmp_blocks = blocks.subarray(m_offset, m_offset + len); - fn(tmp_blocks); - } - } - else - { - v86util.load_file(part_filename, { - done: function done(buffer) - { - const cur = i * this.fixed_chunk_size; - const block = new Uint8Array(buffer); - this.handle_read((start_index + i) * this.fixed_chunk_size, this.fixed_chunk_size|0, block); - blocks.set(block, cur); - finished++; - if(finished === total_count) - { - const tmp_blocks = blocks.subarray(m_offset, m_offset + len); - fn(tmp_blocks); - } - }.bind(this), - }); - } - } - } - else - { - const part_filename = this.basename + "-" + offset + "-" + (offset + len) + this.extension; - - v86util.load_file(part_filename, { - done: function done(buffer) - { - dbg_assert(buffer.byteLength === len); - var block = new Uint8Array(buffer); - this.handle_read(offset, len, block); - fn(block); - }.bind(this), - }); - } - }; - - AsyncXHRPartfileBuffer.prototype.get_from_cache = AsyncXHRBuffer.prototype.get_from_cache; - AsyncXHRPartfileBuffer.prototype.set = AsyncXHRBuffer.prototype.set; - AsyncXHRPartfileBuffer.prototype.handle_read = AsyncXHRBuffer.prototype.handle_read; - //AsyncXHRPartfileBuffer.prototype.get_block_cache = AsyncXHRBuffer.prototype.get_block_cache; - AsyncXHRPartfileBuffer.prototype.get_state = AsyncXHRBuffer.prototype.get_state; - AsyncXHRPartfileBuffer.prototype.set_state = AsyncXHRBuffer.prototype.set_state; - - /** - * Synchronous access to File, loading blocks from the input type=file - * The whole file is loaded into memory during initialisation - * - * @constructor - */ - function SyncFileBuffer(file) - { - this.file = file; - this.byteLength = file.size; - - if(file.size > (1 << 30)) - { - console.warn("SyncFileBuffer: Allocating buffer of " + (file.size >> 20) + " MB ..."); - } - - this.buffer = new ArrayBuffer(file.size); - this.onload = undefined; - this.onprogress = undefined; - } - - SyncFileBuffer.prototype.load = function() - { - this.load_next(0); - }; - - /** - * @param {number} start - */ - SyncFileBuffer.prototype.load_next = function(start) - { - /** @const */ - var PART_SIZE = 4 << 20; - - var filereader = new FileReader(); - - filereader.onload = function(e) - { - var buffer = new Uint8Array(e.target.result); - new Uint8Array(this.buffer, start).set(buffer); - this.load_next(start + PART_SIZE); - }.bind(this); - - if(this.onprogress) - { - this.onprogress({ - loaded: start, - total: this.byteLength, - lengthComputable: true, - }); - } - - if(start < this.byteLength) - { - var end = Math.min(start + PART_SIZE, this.byteLength); - var slice = this.file.slice(start, end); - filereader.readAsArrayBuffer(slice); - } - else - { - this.file = undefined; - this.onload && this.onload({ buffer: this.buffer }); - } - }; - - /** - * @param {number} start - * @param {number} len - * @param {function(!Uint8Array)} fn - */ - SyncFileBuffer.prototype.get = function(start, len, fn) - { - console.assert(start + len <= this.byteLength); - fn(new Uint8Array(this.buffer, start, len)); - }; - - /** - * @param {number} offset - * @param {!Uint8Array} slice - * @param {function()} fn - */ - SyncFileBuffer.prototype.set = function(offset, slice, fn) - { - console.assert(offset + slice.byteLength <= this.byteLength); - - new Uint8Array(this.buffer, offset, slice.byteLength).set(slice); - fn(); - }; - - SyncFileBuffer.prototype.get_buffer = function(fn) - { - fn(this.buffer); - }; - - SyncFileBuffer.prototype.get_state = function() - { - const state = []; - state[0] = this.byteLength; - state[1] = new Uint8Array(this.buffer); - return state; - }; - - SyncFileBuffer.prototype.set_state = function(state) - { - this.byteLength = state[0]; - this.buffer = state[1].slice().buffer; - }; - - /** - * Asynchronous access to File, loading blocks from the input type=file - * - * @constructor - */ - function AsyncFileBuffer(file) - { - this.file = file; - this.byteLength = file.size; - - /** @const */ - this.block_size = 256; - this.block_cache = new Map(); - this.block_cache_is_write = new Set(); - - this.onload = undefined; - this.onprogress = undefined; - } - - AsyncFileBuffer.prototype.load = function() - { - this.onload && this.onload(Object.create(null)); - }; - - /** - * @param {number} offset - * @param {number} len - * @param {function(!Uint8Array)} fn - */ - AsyncFileBuffer.prototype.get = function(offset, len, fn) - { - console.assert(offset % this.block_size === 0); - console.assert(len % this.block_size === 0); - console.assert(len); - - var block = this.get_from_cache(offset, len); - if(block) - { - fn(block); - return; - } - - var fr = new FileReader(); - - fr.onload = function(e) - { - var buffer = e.target.result; - var block = new Uint8Array(buffer); - - this.handle_read(offset, len, block); - fn(block); - }.bind(this); - - fr.readAsArrayBuffer(this.file.slice(offset, offset + len)); - }; - AsyncFileBuffer.prototype.get_from_cache = AsyncXHRBuffer.prototype.get_from_cache; - AsyncFileBuffer.prototype.set = AsyncXHRBuffer.prototype.set; - AsyncFileBuffer.prototype.handle_read = AsyncXHRBuffer.prototype.handle_read; - AsyncFileBuffer.prototype.get_state = AsyncXHRBuffer.prototype.get_state; - AsyncFileBuffer.prototype.set_state = AsyncXHRBuffer.prototype.set_state; - - AsyncFileBuffer.prototype.get_buffer = function(fn) - { - // We must load all parts, unlikely a good idea for big files - fn(); - }; - - AsyncFileBuffer.prototype.get_as_file = function(name) - { - var parts = []; - var existing_blocks = Array.from(this.block_cache.keys()).sort(function(x, y) { return x - y; }); - - var current_offset = 0; - - for(var i = 0; i < existing_blocks.length; i++) - { - var block_index = existing_blocks[i]; - var block = this.block_cache.get(block_index); - var start = block_index * this.block_size; - console.assert(start >= current_offset); - - if(start !== current_offset) - { - parts.push(this.file.slice(current_offset, start)); - current_offset = start; - } - - parts.push(block); - current_offset += block.length; - } - - if(current_offset !== this.file.size) - { - parts.push(this.file.slice(current_offset)); - } - - var file = new File(parts, name); - console.assert(file.size === this.file.size); - - return file; - }; - })(); diff --git a/src/browser/starter.js b/src/browser/starter.js index 6f256567..519b6ef6 100644 --- a/src/browser/starter.js +++ b/src/browser/starter.js @@ -392,7 +392,7 @@ V86Starter.prototype.continue_init = async function(emulator, options) if(file.buffer instanceof ArrayBuffer) { - var buffer = new SyncBuffer(file.buffer); + var buffer = new v86util.SyncBuffer(file.buffer); files_to_load.push({ name: name, loadable: buffer, @@ -496,7 +496,7 @@ V86Starter.prototype.continue_init = async function(emulator, options) if(fs_url) { - console.assert(base_url, "Filesystem: baseurl must be specified"); + dbg_assert(base_url, "Filesystem: baseurl must be specified"); var size; @@ -543,7 +543,7 @@ V86Starter.prototype.continue_init = async function(emulator, options) v86util.load_file(f.url, { done: function(result) { - put_on_settings.call(this, f.name, f.as_json ? result : new SyncBuffer(result)); + put_on_settings.call(this, f.name, f.as_json ? result : new v86util.SyncBuffer(result)); cont(index + 1); }.bind(this), progress: function progress(e) @@ -605,8 +605,8 @@ V86Starter.prototype.continue_init = async function(emulator, options) settings.fs9p.read_file(initrd_path), settings.fs9p.read_file(bzimage_path), ]); - put_on_settings.call(this, "initrd", new SyncBuffer(initrd.buffer)); - put_on_settings.call(this, "bzimage", new SyncBuffer(bzimage.buffer)); + put_on_settings.call(this, "initrd", new v86util.SyncBuffer(initrd.buffer)); + put_on_settings.call(this, "bzimage", new v86util.SyncBuffer(bzimage.buffer)); finish.call(this); } else @@ -616,7 +616,7 @@ V86Starter.prototype.continue_init = async function(emulator, options) } else { - console.assert( + dbg_assert( !options["bzimage_initrd_from_filesystem"], "bzimage_initrd_from_filesystem: Requires a filesystem"); finish.call(this); @@ -784,7 +784,7 @@ V86Starter.prototype.remove_listener = function(event, listener) */ V86Starter.prototype.restore_state = async function(state) { - console.assert(arguments.length === 1); + dbg_assert(arguments.length === 1); this.v86.restore_state(state); }; @@ -796,7 +796,7 @@ V86Starter.prototype.restore_state = async function(state) */ V86Starter.prototype.save_state = async function() { - console.assert(arguments.length === 0); + dbg_assert(arguments.length === 0); return this.v86.save_state(); }; @@ -1167,7 +1167,7 @@ V86Starter.prototype.mount_fs = async function(path, baseurl, basefs, callback) */ V86Starter.prototype.create_file = async function(file, data) { - console.assert(arguments.length === 2); + dbg_assert(arguments.length === 2); var fs = this.fs9p; if(!fs) @@ -1201,7 +1201,7 @@ V86Starter.prototype.create_file = async function(file, data) */ V86Starter.prototype.read_file = async function(file) { - console.assert(arguments.length === 1); + dbg_assert(arguments.length === 1); var fs = this.fs9p; if(!fs) @@ -1280,7 +1280,7 @@ V86Starter.prototype.automatically = function(steps) return; } - console.assert(false, step); + dbg_assert(false, step); }; run(steps); diff --git a/src/buffer.js b/src/buffer.js new file mode 100644 index 00000000..e4c0f4ad --- /dev/null +++ b/src/buffer.js @@ -0,0 +1,753 @@ +"use strict"; + +(function() +{ + v86util.SyncBuffer = SyncBuffer; + v86util.AsyncXHRBuffer = AsyncXHRBuffer; + v86util.AsyncXHRPartfileBuffer = AsyncXHRPartfileBuffer; + v86util.AsyncFileBuffer = AsyncFileBuffer; + v86util.SyncFileBuffer = SyncFileBuffer; + + // The smallest size the emulated hardware can emit + const BLOCK_SIZE = 256; + + const ASYNC_SAFE = false; + + /** + * Synchronous access to ArrayBuffer + * @constructor + */ + function SyncBuffer(buffer) + { + dbg_assert(buffer instanceof ArrayBuffer); + + this.buffer = buffer; + this.byteLength = buffer.byteLength; + this.onload = undefined; + this.onprogress = undefined; + } + + SyncBuffer.prototype.load = function() + { + this.onload && this.onload({ buffer: this.buffer }); + }; + + /** + * @param {number} start + * @param {number} len + * @param {function(!Uint8Array)} fn + */ + SyncBuffer.prototype.get = function(start, len, fn) + { + dbg_assert(start + len <= this.byteLength); + fn(new Uint8Array(this.buffer, start, len)); + }; + + /** + * @param {number} start + * @param {!Uint8Array} slice + * @param {function()} fn + */ + SyncBuffer.prototype.set = function(start, slice, fn) + { + dbg_assert(start + slice.byteLength <= this.byteLength); + + new Uint8Array(this.buffer, start, slice.byteLength).set(slice); + fn(); + }; + + /** + * @param {function(!ArrayBuffer)} fn + */ + SyncBuffer.prototype.get_buffer = function(fn) + { + fn(this.buffer); + }; + + SyncBuffer.prototype.get_state = function() + { + const state = []; + state[0] = this.byteLength; + state[1] = new Uint8Array(this.buffer); + return state; + }; + + SyncBuffer.prototype.set_state = function(state) + { + this.byteLength = state[0]; + this.buffer = state[1].slice().buffer; + }; + + /** + * Asynchronous access to ArrayBuffer, loading blocks lazily as needed, + * using the `Range: bytes=...` header + * + * @constructor + * @param {string} filename Name of the file to download + * @param {number|undefined} size + */ + function AsyncXHRBuffer(filename, size) + { + this.filename = filename; + + this.byteLength = size; + + this.block_cache = new Map(); + this.block_cache_is_write = new Set(); + + this.onload = undefined; + this.onprogress = undefined; + } + + AsyncXHRBuffer.prototype.load = function() + { + if(this.byteLength !== undefined) + { + this.onload && this.onload(Object.create(null)); + return; + } + + // Determine the size using a request + + determine_size(this.filename, (error, size) => + { + if(error) + { + throw new Error("Cannot use: " + this.filename + ". " + error); + } + else + { + dbg_assert(size >= 0); + this.byteLength = size; + this.onload && this.onload(Object.create(null)); + } + }); + }; + + /** + * @param {number} offset + * @param {number} len + * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + */ + AsyncXHRBuffer.prototype.get_from_cache = function(offset, len) + { + var number_of_blocks = len / BLOCK_SIZE; + var block_index = offset / BLOCK_SIZE; + + for(var i = 0; i < number_of_blocks; i++) + { + var block = this.block_cache.get(block_index + i); + + if(!block) + { + return; + } + } + + if(number_of_blocks === 1) + { + return this.block_cache.get(block_index); + } + else + { + var result = new Uint8Array(len); + for(var i = 0; i < number_of_blocks; i++) + { + result.set(this.block_cache.get(block_index + i), i * BLOCK_SIZE); + } + return result; + } + }; + + /** + * @param {number} offset + * @param {number} len + * @param {function(!Uint8Array)} fn + */ + AsyncXHRBuffer.prototype.get = function(offset, len, fn) + { + dbg_assert(offset + len <= this.byteLength); + dbg_assert(offset % BLOCK_SIZE === 0); + dbg_assert(len % BLOCK_SIZE === 0); + dbg_assert(len); + + var block = this.get_from_cache(offset, len); + if(block) + { + if(ASYNC_SAFE) + { + setTimeout(fn.bind(this, block), 0); + } + else + { + fn(block); + } + return; + } + + v86util.load_file(this.filename, { + done: function done(buffer) + { + var block = new Uint8Array(buffer); + this.handle_read(offset, len, block); + fn(block); + }.bind(this), + range: { start: offset, length: len }, + }); + }; + + /** + * Relies on this.byteLength and this.block_cache + * + * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + * + * @param {number} start + * @param {!Uint8Array} data + * @param {function()} fn + */ + AsyncXHRBuffer.prototype.set = function(start, data, fn) + { + var len = data.length; + dbg_assert(start + data.byteLength <= this.byteLength); + dbg_assert(start % BLOCK_SIZE === 0); + dbg_assert(len % BLOCK_SIZE === 0); + dbg_assert(len); + + var start_block = start / BLOCK_SIZE; + var block_count = len / BLOCK_SIZE; + + for(var i = 0; i < block_count; i++) + { + var block = this.block_cache.get(start_block + i); + + if(block === undefined) + { + block = new Uint8Array(BLOCK_SIZE); + this.block_cache.set(start_block + i, block); + } + + var data_slice = data.subarray(i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE); + block.set(data_slice); + + dbg_assert(block.byteLength === data_slice.length); + + this.block_cache_is_write.add(start_block + i); + } + + fn(); + }; + + /** + * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + * @param {number} offset + * @param {number} len + * @param {!Uint8Array} block + */ + AsyncXHRBuffer.prototype.handle_read = function(offset, len, block) + { + // Used by AsyncXHRBuffer, AsyncXHRPartfileBuffer and AsyncFileBuffer + // Overwrites blocks from the original source that have been written since + + var start_block = offset / BLOCK_SIZE; + var block_count = len / BLOCK_SIZE; + + for(var i = 0; i < block_count; i++) + { + const cached_block = this.block_cache.get(start_block + i); + + if(cached_block) + { + block.set(cached_block, i * BLOCK_SIZE); + } + else if(this.cache_reads) + { + const cached = new Uint8Array(BLOCK_SIZE); + cached.set(block.subarray(i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE)); + this.block_cache.set(start_block + i, cached); + } + } + }; + + AsyncXHRBuffer.prototype.get_buffer = function(fn) + { + // We must download all parts, unlikely a good idea for big files + fn(); + }; + + ///** + // * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + // */ + //AsyncXHRBuffer.prototype.get_block_cache = function() + //{ + // var count = Object.keys(this.block_cache).length; + + // var buffer = new Uint8Array(count * BLOCK_SIZE); + // var indices = []; + + // var i = 0; + // for(var index of Object.keys(this.block_cache)) + // { + // var block = this.block_cache.get(index); + // dbg_assert(block.length === BLOCK_SIZE); + // index = +index; + // indices.push(index); + // buffer.set( + // block, + // i * BLOCK_SIZE + // ); + // i++; + // } + + // return { + // buffer, + // indices, + // block_size: BLOCK_SIZE, + // }; + //}; + + /** + * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + */ + AsyncXHRBuffer.prototype.get_state = function() + { + const state = []; + const block_cache = []; + + for(let [index, block] of this.block_cache) + { + dbg_assert(isFinite(index)); + if(this.block_cache_is_write.has(index)) + { + block_cache.push([index, block]); + } + } + + state[0] = block_cache; + return state; + }; + + /** + * @this {AsyncXHRBuffer|AsyncXHRPartfileBuffer|AsyncFileBuffer} + */ + AsyncXHRBuffer.prototype.set_state = function(state) + { + const block_cache = state[0]; + this.block_cache.clear(); + this.block_cache_is_write.clear(); + + for(let [index, block] of block_cache) + { + dbg_assert(isFinite(index)); + this.block_cache.set(index, block); + this.block_cache_is_write.add(index); + } + }; + + /** + * Asynchronous access to ArrayBuffer, loading blocks lazily as needed, + * downloading files named filename-%d-%d.ext (where the %d are start and end offset). + * Or, if partfile_alt_format is set, filename-%08d.ext (where %d is the part number, compatible with gnu split). + * + * @constructor + * @param {string} filename Name of the file to download + * @param {number|undefined} size + * @param {number|undefined} fixed_chunk_size + * @param {boolean|undefined} partfile_alt_format + */ + function AsyncXHRPartfileBuffer(filename, size, fixed_chunk_size, partfile_alt_format) + { + const parts = filename.match(/(.*)(\..*)/); + + if(parts) + { + this.basename = parts[1]; + this.extension = parts[2]; + } + else + { + this.basename = filename; + this.extension = ""; + } + + this.block_cache = new Map(); + this.block_cache_is_write = new Set(); + + this.byteLength = size; + this.fixed_chunk_size = fixed_chunk_size; + this.partfile_alt_format = !!partfile_alt_format; + + this.cache_reads = !!fixed_chunk_size; // TODO: could also be useful in other cases (needs testing) + + this.onload = undefined; + this.onprogress = undefined; + } + + AsyncXHRPartfileBuffer.prototype.load = function() + { + if(this.byteLength !== undefined) + { + this.onload && this.onload(Object.create(null)); + return; + } + dbg_assert(false); + this.onload && this.onload(Object.create(null)); + }; + + /** + * @param {number} offset + * @param {number} len + * @param {function(!Uint8Array)} fn + */ + AsyncXHRPartfileBuffer.prototype.get = function(offset, len, fn) + { + dbg_assert(offset + len <= this.byteLength); + dbg_assert(offset % BLOCK_SIZE === 0); + dbg_assert(len % BLOCK_SIZE === 0); + dbg_assert(len); + + const block = this.get_from_cache(offset, len); + + if(block) + { + if(ASYNC_SAFE) + { + setTimeout(fn.bind(this, block), 0); + } + else + { + fn(block); + } + return; + } + + if(this.fixed_chunk_size) + { + const start_index = Math.floor(offset / this.fixed_chunk_size); + const m_offset = offset - start_index * this.fixed_chunk_size; + dbg_assert(m_offset >= 0); + const total_count = Math.ceil((m_offset + len) / this.fixed_chunk_size); + const blocks = new Uint8Array(total_count * this.fixed_chunk_size); + let finished = 0; + + for(let i = 0; i < total_count; i++) + { + const offset = (start_index + i) * this.fixed_chunk_size; + + const part_filename = + this.partfile_alt_format ? + // matches output of gnu split: + // split -b 512 -a8 -d --additional-suffix .img w95.img w95- + this.basename + "-" + (start_index + i + "").padStart(8, "0") + this.extension + : + this.basename + "-" + offset + "-" + (offset + this.fixed_chunk_size) + this.extension; + + // XXX: unnecessary allocation + const block = this.get_from_cache(offset, this.fixed_chunk_size); + + if(block) + { + const cur = i * this.fixed_chunk_size; + blocks.set(block, cur); + finished++; + if(finished === total_count) + { + const tmp_blocks = blocks.subarray(m_offset, m_offset + len); + fn(tmp_blocks); + } + } + else + { + v86util.load_file(part_filename, { + done: function done(buffer) + { + const cur = i * this.fixed_chunk_size; + const block = new Uint8Array(buffer); + this.handle_read((start_index + i) * this.fixed_chunk_size, this.fixed_chunk_size|0, block); + blocks.set(block, cur); + finished++; + if(finished === total_count) + { + const tmp_blocks = blocks.subarray(m_offset, m_offset + len); + fn(tmp_blocks); + } + }.bind(this), + }); + } + } + } + else + { + const part_filename = this.basename + "-" + offset + "-" + (offset + len) + this.extension; + + v86util.load_file(part_filename, { + done: function done(buffer) + { + dbg_assert(buffer.byteLength === len); + var block = new Uint8Array(buffer); + this.handle_read(offset, len, block); + fn(block); + }.bind(this), + }); + } + }; + + AsyncXHRPartfileBuffer.prototype.get_from_cache = AsyncXHRBuffer.prototype.get_from_cache; + AsyncXHRPartfileBuffer.prototype.set = AsyncXHRBuffer.prototype.set; + AsyncXHRPartfileBuffer.prototype.handle_read = AsyncXHRBuffer.prototype.handle_read; + //AsyncXHRPartfileBuffer.prototype.get_block_cache = AsyncXHRBuffer.prototype.get_block_cache; + AsyncXHRPartfileBuffer.prototype.get_state = AsyncXHRBuffer.prototype.get_state; + AsyncXHRPartfileBuffer.prototype.set_state = AsyncXHRBuffer.prototype.set_state; + + /** + * Synchronous access to File, loading blocks from the input type=file + * The whole file is loaded into memory during initialisation + * + * @constructor + */ + function SyncFileBuffer(file) + { + this.file = file; + this.byteLength = file.size; + + if(file.size > (1 << 30)) + { + console.warn("SyncFileBuffer: Allocating buffer of " + (file.size >> 20) + " MB ..."); + } + + this.buffer = new ArrayBuffer(file.size); + + this.onload = undefined; + this.onprogress = undefined; + } + + SyncFileBuffer.prototype.load = function() + { + this.load_next(0); + }; + + /** + * @param {number} start + */ + SyncFileBuffer.prototype.load_next = function(start) + { + /** @const */ + var PART_SIZE = 4 << 20; + + var filereader = new FileReader(); + + filereader.onload = function(e) + { + var buffer = new Uint8Array(e.target.result); + new Uint8Array(this.buffer, start).set(buffer); + this.load_next(start + PART_SIZE); + }.bind(this); + + if(this.onprogress) + { + this.onprogress({ + loaded: start, + total: this.byteLength, + lengthComputable: true, + }); + } + + if(start < this.byteLength) + { + var end = Math.min(start + PART_SIZE, this.byteLength); + var slice = this.file.slice(start, end); + filereader.readAsArrayBuffer(slice); + } + else + { + this.file = undefined; + this.onload && this.onload({ buffer: this.buffer }); + } + }; + + /** + * @param {number} start + * @param {number} len + * @param {function(!Uint8Array)} fn + */ + SyncFileBuffer.prototype.get = function(start, len, fn) + { + dbg_assert(start + len <= this.byteLength); + fn(new Uint8Array(this.buffer, start, len)); + }; + + /** + * @param {number} offset + * @param {!Uint8Array} slice + * @param {function()} fn + */ + SyncFileBuffer.prototype.set = function(offset, slice, fn) + { + dbg_assert(offset + slice.byteLength <= this.byteLength); + + new Uint8Array(this.buffer, offset, slice.byteLength).set(slice); + fn(); + }; + + SyncFileBuffer.prototype.get_buffer = function(fn) + { + fn(this.buffer); + }; + + SyncFileBuffer.prototype.get_state = function() + { + const state = []; + state[0] = this.byteLength; + state[1] = new Uint8Array(this.buffer); + return state; + }; + + SyncFileBuffer.prototype.set_state = function(state) + { + this.byteLength = state[0]; + this.buffer = state[1].slice().buffer; + }; + + /** + * Asynchronous access to File, loading blocks from the input type=file + * + * @constructor + */ + function AsyncFileBuffer(file) + { + this.file = file; + this.byteLength = file.size; + + this.block_cache = new Map(); + this.block_cache_is_write = new Set(); + + this.onload = undefined; + this.onprogress = undefined; + } + + AsyncFileBuffer.prototype.load = function() + { + this.onload && this.onload(Object.create(null)); + }; + + /** + * @param {number} offset + * @param {number} len + * @param {function(!Uint8Array)} fn + */ + AsyncFileBuffer.prototype.get = function(offset, len, fn) + { + dbg_assert(offset % BLOCK_SIZE === 0); + dbg_assert(len % BLOCK_SIZE === 0); + dbg_assert(len); + + var block = this.get_from_cache(offset, len); + if(block) + { + fn(block); + return; + } + + var fr = new FileReader(); + + fr.onload = function(e) + { + var buffer = e.target.result; + var block = new Uint8Array(buffer); + + this.handle_read(offset, len, block); + fn(block); + }.bind(this); + + fr.readAsArrayBuffer(this.file.slice(offset, offset + len)); + }; + AsyncFileBuffer.prototype.get_from_cache = AsyncXHRBuffer.prototype.get_from_cache; + AsyncFileBuffer.prototype.set = AsyncXHRBuffer.prototype.set; + AsyncFileBuffer.prototype.handle_read = AsyncXHRBuffer.prototype.handle_read; + AsyncFileBuffer.prototype.get_state = AsyncXHRBuffer.prototype.get_state; + AsyncFileBuffer.prototype.set_state = AsyncXHRBuffer.prototype.set_state; + + AsyncFileBuffer.prototype.get_buffer = function(fn) + { + // We must load all parts, unlikely a good idea for big files + fn(); + }; + + AsyncFileBuffer.prototype.get_as_file = function(name) + { + var parts = []; + var existing_blocks = Array.from(this.block_cache.keys()).sort(function(x, y) { return x - y; }); + + var current_offset = 0; + + for(var i = 0; i < existing_blocks.length; i++) + { + var block_index = existing_blocks[i]; + var block = this.block_cache.get(block_index); + var start = block_index * BLOCK_SIZE; + dbg_assert(start >= current_offset); + + if(start !== current_offset) + { + parts.push(this.file.slice(current_offset, start)); + current_offset = start; + } + + parts.push(block); + current_offset += block.length; + } + + if(current_offset !== this.file.size) + { + parts.push(this.file.slice(current_offset)); + } + + var file = new File(parts, name); + dbg_assert(file.size === this.file.size); + + return file; + }; + + if(typeof XMLHttpRequest === "undefined") + { + var determine_size = function(path, cb) + { + require("fs")["stat"](path, (err, stats) => + { + if(err) + { + cb(err); + } + else + { + cb(null, stats.size); + } + }); + }; + } + else + { + var determine_size = function(url, cb) + { + v86util.load_file(url, { + done: (buffer, http) => + { + var header = http.getResponseHeader("Content-Range") || ""; + var match = header.match(/\/(\d+)\s*$/); + + if(match) + { + cb(null, +match[1]); + } + else + { + const error = "`Range: bytes=...` header not supported (Got `" + header + "`)"; + cb(error); + } + }, + headers: { + Range: "bytes=0-0", + } + }); + }; + } +})(); diff --git a/src/lib.js b/src/lib.js index 42bd3778..9350032c 100644 --- a/src/lib.js +++ b/src/lib.js @@ -154,72 +154,6 @@ else dbg_assert(false, "Unsupported platform: No cryptographic random values"); } - -/** - * Synchronous access to ArrayBuffer - * @constructor - */ -function SyncBuffer(buffer) -{ - dbg_assert(buffer instanceof ArrayBuffer); - - this.buffer = buffer; - this.byteLength = buffer.byteLength; - this.onload = undefined; - this.onprogress = undefined; -} - -SyncBuffer.prototype.load = function() -{ - this.onload && this.onload({ buffer: this.buffer }); -}; - -/** - * @param {number} start - * @param {number} len - * @param {function(!Uint8Array)} fn - */ -SyncBuffer.prototype.get = function(start, len, fn) -{ - dbg_assert(start + len <= this.byteLength); - fn(new Uint8Array(this.buffer, start, len)); -}; - -/** - * @param {number} start - * @param {!Uint8Array} slice - * @param {function()} fn - */ -SyncBuffer.prototype.set = function(start, slice, fn) -{ - dbg_assert(start + slice.byteLength <= this.byteLength); - - new Uint8Array(this.buffer, start, slice.byteLength).set(slice); - fn(); -}; - -/** - * @param {function(!ArrayBuffer)} fn - */ -SyncBuffer.prototype.get_buffer = function(fn) -{ - fn(this.buffer); -}; - -SyncBuffer.prototype.get_state = function() -{ - const state = []; - state[0] = this.byteLength; - state[1] = new Uint8Array(this.buffer); - return state; -}; - -SyncBuffer.prototype.set_state = function(state) -{ - this.byteLength = state[0]; - this.buffer = state[1].slice().buffer; -}; - (function() { if(typeof Math.clz32 === "function" && Math.clz32(0) === 32 && @@ -571,7 +505,7 @@ v86util.Bitmap = function(length_or_buffer) } else { - console.assert(false); + dbg_assert(false, "v86util.Bitmap: Invalid argument"); } }; diff --git a/src/sb16.js b/src/sb16.js index b5ef938c..37a6b084 100644 --- a/src/sb16.js +++ b/src/sb16.js @@ -150,7 +150,7 @@ function SB16(cpu, bus) this.dma_buffer_uint8 = new Uint8Array(this.dma_buffer); this.dma_buffer_int16 = new Int16Array(this.dma_buffer); this.dma_buffer_uint16 = new Uint16Array(this.dma_buffer); - this.dma_syncbuffer = new SyncBuffer(this.dma_buffer); + this.dma_syncbuffer = new v86util.SyncBuffer(this.dma_buffer); this.dma_waiting_transfer = false; this.dma_paused = false; this.sampling_rate = 22050; @@ -399,7 +399,7 @@ SB16.prototype.set_state = function(state) this.dma_buffer_int8 = new Int8Array(this.dma_buffer); this.dma_buffer_int16 = new Int16Array(this.dma_buffer); this.dma_buffer_uint16 = new Uint16Array(this.dma_buffer); - this.dma_syncbuffer = new SyncBuffer(this.dma_buffer); + this.dma_syncbuffer = new v86util.SyncBuffer(this.dma_buffer); if(this.dma_paused) {