rust restructuring, some refactoring

.gitignore

@@ -19,5 +19,4 @@ images/
 node_modules/
 package-lock.json
 profile*.json
-src/wasmgen/target
-src/wasmgen/Cargo.lock
+Cargo.lock

.travis-run-lint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -e
+make jshint rustfmt clang-tidy

.travis-run-rust.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 set -e
-make jshint
+make rust-test

.travis-run-wasmgen.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-set -e
-make wasmgen-test

.travis.yml

@@ -14,8 +14,8 @@ env:
     - TEST_SUITE=integration
     - TEST_SUITE=unit-qemu
     - TEST_SUITE=nasm
-    - TEST_SUITE=codegen
-    - TEST_SUITE=jshint
+    - TEST_SUITE=rust
+    - TEST_SUITE=lint
     - TEST_SUITE=jit-paging
     - TEST_SUITE=expect
     - TEST_SUITE=devices

Cargo.toml

@@ -1,7 +1,6 @@
 [package]
-name = "wasmgen"
+name = "v86oxide"
 version = "0.1.0"
-authors = ["Awal Garg <awalgarg@gmail.com>"]
 publish = false
 
 [dependencies]
@@ -9,11 +8,14 @@ lazy_static = "1.0"
 
 [lib]
 crate-type = ["cdylib"]
+path = "src/rust/lib.rs"
 
 [profile.dev]
+lto = false
 panic = "abort"
 
 [profile.release]
 lto = true
 incremental = false
 panic = "abort"
+

Makefile

@@ -23,9 +23,9 @@ ifeq ($(JIT_ALWAYS),)
 JIT_ALWAYS=false
 endif
 
-all: build/v86_all.js build/v86.wasm build/wasmgen.wasm
+all: build/v86_all.js build/libv86.js build/v86.wasm build/v86oxide.wasm
+all-debug: build/libv86-debug.js build/v86-debug.wasm build/v86oxide-debug.wasm
 browser: build/v86_all.js
-wasm: build/v86.wasm build/wasmgen.wasm
 
 # Used for nodejs builds and in order to profile code.
 # `debug` gives identifiers a readable name, make sure it doesn't have any side effects.
@@ -104,6 +104,11 @@ CC_FLAGS=\
 		-s WASM=1 \
 		-s SIDE_MODULE=1
 
+CARGO_FLAGS=\
+			--target wasm32-unknown-unknown \
+			--target-dir build/ \
+			-- -Clink-args="--import-memory"
+
 CORE_FILES=const.js config.js io.js main.js lib.js coverage.js ide.js pci.js floppy.js \
 	   memory.js dma.js pit.js vga.js ps2.js pic.js rtc.js uart.js hpet.js acpi.js apic.js ioapic.js \
 	   state.js ne2k.js virtio.js bus.js log.js \
@@ -114,6 +119,8 @@ BROWSER_FILES=screen.js \
 		  keyboard.js mouse.js serial.js \
 		  network.js lib.js starter.js worker_bus.js dummy_screen.js print_stats.js
 
+RUST_FILES=$(shell find src/rust/ -name '*.rs')
+
 CORE_FILES:=$(addprefix src/,$(CORE_FILES))
 LIB_FILES:=$(addprefix lib/,$(LIB_FILES))
 BROWSER_FILES:=$(addprefix src/browser/,$(BROWSER_FILES))
@@ -223,19 +230,19 @@ build/v86-debug.wasm: src/native/*.c src/native/*.h src/native/codegen/*.c src/n
 		-o build/v86-debug.wasm
 	ls -lh build/v86-debug.wasm
 
-build/wasmgen.wasm: src/wasmgen/src/*.rs src/wasmgen/Cargo.toml
+build/v86oxide.wasm: $(RUST_FILES) Cargo.toml
 	mkdir -p build/
-	-ls -lh build/wasmgen.wasm
-	(cd src/wasmgen && cargo +nightly rustc --release --target wasm32-unknown-unknown -- -Clink-args="--import-memory")
-	cp src/wasmgen/target/wasm32-unknown-unknown/release/wasmgen.wasm build/wasmgen.wasm
-	ls -lh build/wasmgen.wasm
+	-ls -lh build/v86oxide.wasm
+	cargo +nightly rustc --release $(CARGO_FLAGS)
+	cp build/wasm32-unknown-unknown/release/v86oxide.wasm build/v86oxide.wasm
+	ls -lh build/v86oxide.wasm
 
-build/wasmgen-debug.wasm: src/wasmgen/src/*.rs src/wasmgen/Cargo.toml
+build/v86oxide-debug.wasm: $(RUST_FILES) Cargo.toml
 	mkdir -p build/
-	-ls -lh build/wasmgen-debug.wasm
-	(cd src/wasmgen && cargo +nightly rustc --target wasm32-unknown-unknown -- -Clink-args="--import-memory")
-	cp src/wasmgen/target/wasm32-unknown-unknown/debug/wasmgen.wasm build/wasmgen-debug.wasm
-	ls -lh build/wasmgen-debug.wasm
+	-ls -lh build/v86oxide-debug.wasm
+	cargo +nightly rustc $(CARGO_FLAGS)
+	cp build/wasm32-unknown-unknown/debug/v86oxide.wasm build/v86oxide-debug.wasm
+	ls -lh build/v86oxide-debug.wasm
 
 clean:
 	-rm build/libv86.js
@@ -243,8 +250,8 @@ clean:
 	-rm build/v86_all.js
 	-rm build/v86.wasm
 	-rm build/v86-debug.wasm
-	-rm build/wasmgen.wasm
-	-rm build/wasmgen-debug.wasm
+	-rm build/v86oxide.wasm
+	-rm build/v86oxide-debug.wasm
 	-rm $(INSTRUCTION_TABLES)
 	-rm $(addsuffix .bak,$(INSTRUCTION_TABLES))
 	-rm $(addsuffix .diff,$(INSTRUCTION_TABLES))
@@ -275,44 +282,45 @@ $(CLOSURE):
 	mv $(CLOSURE_DIR)/*.jar $(CLOSURE)
 	rm $(CLOSURE_DIR)/compiler-latest.zip
 
-tests: build/libv86.js build/v86.wasm build/wasmgen.wasm
+tests: build/libv86.js build/v86.wasm build/v86oxide.wasm
 	./tests/full/run.js
 
-nasmtests: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+nasmtests: all-debug
 	$(MAKE) -C $(NASM_TEST_DIR) all
 	$(NASM_TEST_DIR)/gen_fixtures.js
 	$(NASM_TEST_DIR)/run.js
 
-nasmtests-force-jit: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+nasmtests-force-jit: all-debug
 	$(MAKE) -C $(NASM_TEST_DIR) all
 	$(NASM_TEST_DIR)/gen_fixtures.js
 	$(NASM_TEST_DIR)/run.js --force-jit
 
-jitpagingtests: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+jitpagingtests: all-debug
 	$(MAKE) -C tests/jit-paging test-jit
 	./tests/jit-paging/run.js
 
-qemutests: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+qemutests: all-debug
 	$(MAKE) -C tests/qemu test-i386
 	./tests/qemu/run.js > /tmp/v86-test-result
 	#./tests/qemu/test-i386 > /tmp/v86-test-reference
 	./tests/qemu/run-qemu.js > /tmp/v86-test-reference
 	diff /tmp/v86-test-result /tmp/v86-test-reference
 
-kvm-unit-test: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+kvm-unit-test: all-debug
 	(cd tests/kvm-unit-tests && ./configure)
 	$(MAKE) -C tests/kvm-unit-tests
 	tests/kvm-unit-tests/run.js tests/kvm-unit-tests/x86/realmode.flat
 
-expect-tests: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm build/libwabt.js
+expect-tests: all-debug build/libwabt.js
 	make -C tests/expect/tests
 	./tests/expect/run.js
 
-devices-test: build/libv86-debug.js build/v86-debug.wasm build/wasmgen-debug.wasm
+devices-test: all-debug
 	./tests/devices/virtio_9p.js
 
-wasmgen-test:
-	(cd src/wasmgen && env RUST_BACKTRACE=full RUST_TEST_THREADS=1 cargo test -- --nocapture)
+rust-test:
+	env RUST_BACKTRACE=full RUST_TEST_THREADS=1 cargo test --target-dir build -- --nocapture
+	./tests/rust/verify-wasmgen-dummy-output.js
 
 covreport:
 	mkdir -p $(COVERAGE_DIR)/build/
@@ -324,6 +332,9 @@ node_modules/.bin/jshint:
 jshint: node_modules/.bin/jshint
 	./node_modules/.bin/jshint --config=./.jshint.json src tests gen
 
+rustfmt: $(RUST_FILES)
+	cargo fmt --all -- --write-mode check
+
 build/capstone-x86.min.js:
 	mkdir -p build
 	wget -P build https://github.com/AlexAltea/capstone.js/releases/download/v3.0.5-rc1/capstone-x86.min.js

docker/test-image/Dockerfile

@@ -12,4 +12,6 @@ RUN \
         rm ./rustup.sh && \
         export PATH="$HOME/.cargo/bin:$PATH" && \
         rustup toolchain install nightly && \
-        rustup target add wasm32-unknown-unknown --toolchain nightly
+        rustup target add wasm32-unknown-unknown --toolchain nightly && \
+        rustup component add rustfmt-preview --toolchain nightly
+

src/browser/starter.js

@@ -260,11 +260,11 @@ function V86Starter(options)
         "NaN": NaN,
     };
 
-    const wasmgen_mem = new WebAssembly.Memory({ "initial": 100 });
-    const wasmgen_externs = {
-        "memory": wasmgen_mem,
+    const v86oxide_mem = new WebAssembly.Memory({ "initial": 100 });
+    const v86oxide_externs = {
+        "memory": v86oxide_mem,
         "log_from_wasm": function(offset, len) {
-            const str = v86util.read_sized_string_from_mem(wasmgen_mem, offset, len);
+            const str = v86util.read_sized_string_from_mem(v86oxide_mem, offset, len);
             dbg_log(str, LOG_CPU);
         },
         "abort": function() {
@@ -273,20 +273,20 @@ function V86Starter(options)
     };
 
     let wasm_file = DEBUG ? "v86-debug.wasm" : "v86.wasm";
-    let wasmgen_bin = DEBUG ? "wasmgen-debug.wasm" : "wasmgen.wasm";
+    let v86oxide_bin = DEBUG ? "v86oxide-debug.wasm" : "v86oxide.wasm";
 
     if(typeof window === "undefined" && typeof __dirname === "string")
     {
         wasm_file = __dirname + "/" + wasm_file;
-        wasmgen_bin = __dirname + "/" + wasmgen_bin;
+        v86oxide_bin = __dirname + "/" + v86oxide_bin;
     }
     else
     {
         wasm_file = "build/" + wasm_file;
-        wasmgen_bin = "build/" + wasmgen_bin;
+        v86oxide_bin = "build/" + v86oxide_bin;
     }
 
-    const wasmgen_exports = [
+    const v86oxide_exports = [
         "wg_get_code_section",
         "wg_get_instruction_body",
         "wg_commit_instruction_body_to_cs",
@@ -339,13 +339,13 @@ function V86Starter(options)
         "wg_load_aligned_u16_from_stack",
     ];
 
-    v86util.minimal_load_wasm(wasmgen_bin, { "env": wasmgen_externs }, (wasmgen) => {
-        for(const fn_name of wasmgen_exports)
+    v86util.minimal_load_wasm(v86oxide_bin, { "env": v86oxide_externs }, (v86oxide) => {
+        for(const fn_name of v86oxide_exports)
         {
-            dbg_assert(typeof wasmgen.exports[fn_name] === "function", `Function ${fn_name} not found in wasmgen exports`);
-            wasm_shared_funcs[`_${fn_name}`] = wasmgen.exports[fn_name];
+            dbg_assert(typeof v86oxide.exports[fn_name] === "function", `Function ${fn_name} not found in v86oxide exports`);
+            wasm_shared_funcs[`_${fn_name}`] = v86oxide.exports[fn_name];
         }
-        wasmgen.exports["setup"]();
+        v86oxide.exports["wg_setup"]();
 
     //XXX: fix indentation break
 
@@ -359,7 +359,7 @@ function V86Starter(options)
             mem8 = new Uint8Array(mem);
             wm.instance.exports["__post_instantiate"]();
             coverage_logger.init(wm);
-            emulator = this.v86 = new v86(this.emulator_bus, wm, wasmgen, coverage_logger);
+            emulator = this.v86 = new v86(this.emulator_bus, wm, v86oxide, coverage_logger);
             cpu = emulator.cpu;
 
     // XXX: Leaving unindented to minimize diff; still a part of the cb to load_wasm!

src/cpu.js

@@ -11,10 +11,10 @@ var CPU_LOG_VERBOSE = false;
 
 
 /** @constructor */
-function CPU(bus, wm, wasmgen, coverage_logger)
+function CPU(bus, wm, v86oxide, coverage_logger)
 {
     this.wm = wm;
-    this.wasmgen = wasmgen;
+    this.v86oxide = v86oxide;
     this.coverage_logger = coverage_logger;
     this.wasm_patch(wm);
     this.create_jit_imports();
@@ -217,10 +217,10 @@ function CPU(bus, wm, wasmgen, coverage_logger)
 
 CPU.prototype.wasmgen_get_module_code = function()
 {
-    const ptr = this.wasmgen.exports["wg_get_op_ptr"]();
-    const len = this.wasmgen.exports["wg_get_op_len"]();
+    const ptr = this.v86oxide.exports["wg_get_op_ptr"]();
+    const len = this.v86oxide.exports["wg_get_op_len"]();
 
-    const output_buffer_view = new Uint8Array(this.wasmgen.memory.buffer, ptr, len);
+    const output_buffer_view = new Uint8Array(this.v86oxide.memory.buffer, ptr, len);
     return output_buffer_view;
 };
 

src/rust/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+mod dbg;
+
+#[macro_use]
+mod util;
+
+pub mod wasmgen;
+

src/rust/wasmgen/c_api.rs

@@ -1,6 +1,7 @@
-use ::util::PackedStr;
+use util::PackedStr;
 
-pub use ::module_init::{ setup, get_module };
+pub use wasmgen::module_init::wg_setup;
+use wasmgen::module_init::get_module;
 
 #[no_mangle]
 pub fn wg_get_code_section() -> *mut Vec<u8> {
@@ -53,14 +54,14 @@ pub fn wg_commit_instruction_body_to_cs() {
 mod tests {
     use std::io::prelude::*;
     use std::fs::File;
-    use ::c_api::*;
-    use ::util::*;
-    use ::wasm_util::*;
-    use ::module_init::*;
+    use util::*;
+    use wasmgen::c_api::*;
+    use wasmgen::wasm_util::*;
+    use wasmgen::module_init::*;
 
     #[test]
     fn c_api_test() {
-        setup();
+        wg_setup();
         let m = get_module();
         let cs = &mut get_module().cs;
         let instruction_body = &mut get_module().instruction_body;
@@ -73,19 +74,19 @@ mod tests {
 
         wg_push_i32(cs, 2);
         wg_call_fn(instruction_body, m.get_fn_idx(pack_str("baz"), FN1_RET_TYPE_INDEX));
-        wg_drop(instruction_body);
+        wg_call_fn(instruction_body, m.get_fn_idx(pack_str("foo"), FN1_TYPE_INDEX));
 
         wg_commit_instruction_body_to_cs();
 
-        wg_finish(1);
+        wg_finish(0);
 
         let op_ptr = wg_get_op_ptr();
         let op_len = wg_get_op_len();
         dbg_log!("op_ptr: {:?}, op_len: {:?}", op_ptr, op_len);
 
-        // XXX: move file path
-        let mut f = File::create("c_api_test.wasm").expect("creating c_api_test.wasm");
-        f.write_all(&get_module().op).expect("write c_api_test.wasm");
+        let mut f = File::create("build/wg_dummy_output.wasm").expect("creating wg_dummy_output.wasm");
+        f.write_all(&get_module().op).expect("write wg_dummy_output.wasm");
     }
 
 }
+

src/rust/wasmgen/mod.rs

@@ -1,11 +1,6 @@
-#[macro_use]
-mod dbg;
-
-#[macro_use]
-mod util;
-
 mod wasm_opcodes;
 mod module_init;
 
 pub mod c_api;
 pub mod wasm_util;
+

src/rust/wasmgen/module_init.rs

@@ -1,20 +1,27 @@
 use std::ptr::NonNull;
 use std::mem;
 
-use ::util::{
+use util::{
     PackedStr, unpack_str,
     write_fixed_leb16_at_idx, write_fixed_leb32_at_idx, write_leb_u32,
-    SafeToU8, SafeToU16, SafeToI32,
+    SafeToU8, SafeToU16,
 };
-use ::wasm_opcodes as op;
+use wasmgen::wasm_opcodes as op;
 
+#[allow(dead_code)]
 pub const FN0_TYPE_INDEX: u8 = 0;
+#[allow(dead_code)]
 pub const FN1_TYPE_INDEX: u8 = 1;
+#[allow(dead_code)]
 pub const FN2_TYPE_INDEX: u8 = 2;
+#[allow(dead_code)]
 pub const FN3_TYPE_INDEX: u8 = 3;
 
+#[allow(dead_code)]
 pub const FN0_RET_TYPE_INDEX: u8 = 4;
+#[allow(dead_code)]
 pub const FN1_RET_TYPE_INDEX: u8 = 5;
+#[allow(dead_code)]
 pub const FN2_RET_TYPE_INDEX: u8 = 6;
 
 pub const NR_FN_TYPE_INDEXES: u8 = 7;
@@ -24,7 +31,7 @@ static mut MODULE_PTR: NonNull<WasmBuilder> = unsafe {
 };
 
 #[no_mangle]
-pub fn setup() {
+pub fn wg_setup() {
     let wm = Box::new(WasmBuilder::new());
     unsafe {
         MODULE_PTR = NonNull::new(Box::into_raw(wm)).expect("assigning module ptr");
@@ -32,7 +39,6 @@ pub fn setup() {
     get_module().init();
 }
 
-#[no_mangle]
 pub fn get_module<'a>() -> &'a mut WasmBuilder {
     unsafe {
         MODULE_PTR.as_mut()
@@ -301,15 +307,12 @@ impl WasmBuilder {
     }
 
     pub fn get_fn_idx(&mut self, fn_name: PackedStr, type_index: u8) -> u16 {
-        dbg_log!("getting fn idx for '{}'", unpack_str(fn_name));
         match self.get_import_index(fn_name) {
             Some(idx) => {
-                dbg_log!("found existing entry at idx {}", idx);
                 idx
             },
             None => {
                 let idx = self.write_import_entry(fn_name, type_index);
-                dbg_log!("wrote new import entry at idx {}", idx);
                 idx
             },
         }
@@ -331,8 +334,8 @@ impl WasmBuilder {
 
 #[cfg(test)]
 mod tests {
-    use ::module_init::*;
-    use ::util::pack_str;
+    use wasmgen::module_init::*;
+    use util::pack_str;
 
     #[test]
     fn import_table_management() {

src/rust/wasmgen/wasm_util.rs

@@ -1,5 +1,5 @@
-use ::wasm_opcodes as op;
-use ::util::{ write_fixed_leb16_at_idx, write_leb_i32, write_leb_u32 };
+use wasmgen::wasm_opcodes as op;
+use util::{ write_fixed_leb16_at_idx, write_leb_i32, write_leb_u32 };
 
 #[no_mangle]
 pub fn wg_push_i32(buf: &mut Vec<u8>, v: i32) {

tests/rust/verify-wasmgen-dummy-output.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+"use strict";
+
+process.on("unhandledRejection", exn => { throw exn; });
+
+const fs = require('fs');
+const path = require('path');
+
+const DUMMY_MODULE_PATH = path.resolve(__dirname, '../../build/wg_dummy_output.wasm');
+const dummy_module = fs.readFileSync(DUMMY_MODULE_PATH);
+
+const wm = new WebAssembly.Module(dummy_module);
+const mem = new WebAssembly.Memory({ initial: 256 });
+
+// These tests have to be kept in sync with src/rust/wasmgen/module_init.rs' tests
+// XXX: make the test more complex, involving locals, conditionals and stuff
+
+let baz_recd_arg;
+function baz(arg) {
+    baz_recd_arg = arg;
+    return 456;
+}
+
+let foo_recd_arg;
+function foo(arg) {
+    foo_recd_arg = arg;
+}
+
+const i = new WebAssembly.Instance(wm, { 'e': { m: mem, baz, foo } });
+i.exports.f();
+
+console.assert(baz_recd_arg === 2, `baz returned: "${baz_recd_arg}"`);
+console.assert(foo_recd_arg === 456, `foo returned: "${foo_recd_arg}"`);
+