ethonion10/v86
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

wip code tlb

Browse source
This commit is contained in:
Fabian 2022-10-13 15:08:38 -05:00
parent 2887a363f1
commit e1b6e34c19
7 changed files with 468 additions and 368 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/browser/print_stats.js
View file

@ -14,13 +14,11 @@ const print_stats = {
const stat_names = [
"COMPILE",
"COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS",
"COMPILE_SUCCESS",
"COMPILE_WRONG_ADDRESS_SPACE",
"COMPILE_CUT_OFF_AT_END_OF_PAGE",
"COMPILE_WITH_LOOP_SAFETY",
"COMPILE_PAGE",
"COMPILE_PAGE/COMPILE_SUCCESS",
"COMPILE_PAGE_SKIPPED_NO_NEW_ENTRY_POINTS",
"COMPILE_PAGE/COMPILE",
"COMPILE_BASIC_BLOCK",
"COMPILE_DUPLICATED_BASIC_BLOCK",
"COMPILE_WASM_BLOCK",
@ -29,14 +27,12 @@ const print_stats = {
"COMPILE_ENTRY_POINT",
"COMPILE_WASM_TOTAL_BYTES",
"COMPILE_WASM_TOTAL_BYTES/COMPILE_PAGE",
"JIT_CACHE_OVERRIDE",
"JIT_CACHE_OVERRIDE_DIFFERENT_STATE_FLAGS",
"RUN_INTERPRETED",
"RUN_INTERPRETED_PENDING",
"RUN_INTERPRETED_PAGE_HAS_CODE",
"RUN_INTERPRETED_PAGE_HAS_ENTRY_AFTER_PAGE_WALK",
"RUN_INTERPRETED_NEAR_END_OF_PAGE",
"RUN_INTERPRETED_DIFFERENT_STATE",
"RUN_INTERPRETED_MISSED_COMPILED_ENTRY_RUN_INTERPRETED",
"RUN_INTERPRETED_MISSED_COMPILED_ENTRY_LOOKUP",
"RUN_INTERPRETED_STEPS",
"RUN_FROM_CACHE",
"RUN_FROM_CACHE_STEPS",
@ -85,7 +81,6 @@ const print_stats = {
"SAFE_READ_WRITE_SLOW_HAS_CODE",
"PAGE_FAULT",
"TLB_MISS",
"DO_RUN",
"DO_MANY_CYCLES",
"CYCLE_INTERNAL",
"INVALIDATE_ALL_MODULES_NO_FREE_WASM_INDICES",

10
src/rust/control_flow.rs
View file

@ -269,11 +269,11 @@ pub fn loopify(nodes: &Graph) -> Vec<WasmStructure> {
.collect();
if entries_to_group.len() != 1 {
dbg_log!(
"Compiling multi-entry loop with {} entries and {} basic blocks",
entries_to_group.len(),
group.len()
);
//dbg_log!(
// "Compiling multi-entry loop with {} entries and {} basic blocks",
// entries_to_group.len(),
// group.len()
//);
}
if entries_to_group.len() * group.len() > MAX_EXTRA_BASIC_BLOCKS {

105
src/rust/cpu/cpu.rs
View file

@ -33,6 +33,7 @@ use profiler;
use profiler::stat::*;
use state_flags::CachedStateFlags;
use std::collections::HashSet;
use std::ptr::NonNull;
pub use util::dbg_trace;
/// The offset for our generated functions in the wasm table. Every index less than this is
@ -54,7 +55,6 @@ pub union reg128 {
pub f64: [f64; 2],
}
/// Setting this to true will make execution extremely slow
pub const CHECK_MISSED_ENTRY_POINTS: bool = false;
pub const INTERPRETER_ITERATION_LIMIT: u32 = 100_001;
@ -263,6 +263,7 @@ pub const CPU_EXCEPTION_AC: i32 = 17;
pub const CPU_EXCEPTION_MC: i32 = 18;
pub const CPU_EXCEPTION_XM: i32 = 19;
pub const CPU_EXCEPTION_VE: i32 = 20;
pub const CHECK_TLB_INVARIANTS: bool = false;
pub const DEBUG: bool = cfg!(debug_assertions);
@ -278,7 +279,15 @@ pub static mut rdtsc_imprecision_offset: u64 = 0;
pub static mut rdtsc_last_value: u64 = 0;
pub static mut tsc_offset: u64 = 0;
pub struct Code {
pub wasm_table_index: jit::WasmTableIndex,
pub state_flags: CachedStateFlags,
pub state_table: [u16; 0x1000],
}
pub static mut tlb_data: [i32; 0x100000] = [0; 0x100000];
pub static mut tlb_code: [Option<NonNull<Code>>; 0x100000] = [None; 0x100000];
pub static mut valid_tlb_entries: [i32; 10000] = [0; 10000];
pub static mut valid_tlb_entries_count: i32 = 0;
@ -2096,6 +2105,8 @@ pub unsafe fn do_page_walk(
// of memory accesses
tlb_data[page as usize] =
(high + memory::mem8 as u32) as i32 ^ page << 12 | info_bits as i32;
jit::update_tlb_code(Page::page_of(addr as u32), Page::page_of(high));
}
return Ok(high);
@ -2108,6 +2119,7 @@ pub unsafe fn full_clear_tlb() {
*last_virt_eip = -1;
for i in 0..valid_tlb_entries_count {
let page = valid_tlb_entries[i as usize];
clear_tlb_code(page);
tlb_data[page as usize] = 0;
}
valid_tlb_entries_count = 0;
@ -2134,7 +2146,8 @@ pub unsafe fn clear_tlb() {
global_page_offset += 1;
}
else {
tlb_data[page as usize] = 0
clear_tlb_code(page);
tlb_data[page as usize] = 0;
}
}
valid_tlb_entries_count = global_page_offset;
@ -2177,6 +2190,7 @@ pub unsafe fn trigger_pagefault_jit(fault: PageFault) {
*cr.offset(2) = addr;
// invalidate tlb entry
let page = ((addr as u32) >> 12) as i32;
clear_tlb_code(page);
tlb_data[page as usize] = 0;
if DEBUG {
if cpu_exception_hook(CPU_EXCEPTION_PF) {
@ -2247,6 +2261,7 @@ pub unsafe fn trigger_pagefault(fault: PageFault) {
*cr.offset(2) = addr;
// invalidate tlb entry
let page = ((addr as u32) >> 12) as i32;
clear_tlb_code(page);
tlb_data[page as usize] = 0;
*instruction_pointer = *previous_ip;
call_interrupt_vector(
@ -2269,6 +2284,9 @@ pub fn tlb_set_has_code(physical_page: Page, has_code: bool) {
tlb_data[page as usize] =
if has_code { entry | TLB_HAS_CODE } else { entry & !TLB_HAS_CODE }
}
if !has_code {
clear_tlb_code(page);
}
}
}
}
@ -2814,23 +2832,54 @@ pub unsafe fn run_instruction0f_32(opcode: i32) { ::gen::interpreter0f::run(opco
pub unsafe fn cycle_internal() {
profiler::stat_increment(CYCLE_INTERNAL);
if !::config::FORCE_DISABLE_JIT {
*previous_ip = *instruction_pointer;
let phys_addr = return_on_pagefault!(get_phys_eip()) as u32;
let state_flags = pack_current_state_flags();
let entry = jit::jit_find_cache_entry(phys_addr, state_flags);
let mut jit_entry = None;
let initial_eip = *instruction_pointer;
if entry != jit::CachedCode::NONE {
match tlb_code[(initial_eip as u32 >> 12) as usize] {
None => {},
Some(c) => {
let c = c.as_ref();
if state_flags == c.state_flags {
let state = c.state_table[initial_eip as usize & 0xFFF];
if state != u16::MAX {
jit_entry = Some((c.wasm_table_index.to_u16(), state));
}
else {
profiler::stat_increment(if is_near_end_of_page(initial_eip as u32) {
RUN_INTERPRETED_NEAR_END_OF_PAGE
}
else {
RUN_INTERPRETED_PAGE_HAS_CODE
})
}
}
else {
profiler::stat_increment(RUN_INTERPRETED_DIFFERENT_STATE);
}
},
}
if let Some((wasm_table_index, initial_state)) = jit_entry {
if jit::CHECK_JIT_STATE_INVARIANTS {
match get_phys_eip() {
Err(()) => dbg_assert!(false),
Ok(phys_eip) => {
let entry = jit::jit_find_cache_entry(phys_eip, state_flags);
dbg_assert!(entry.wasm_table_index.to_u16() == wasm_table_index);
dbg_assert!(entry.initial_state == initial_state);
},
}
}
profiler::stat_increment(RUN_FROM_CACHE);
let initial_instruction_counter = *instruction_counter;
let wasm_table_index = entry.wasm_table_index;
let initial_state = entry.initial_state;
#[cfg(debug_assertions)]
{
in_jit = true;
}
let initial_eip = *instruction_pointer;
call_indirect1(
wasm_table_index.to_u16() as i32 + WASM_TABLE_OFFSET as i32,
wasm_table_index as i32 + WASM_TABLE_OFFSET as i32,
initial_state,
);
#[cfg(debug_assertions)]
@ -2876,9 +2925,23 @@ pub unsafe fn cycle_internal() {
}
}
else {
let initial_eip = *instruction_pointer;
*previous_ip = initial_eip;
let phys_addr = return_on_pagefault!(get_phys_eip());
jit::record_entry_point(phys_addr);
match tlb_code[(initial_eip as u32 >> 12) as usize] {
None => {},
Some(c) => {
let c = c.as_ref();
if state_flags == c.state_flags
&& c.state_table[initial_eip as usize & 0xFFF] != u16::MAX
{
profiler::stat_increment(RUN_INTERPRETED_PAGE_HAS_ENTRY_AFTER_PAGE_WALK);
}
},
}
#[cfg(feature = "profiler")]
{
if CHECK_MISSED_ENTRY_POINTS {
@ -2965,19 +3028,11 @@ unsafe fn jit_run_interpreted(phys_addr: u32) {
if entry != jit::CachedCode::NONE {
profiler::stat_increment(RUN_INTERPRETED_MISSED_COMPILED_ENTRY_RUN_INTERPRETED);
//dbg_log!(
// "missed entry point at {:x} prev_opcode={:x} opcode={:x}",
// phys_addr,
// prev_opcode,
// opcode
//);
}
}
if cfg!(debug_assertions) {
debug_last_jump = LastJump::Interpreted {
phys_addr: phys_addr as u32,
};
debug_last_jump = LastJump::Interpreted { phys_addr };
}
*instruction_counter += 1;
@ -3989,12 +4044,22 @@ pub unsafe fn get_opstats_buffer(
#[cfg(not(feature = "profiler"))]
pub unsafe fn get_opstats_buffer() -> f64 { 0.0 }
pub fn clear_tlb_code(page: i32) {
unsafe {
if let Some(c) = tlb_code[page as usize] {
drop(Box::from_raw(c.as_ptr()));
}
tlb_code[page as usize] = None;
}
}
pub unsafe fn invlpg(addr: i32) {
let page = (addr as u32 >> 12) as i32;
// Note: Doesn't remove this page from valid_tlb_entries: This isn't
// necessary, because when valid_tlb_entries grows too large, it will be
// empties by calling clear_tlb, which removes this entry as it isn't global.
// This however means that valid_tlb_entries can contain some invalid entries
clear_tlb_code(page);
tlb_data[page as usize] = 0;
*last_virt_eip = -1;
}

633
src/rust/jit.rs
View file

@ -81,6 +81,8 @@ pub fn get_jit_state() -> &'static mut JitState { unsafe { jit_state.as_mut() }
#[no_mangle]
pub fn rust_init() {
dbg_assert!(std::mem::size_of::<[Option<NonNull<cpu::Code>>; 0x100000]>() == 0x100000 * 4);
let x = Box::new(JitState::create_and_initialise());
unsafe {
jit_state = NonNull::new(Box::into_raw(x)).unwrap()
@ -93,20 +95,15 @@ pub fn rust_init() {
}));
}
pub struct Entry {
#[cfg(any(debug_assertions, feature = "profiler"))]
pub len: u32,
#[cfg(debug_assertions)]
pub opcode: u32,
pub initial_state: u16,
pub wasm_table_index: WasmTableIndex,
pub state_flags: CachedStateFlags,
struct PageInfo {
wasm_table_index: WasmTableIndex,
hidden_wasm_table_indices: Vec<WasmTableIndex>,
entry_points: Vec<(u16, u16)>,
state_flags: CachedStateFlags,
}
enum PageState {
Compiling { entries: Vec<(u32, Entry)> },
enum CompilingPageState {
Compiling { pages: HashMap<Page, PageInfo> },
CompilingWritten,
}
@ -119,25 +116,60 @@ pub struct JitState {
// or HashSet<u32> rather than nested
entry_points: HashMap<Page, HashSet<u16>>,
hot_pages: [u32; HASH_PRIME as usize],
pages: HashMap<Page, PageInfo>,
wasm_table_index_free_list: Vec<WasmTableIndex>,
used_wasm_table_indices: HashMap<WasmTableIndex, HashSet<Page>>,
// All pages from used_wasm_table_indices
// Used to improve the performance of jit_dirty_page and jit_page_has_code
all_pages: HashSet<Page>,
cache: HashMap<u32, Entry>,
compiling: Option<(WasmTableIndex, PageState)>,
compiling: Option<(WasmTableIndex, CompilingPageState)>,
}
pub fn check_jit_state_invariants(ctx: &mut JitState) {
if !CHECK_JIT_STATE_INVARIANTS {
return;
}
let mut all_pages = HashSet::new();
for pages in ctx.used_wasm_table_indices.values() {
all_pages.extend(pages);
match &ctx.compiling {
Some((_, CompilingPageState::Compiling { pages })) => {
dbg_assert!(pages.keys().all(|page| ctx.entry_points.contains_key(page)));
},
_ => {},
}
let free: HashSet<WasmTableIndex> =
HashSet::from_iter(ctx.wasm_table_index_free_list.iter().cloned());
let used = HashSet::from_iter(ctx.pages.values().map(|info| info.wasm_table_index));
let compiling = HashSet::from_iter(ctx.compiling.as_ref().map(|&(index, _)| index));
dbg_assert!(free.intersection(&used).next().is_none());
dbg_assert!(used.intersection(&compiling).next().is_none());
dbg_assert!(free.len() + used.len() + compiling.len() == (WASM_TABLE_SIZE - 1) as usize);
match &ctx.compiling {
Some((_, CompilingPageState::Compiling { pages })) => {
dbg_assert!(pages.keys().all(|page| ctx.entry_points.contains_key(page)));
},
_ => {},
}
for i in 0..unsafe { cpu::valid_tlb_entries_count } {
let page = unsafe { cpu::valid_tlb_entries[i as usize] };
let entry = unsafe { cpu::tlb_data[page as usize] };
if 0 != entry {
let tlb_physical_page = Page::of_u32(
(entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
);
let w = match unsafe { cpu::tlb_code[page as usize] } {
None => None,
Some(c) => unsafe {
Some(c.as_ref().wasm_table_index)
},
};
let tlb_has_code = entry & cpu::TLB_HAS_CODE == cpu::TLB_HAS_CODE;
let infos = ctx.pages.get(&tlb_physical_page);
let entry_points = ctx.entry_points.get(&tlb_physical_page);
dbg_assert!(tlb_has_code || !w.is_some());
dbg_assert!(tlb_has_code || !infos.is_some());
dbg_assert!(tlb_has_code || !entry_points.is_some());
//dbg_assert!((w.is_some() || page.is_some() || entry_points.is_some()) == tlb_has_code); // XXX: check this
}
}
dbg_assert!(ctx.all_pages == all_pages);
}
impl JitState {
@ -150,11 +182,9 @@ impl JitState {
entry_points: HashMap::new(),
hot_pages: [0; HASH_PRIME as usize],
pages: HashMap::new(),
wasm_table_index_free_list: Vec::from_iter(wasm_table_indices),
used_wasm_table_indices: HashMap::new(),
all_pages: HashSet::new(),
cache: HashMap::new(),
compiling: None,
}
}
@ -249,22 +279,27 @@ pub fn is_near_end_of_page(address: u32) -> bool {
}
pub fn jit_find_cache_entry(phys_address: u32, state_flags: CachedStateFlags) -> CachedCode {
if is_near_end_of_page(phys_address) {
profiler::stat_increment(stat::RUN_INTERPRETED_NEAR_END_OF_PAGE);
}
// TODO: dedup with jit_find_cache_entry_in_page?
// NOTE: This is currently only used for invariant/missed-entry-point checking
let ctx = get_jit_state();
match ctx.cache.get(&phys_address) {
Some(entry) => {
if entry.state_flags == state_flags {
return CachedCode {
wasm_table_index: entry.wasm_table_index,
initial_state: entry.initial_state,
};
}
else {
profiler::stat_increment(stat::RUN_INTERPRETED_DIFFERENT_STATE);
match ctx.pages.get(&Page::page_of(phys_address)) {
Some(PageInfo {
wasm_table_index,
state_flags: s,
entry_points,
hidden_wasm_table_indices: _,
}) => {
if *s == state_flags {
let page_offset = phys_address as u16 & 0xFFF;
if let Some(&(_, initial_state)) =
entry_points.iter().find(|(p, _)| p == &page_offset)
{
return CachedCode {
wasm_table_index: *wasm_table_index,
initial_state,
};
}
}
},
None => {},
@ -275,22 +310,28 @@ pub fn jit_find_cache_entry(phys_address: u32, state_flags: CachedStateFlags) ->
#[no_mangle]
pub fn jit_find_cache_entry_in_page(
phys_address: u32,
virt_address: u32,
wasm_table_index: WasmTableIndex,
state_flags: u32,
) -> i32 {
// TODO: generate code for this
profiler::stat_increment(stat::INDIRECT_JUMP);
let state_flags = CachedStateFlags::of_u32(state_flags);
let ctx = get_jit_state();
match ctx.cache.get(&phys_address) {
Some(entry) => {
if entry.state_flags == state_flags && entry.wasm_table_index == wasm_table_index {
return entry.initial_state as i32;
}
},
None => {},
unsafe {
match cpu::tlb_code[(virt_address >> 12) as usize] {
None => {},
Some(c) => {
let c = c.as_ref();
if state_flags == c.state_flags && wasm_table_index == c.wasm_table_index {
let state = c.state_table[virt_address as usize & 0xFFF];
if state != u16::MAX {
return state.into();
}
}
},
}
}
profiler::stat_increment(stat::INDIRECT_JUMP_NO_ENTRY);
@ -363,15 +404,28 @@ fn jit_find_basic_blocks(
if !pages.contains(&phys_page) {
// page seen for the first time, handle entry points
if let Some(entry_points) = ctx.entry_points.get(&phys_page) {
if entry_points.iter().all(|&entry_point| {
ctx.cache
.contains_key(&(phys_page.to_address() | u32::from(entry_point)))
}) {
profiler::stat_increment(stat::COMPILE_PAGE_SKIPPED_NO_NEW_ENTRY_POINTS);
let existing_entry_points = match ctx.pages.get(&phys_page) {
Some(PageInfo { entry_points, .. }) => {
HashSet::from_iter(entry_points.iter().map(|x| x.0))
},
None => HashSet::new(),
};
if entry_points
.iter()
.all(|entry_point| existing_entry_points.contains(entry_point))
{
page_blacklist.insert(phys_page);
return None;
}
// XXX: Remove this paragraph
//let old_length = entry_points.len();
//entry_points.extend(existing_entry_points);
//dbg_assert!(
// entry_points.union(&existing_entry_points).count() == entry_points.len()
//);
let address_hash = jit_hot_hash_page(phys_page) as usize;
ctx.hot_pages[address_hash] = 0;
@ -383,6 +437,8 @@ fn jit_find_basic_blocks(
}
else {
// no entry points: ignore this page?
page_blacklist.insert(phys_page);
return None;
}
pages.insert(phys_page);
@ -698,25 +754,37 @@ fn jit_analyze_and_generate(
) {
let page = Page::page_of(phys_entry_point);
if ctx.compiling.is_some() {
return;
}
dbg_assert!(ctx.compiling.is_none());
let entry_points = ctx.entry_points.get(&page);
let entry_points = match entry_points {
let entry_points = match ctx.entry_points.get(&page) {
None => return,
Some(entry_points) => entry_points,
};
if entry_points.iter().all(|&entry_point| {
ctx.cache
.contains_key(&(page.to_address() | u32::from(entry_point)))
}) {
let existing_entry_points = match ctx.pages.get(&page) {
Some(PageInfo { entry_points, .. }) => HashSet::from_iter(entry_points.iter().map(|x| x.0)),
None => HashSet::new(),
};
if entry_points
.iter()
.all(|entry_point| existing_entry_points.contains(entry_point))
{
profiler::stat_increment(stat::COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS);
return;
}
// XXX: check and remove
//let old_length = entry_points.len();
//entry_points.extend(existing_entry_points);
//dbg_log!(
// "{} + {} = {}",
// entry_points.len(),
// existing_entry_points.len(),
// entry_points.union(&existing_entry_points).count()
//);
//dbg_assert!(entry_points.union(&existing_entry_points).count() == entry_points.len());
profiler::stat_increment(stat::COMPILE);
let cpu = CpuContext {
@ -851,9 +919,6 @@ fn jit_analyze_and_generate(
dbg_assert!(!pages.is_empty());
dbg_assert!(pages.len() <= MAX_PAGES);
ctx.used_wasm_table_indices
.insert(wasm_table_index, pages.clone());
ctx.all_pages.extend(pages.clone());
let basic_block_by_addr: HashMap<u32, BasicBlock> =
basic_blocks.into_iter().map(|b| (b.addr, b)).collect();
@ -861,13 +926,26 @@ fn jit_analyze_and_generate(
let entries = jit_generate_module(
structure,
&basic_block_by_addr,
cpu.clone(),
cpu,
&mut ctx.wasm_builder,
wasm_table_index,
state_flags,
);
dbg_assert!(!entries.is_empty());
let mut page_info = HashMap::new();
for &(addr, state) in &entries {
let code = page_info
.entry(Page::page_of(addr))
.or_insert_with(|| PageInfo {
wasm_table_index,
state_flags,
entry_points: Vec::new(),
hidden_wasm_table_indices: Vec::new(),
});
code.entry_points.push((addr as u16 & 0xFFF, state));
}
profiler::stat_increment_by(
stat::COMPILE_WASM_TOTAL_BYTES,
ctx.wasm_builder.get_output_len() as u64,
@ -877,7 +955,10 @@ fn jit_analyze_and_generate(
cpu::tlb_set_has_code_multiple(&pages, true);
dbg_assert!(ctx.compiling.is_none());
ctx.compiling = Some((wasm_table_index, PageState::Compiling { entries }));
ctx.compiling = Some((
wasm_table_index,
CompilingPageState::Compiling { pages: page_info },
));
let phys_addr = page.to_address();
@ -890,8 +971,6 @@ fn jit_analyze_and_generate(
ctx.wasm_builder.get_output_len(),
);
profiler::stat_increment(stat::COMPILE_SUCCESS);
check_jit_state_invariants(ctx);
}
@ -910,80 +989,123 @@ pub fn codegen_finalize_finished(
Page::page_of(phys_addr).to_address()
);
let entries = match mem::replace(&mut ctx.compiling, None) {
let pages = match mem::replace(&mut ctx.compiling, None) {
None => {
dbg_assert!(false);
return;
},
Some((in_progress_wasm_table_index, PageState::CompilingWritten)) => {
Some((in_progress_wasm_table_index, CompilingPageState::CompilingWritten)) => {
dbg_assert!(wasm_table_index == in_progress_wasm_table_index);
profiler::stat_increment(stat::INVALIDATE_MODULE_WRITTEN_WHILE_COMPILED);
free_wasm_table_index(ctx, wasm_table_index);
check_jit_state_invariants(ctx);
return;
},
Some((in_progress_wasm_table_index, PageState::Compiling { entries })) => {
Some((in_progress_wasm_table_index, CompilingPageState::Compiling { pages })) => {
dbg_assert!(wasm_table_index == in_progress_wasm_table_index);
entries
dbg_assert!(!pages.is_empty());
pages
},
};
let mut check_for_unused_wasm_table_index = HashSet::new();
dbg_assert!(!entries.is_empty());
for (addr, entry) in entries {
let maybe_old_entry = ctx.cache.insert(addr, entry);
if let Some(old_entry) = maybe_old_entry {
check_for_unused_wasm_table_index.insert(old_entry.wasm_table_index);
profiler::stat_increment(stat::JIT_CACHE_OVERRIDE);
if old_entry.state_flags != state_flags {
profiler::stat_increment(stat::JIT_CACHE_OVERRIDE_DIFFERENT_STATE_FLAGS)
for i in 0..unsafe { cpu::valid_tlb_entries_count } {
let page = unsafe { cpu::valid_tlb_entries[i as usize] };
let entry = unsafe { cpu::tlb_data[page as usize] };
if 0 != entry {
let tlb_physical_page = Page::of_u32(
(entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
);
if let Some(info) = pages.get(&tlb_physical_page) {
set_tlb_code(
Page::of_u32(page as u32),
wasm_table_index,
&info.entry_points,
state_flags,
);
}
}
}
for index in check_for_unused_wasm_table_index {
let pages = ctx.used_wasm_table_indices.get(&index).unwrap();
let mut check_for_unused_wasm_table_index = HashSet::new();
let mut is_used = false;
'outer: for p in pages {
for addr in p.address_range() {
if let Some(entry) = ctx.cache.get(&addr) {
if entry.wasm_table_index == index {
is_used = true;
break 'outer;
}
}
}
for (page, mut info) in pages {
if let Some(old_entry) = ctx.pages.remove(&page) {
info.hidden_wasm_table_indices
.extend(old_entry.hidden_wasm_table_indices);
info.hidden_wasm_table_indices
.push(old_entry.wasm_table_index);
check_for_unused_wasm_table_index.insert(old_entry.wasm_table_index);
}
ctx.pages.insert(page, info);
}
let unused: Vec<&WasmTableIndex> = check_for_unused_wasm_table_index
.iter()
.filter(|&&i| ctx.pages.values().all(|page| page.wasm_table_index != i))
.collect();
for &index in unused {
for p in ctx.pages.values_mut() {
p.hidden_wasm_table_indices.retain(|&w| w != index);
}
if !is_used {
profiler::stat_increment(stat::INVALIDATE_MODULE_UNUSED_AFTER_OVERWRITE);
free_wasm_table_index(ctx, index);
}
if !is_used {
for (_, entry) in &ctx.cache {
dbg_assert!(entry.wasm_table_index != index);
}
}
else {
let mut ok = false;
for (_, entry) in &ctx.cache {
if entry.wasm_table_index == index {
ok = true;
break;
}
}
dbg_assert!(ok);
}
dbg_log!("unused after overwrite {}", index.to_u16());
profiler::stat_increment(stat::INVALIDATE_MODULE_UNUSED_AFTER_OVERWRITE);
free_wasm_table_index(ctx, index);
}
check_jit_state_invariants(ctx);
}
pub fn update_tlb_code(virt_page: Page, phys_page: Page) {
let ctx = get_jit_state();
match ctx.pages.get(&phys_page) {
Some(PageInfo {
wasm_table_index,
entry_points,
state_flags,
hidden_wasm_table_indices: _,
}) => set_tlb_code(virt_page, *wasm_table_index, entry_points, *state_flags),
None => cpu::clear_tlb_code(phys_page.to_u32() as i32),
};
}
pub fn set_tlb_code(
virt_page: Page,
wasm_table_index: WasmTableIndex,
entries: &Vec<(u16, u16)>,
state_flags: CachedStateFlags,
) {
let c = match unsafe { cpu::tlb_code[virt_page.to_u32() as usize] } {
None => {
let state_table = [u16::MAX; 0x1000];
unsafe {
let mut c = NonNull::new_unchecked(Box::into_raw(Box::new(cpu::Code {
wasm_table_index,
state_flags,
state_table,
})));
cpu::tlb_code[virt_page.to_u32() as usize] = Some(c);
c.as_mut()
}
},
Some(mut c) => unsafe {
let c = c.as_mut();
c.state_table.fill(u16::MAX);
c.state_flags = state_flags;
c.wasm_table_index = wasm_table_index;
c
},
};
for &(addr, state) in entries {
dbg_assert!(state != u16::MAX);
c.state_table[addr as usize] = state;
}
}
fn jit_generate_module(
structure: Vec<WasmStructure>,
basic_blocks: &HashMap<u32, BasicBlock>,
@ -991,7 +1113,7 @@ fn jit_generate_module(
builder: &mut WasmBuilder,
wasm_table_index: WasmTableIndex,
state_flags: CachedStateFlags,
) -> Vec<(u32, Entry)> {
) -> Vec<(u32, u16)> {
builder.reset();
let mut register_locals = (0..8)
@ -1169,12 +1291,6 @@ fn jit_generate_module(
BasicBlockType::AbsoluteEip => {
// Check if we can stay in this module, if not exit
codegen::gen_get_eip(ctx.builder);
let new_eip = ctx.builder.set_new_local();
codegen::gen_get_phys_eip_plus_mem(ctx, &new_eip);
ctx.builder.const_i32(unsafe { memory::mem8 } as i32);
ctx.builder.sub_i32();
ctx.builder.free_local(new_eip);
ctx.builder.const_i32(wasm_table_index.to_u16() as i32);
ctx.builder.const_i32(state_flags.to_u32() as i32);
ctx.builder.call_fn3_ret("jit_find_cache_entry_in_page");
@ -1792,9 +1908,7 @@ fn jit_generate_module(
ctx.builder.finish();
let mut entries = Vec::new();
for &addr in entry_blocks.iter() {
let entries = Vec::from_iter(entry_blocks.iter().map(|addr| {
let block = basic_blocks.get(&addr).unwrap();
let index = *index_for_addr.get(&addr).unwrap();
@ -1805,20 +1919,8 @@ fn jit_generate_module(
// This doesn't have any downside, besides making the hash table slightly larger
let initial_state = index.safe_to_u16();
let entry = Entry {
wasm_table_index,
initial_state,
state_flags,
#[cfg(any(debug_assertions, feature = "profiler"))]
len: block.end_addr - block.addr,
#[cfg(debug_assertions)]
opcode: memory::read32s(block.addr) as u32,
};
entries.push((block.addr, entry));
}
(block.addr, initial_state)
}));
for b in basic_blocks.values() {
if b.is_entry_block {
@ -1960,99 +2062,99 @@ fn free_wasm_table_index(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {
},
_ => {},
}
dbg_assert!(
!ctx.pages
.values()
.any(|info| info.wasm_table_index == wasm_table_index)
);
dbg_assert!(
!ctx.pages
.values()
.any(|info| info.hidden_wasm_table_indices.contains(&wasm_table_index))
);
for i in 0..unsafe { cpu::valid_tlb_entries_count } {
let page = unsafe { cpu::valid_tlb_entries[i as usize] };
unsafe {
match cpu::tlb_code[page as usize] {
None => {},
Some(c) => {
let c = c.as_ref();
dbg_assert!(c.wasm_table_index != wasm_table_index);
},
}
}
}
}
match ctx.used_wasm_table_indices.remove(&wasm_table_index) {
None => {
dbg_assert!(false);
},
Some(_pages) => {
//dbg_assert!(!pages.is_empty()); // only if CompilingWritten
},
}
ctx.wasm_table_index_free_list.push(wasm_table_index);
dbg_assert!(
ctx.wasm_table_index_free_list.len() + ctx.used_wasm_table_indices.len()
== WASM_TABLE_SIZE as usize - 1
);
// It is not strictly necessary to clear the function, but it will fail more predictably if we
// accidentally use the function and may garbage collect unused modules earlier
jit_clear_func(wasm_table_index);
rebuild_all_pages(ctx);
check_jit_state_invariants(ctx);
}
pub fn rebuild_all_pages(ctx: &mut JitState) {
// rebuild ctx.all_pages
let mut all_pages = HashSet::new();
for pages in ctx.used_wasm_table_indices.values() {
all_pages.extend(pages);
}
ctx.all_pages = all_pages;
}
/// Register a write in this page: Delete all present code
pub fn jit_dirty_page(ctx: &mut JitState, page: Page) {
let mut did_have_code = false;
if ctx.all_pages.contains(&page) {
if let Some(PageInfo {
wasm_table_index,
hidden_wasm_table_indices,
state_flags: _,
entry_points: _,
}) = ctx.pages.remove(&page)
{
profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_CODE);
did_have_code = true;
let mut index_to_free = HashSet::new();
let compiling = match &ctx.compiling {
Some((wasm_table_index, _)) => Some(*wasm_table_index),
None => None,
};
for (&wasm_table_index, pages) in &ctx.used_wasm_table_indices {
if Some(wasm_table_index) != compiling && pages.contains(&page) {
index_to_free.insert(wasm_table_index);
}
free(ctx, wasm_table_index);
for wasm_table_index in hidden_wasm_table_indices {
free(ctx, wasm_table_index);
}
match &ctx.compiling {
None => {},
Some((_, PageState::CompilingWritten)) => {},
Some((wasm_table_index, PageState::Compiling { .. })) => {
let pages = ctx
.used_wasm_table_indices
.get_mut(wasm_table_index)
.unwrap();
if pages.contains(&page) {
pages.clear();
ctx.compiling = Some((*wasm_table_index, PageState::CompilingWritten));
rebuild_all_pages(ctx);
}
},
}
for index in &index_to_free {
match ctx.used_wasm_table_indices.get(&index) {
None => {
dbg_assert!(false);
},
Some(pages) => {
for &p in pages {
for addr in p.address_range() {
if let Some(e) = ctx.cache.get(&addr) {
if index_to_free.contains(&e.wasm_table_index) {
ctx.cache.remove(&addr);
fn free(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {
for i in 0..unsafe { cpu::valid_tlb_entries_count } {
let page = unsafe { cpu::valid_tlb_entries[i as usize] };
let entry = unsafe { cpu::tlb_data[page as usize] };
if 0 != entry {
let tlb_physical_page = Page::of_u32(
(entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
);
match unsafe { cpu::tlb_code[page as usize] } {
None => {},
Some(c) => unsafe {
let w = c.as_ref().wasm_table_index;
if wasm_table_index == w {
drop(Box::from_raw(c.as_ptr()));
cpu::tlb_code[page as usize] = None;
if !ctx.entry_points.contains_key(&tlb_physical_page) {
cpu::tlb_data[page as usize] &= !cpu::TLB_HAS_CODE; // XXX
}
}
}
},
}
},
}
}
}
for index in index_to_free {
profiler::stat_increment(stat::INVALIDATE_MODULE_DIRTY_PAGE);
free_wasm_table_index(ctx, index)
ctx.pages.retain(
|
_,
&mut PageInfo {
wasm_table_index: w,
..
},
| w != wasm_table_index,
);
for info in ctx.pages.values_mut() {
info.hidden_wasm_table_indices
.retain(|&w| w != wasm_table_index)
}
free_wasm_table_index(ctx, wasm_table_index);
}
}
@ -2064,16 +2166,27 @@ pub fn jit_dirty_page(ctx: &mut JitState, page: Page) {
// don't try to compile code in this page anymore until it's hot again
ctx.hot_pages[jit_hot_hash_page(page) as usize] = 0;
match &ctx.compiling {
Some((index, CompilingPageState::Compiling { pages })) => {
if pages.contains_key(&page) {
ctx.compiling = Some((*index, CompilingPageState::CompilingWritten));
}
},
_ => {},
}
},
}
for pages in ctx.used_wasm_table_indices.values() {
dbg_assert!(!pages.contains(&page));
match &ctx.compiling {
Some((_, CompilingPageState::Compiling { pages })) => {
dbg_assert!(!pages.contains_key(&page));
},
_ => {},
}
check_jit_state_invariants(ctx);
dbg_assert!(!ctx.all_pages.contains(&page));
dbg_assert!(!jit_page_has_code_ctx(ctx, page));
if did_have_code {
@ -2121,17 +2234,11 @@ pub fn jit_clear_cache_js() { jit_clear_cache(get_jit_state()) }
pub fn jit_clear_cache(ctx: &mut JitState) {
let mut pages_with_code = HashSet::new();
for page in ctx.entry_points.keys() {
pages_with_code.insert(*page);
}
for &p in &ctx.all_pages {
for &p in ctx.entry_points.keys() {
pages_with_code.insert(p);
}
for addr in ctx.cache.keys() {
dbg_assert!(pages_with_code.contains(&Page::page_of(*addr)));
}
for pages in ctx.used_wasm_table_indices.values() {
dbg_assert!(pages_with_code.is_superset(pages));
for &p in ctx.pages.keys() {
pages_with_code.insert(p);
}
for page in pages_with_code {
@ -2142,7 +2249,7 @@ pub fn jit_clear_cache(ctx: &mut JitState) {
pub fn jit_page_has_code(page: Page) -> bool { jit_page_has_code_ctx(get_jit_state(), page) }
pub fn jit_page_has_code_ctx(ctx: &mut JitState, page: Page) -> bool {
ctx.all_pages.contains(&page) || ctx.entry_points.contains_key(&page)
ctx.pages.contains_key(&page) || ctx.entry_points.contains_key(&page)
}
#[no_mangle]
@ -2156,51 +2263,49 @@ pub fn jit_get_wasm_table_index_free_list_count() -> u32 {
}
#[no_mangle]
pub fn jit_get_cache_size() -> u32 {
if cfg!(feature = "profiler") { get_jit_state().cache.len() as u32 } else { 0 }
if cfg!(feature = "profiler") {
get_jit_state()
.pages
.values()
.map(|p| p.entry_points.len() as u32)
.sum()
}
else {
0
}
}
#[cfg(feature = "profiler")]
pub fn check_missed_entry_points(phys_address: u32, state_flags: CachedStateFlags) {
let ctx = get_jit_state();
// backwards until beginning of page
for offset in 0..=(phys_address & 0xFFF) {
let addr = phys_address - offset;
dbg_assert!(phys_address >= addr);
if let Some(entry) = ctx.cache.get(&addr) {
if entry.state_flags != state_flags || phys_address >= addr + entry.len {
// give up search on first entry that is not a match
break;
}
profiler::stat_increment(stat::RUN_INTERPRETED_MISSED_COMPILED_ENTRY_LOOKUP);
let last_jump_type = unsafe { cpu::debug_last_jump.name() };
let last_jump_addr = unsafe { cpu::debug_last_jump.phys_address() }.unwrap_or(0);
let last_jump_opcode =
if last_jump_addr != 0 { memory::read32s(last_jump_addr) } else { 0 };
let opcode = memory::read32s(phys_address);
dbg_log!(
"Compiled exists, but no entry point, \
start={:x} end={:x} phys_addr={:x} opcode={:02x} {:02x} {:02x} {:02x}. \
Last jump at {:x} ({}) opcode={:02x} {:02x} {:02x} {:02x}",
addr,
addr + entry.len,
phys_address,
opcode & 0xFF,
opcode >> 8 & 0xFF,
opcode >> 16 & 0xFF,
opcode >> 16 & 0xFF,
last_jump_addr,
last_jump_type,
last_jump_opcode & 0xFF,
last_jump_opcode >> 8 & 0xFF,
last_jump_opcode >> 16 & 0xFF,
last_jump_opcode >> 16 & 0xFF,
);
if let Some(infos) = ctx.pages.get(&Page::page_of(phys_address)) {
if infos.state_flags != state_flags {
return;
}
let last_jump_type = unsafe { cpu::debug_last_jump.name() };
let last_jump_addr = unsafe { cpu::debug_last_jump.phys_address() }.unwrap_or(0);
let last_jump_opcode =
if last_jump_addr != 0 { memory::read32s(last_jump_addr) } else { 0 };
let opcode = memory::read32s(phys_address);
dbg_log!(
"Compiled exists, but no entry point, \
phys_addr={:x} opcode={:02x} {:02x} {:02x} {:02x}. \
Last jump at {:x} ({}) opcode={:02x} {:02x} {:02x} {:02x}",
phys_address,
opcode & 0xFF,
opcode >> 8 & 0xFF,
opcode >> 16 & 0xFF,
opcode >> 16 & 0xFF,
last_jump_addr,
last_jump_type,
last_jump_opcode & 0xFF,
last_jump_opcode >> 8 & 0xFF,
last_jump_opcode >> 16 & 0xFF,
last_jump_opcode >> 16 & 0xFF,
);
}
}

13
src/rust/profiler.rs
View file

@ -2,12 +2,10 @@
pub enum stat {
COMPILE,
COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS,
COMPILE_SUCCESS,
COMPILE_WRONG_ADDRESS_SPACE,
COMPILE_CUT_OFF_AT_END_OF_PAGE,
COMPILE_WITH_LOOP_SAFETY,
COMPILE_PAGE,
COMPILE_PAGE_SKIPPED_NO_NEW_ENTRY_POINTS,
COMPILE_BASIC_BLOCK,
COMPILE_DUPLICATED_BASIC_BLOCK,
COMPILE_WASM_BLOCK,
@ -16,15 +14,12 @@ pub enum stat {
COMPILE_ENTRY_POINT,
COMPILE_WASM_TOTAL_BYTES,
JIT_CACHE_OVERRIDE,
JIT_CACHE_OVERRIDE_DIFFERENT_STATE_FLAGS,
RUN_INTERPRETED,
RUN_INTERPRETED_PENDING,
RUN_INTERPRETED_PAGE_HAS_CODE,
RUN_INTERPRETED_PAGE_HAS_ENTRY_AFTER_PAGE_WALK,
RUN_INTERPRETED_NEAR_END_OF_PAGE,
RUN_INTERPRETED_DIFFERENT_STATE,
RUN_INTERPRETED_MISSED_COMPILED_ENTRY_RUN_INTERPRETED,
RUN_INTERPRETED_MISSED_COMPILED_ENTRY_LOOKUP,
RUN_INTERPRETED_STEPS,
RUN_FROM_CACHE,
@ -81,7 +76,6 @@ pub enum stat {
PAGE_FAULT,
TLB_MISS,
DO_RUN,
DO_MANY_CYCLES,
CYCLE_INTERNAL,
@ -144,6 +138,3 @@ pub fn profiler_stat_get(stat: stat) -> f64 {
#[no_mangle]
pub fn profiler_is_enabled() -> bool { cfg!(feature = "profiler") }
#[no_mangle]
pub fn profiler_stat_increment_do_run() { stat_increment(stat::DO_RUN); }

32
tests/expect/tests/call-ret.wast
View file

@ -22,7 +22,6 @@
(type $t20 (func (param i32 i64 i64 i32) (result i32)))
(import "e" "safe_write32_slow_jit" (func $e.safe_write32_slow_jit (type $t16)))
(import "e" "safe_read32s_slow_jit" (func $e.safe_read32s_slow_jit (type $t7)))
(import "e" "get_phys_eip_slow_jit" (func $e.get_phys_eip_slow_jit (type $t6)))
(import "e" "jit_find_cache_entry_in_page" (func $e.jit_find_cache_entry_in_page (type $t16)))
(import "e" "instr_F4" (func $e.instr_F4 (type $t0)))
(import "e" "trigger_fault_end_jit" (func $e.trigger_fault_end_jit (type $t0)))
@ -225,39 +224,12 @@
(i32.const 740))
(i32.add)
(i32.store offset=556)
(set_local $l9
(i32.load
(i32.const 556)))
(block $B9
(br_if $B9
(i32.eq
(i32.and
(tee_local $l10
(i32.load offset={normalised output}
(i32.shl
(i32.shr_u
(get_local $l9)
(i32.const 12))
(i32.const 2))))
(i32.const 4041))
(i32.const 1)))
(br_if $B1
(i32.and
(tee_local $l10
(call $e.get_phys_eip_slow_jit
(get_local $l9)))
(i32.const 1))))
(br_if $L2
(i32.ge_s
(tee_local $p0
(call $e.jit_find_cache_entry_in_page
(i32.sub
(i32.xor
(i32.and
(get_local $l10)
(i32.const -4096))
(get_local $l9))
(i32.const 5402624))
(i32.load
(i32.const 556))
(i32.const 899)
(i32.const 3)))
(i32.const 0)))

32
tests/expect/tests/indirect-call.wast
View file

@ -24,7 +24,6 @@
(import "e" "trigger_gp_jit" (func $e.trigger_gp_jit (type $t2)))
(import "e" "safe_read32s_slow_jit" (func $e.safe_read32s_slow_jit (type $t7)))
(import "e" "safe_write32_slow_jit" (func $e.safe_write32_slow_jit (type $t16)))
(import "e" "get_phys_eip_slow_jit" (func $e.get_phys_eip_slow_jit (type $t6)))
(import "e" "jit_find_cache_entry_in_page" (func $e.jit_find_cache_entry_in_page (type $t16)))
(import "e" "trigger_fault_end_jit" (func $e.trigger_fault_end_jit (type $t0)))
(import "e" "m" (memory {normalised output}))
@ -250,39 +249,12 @@
(i32.store offset=556
(i32.const 0)
(get_local $l9))
(set_local $l9
(i32.load
(i32.const 556)))
(block $B9
(br_if $B9
(i32.eq
(i32.and
(tee_local $l10
(i32.load offset={normalised output}
(i32.shl
(i32.shr_u
(get_local $l9)
(i32.const 12))
(i32.const 2))))
(i32.const 4041))
(i32.const 1)))
(br_if $B1
(i32.and
(tee_local $l10
(call $e.get_phys_eip_slow_jit
(get_local $l9)))
(i32.const 1))))
(br_if $L2
(i32.ge_s
(tee_local $p0
(call $e.jit_find_cache_entry_in_page
(i32.sub
(i32.xor
(i32.and
(get_local $l10)
(i32.const -4096))
(get_local $l9))
(i32.const 5402624))
(i32.load
(i32.const 556))
(i32.const 899)
(i32.const 3)))
(i32.const 0)))