diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6be4ae55..8a94aa2d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,6 +46,11 @@ jobs: name: "Linux x86 (nightly tail calls)" target: x86_64-unknown-linux-gnu args: "--features tinywasm/nightly-tail-calls" + - os: ubuntu-26.04 + rust: nightly + name: "Linux x86 (nightly tail calls, release with debug assertions)" + target: x86_64-unknown-linux-gnu + args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls" - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index f790d63e..f62cbdce 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,42 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +// The last argument of both handler types is the height of the 32-bit value stack, which most +// instructions push to or pop from. Each handler writes it back to the stack before its body runs +// and reads it again before dispatching, so no handler loads the height its predecessor stored. +// Most keep it in a register in between. A few load it back before dispatching: the 8- and 16-lane +// SIMD ops, and the fused `LoadLocal*` handlers, whose hot path joins a cold call. +// +// Between two handlers only what the calling convention passes in registers stays out of memory. +// On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte +// `Instruction`, by reference, so every dispatch would store it and the next handler load it back. +// The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. +// Elsewhere the handlers keep the Rust ABI, which passes eight integer arguments in registers on +// arm64, six on x86-64 System V (as many as the Unbudgeted handlers take) and four on Windows x64. +macro_rules! handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + $(#[$meta])* fn $($rest)* + }; +} + +macro_rules! handler_types { + ($($abi:literal)?) => { + // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. + #[allow(improper_ctypes_definitions)] + type UnbudgetedHandler = + for<'store> $(extern $abi)? fn(&mut Executor<'store>, &[Instruction], usize, Instruction, usize) -> ExecResult<()>; + #[allow(improper_ctypes_definitions)] + type BoundedHandler = for<'store> $(extern $abi)? fn(&mut Executor<'store>, usize, Instruction, u32, usize) -> ExecResult<()>; + }; +} +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +handler_types!("C-unwind"); +#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +handler_types!(); macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -19,22 +53,24 @@ macro_rules! define_unbudgeted_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_>, instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); let Some(&next) = instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction); + become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, instructions, next_instr_ptr, next); + become handler($executor, instructions, next_instr_ptr, next, height32); }}; } macro_rules! $dispatch_flow { @@ -51,15 +87,16 @@ macro_rules! define_unbudgeted_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -76,17 +113,19 @@ macro_rules! define_bounded_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); if instructions_until_checkpoint == 0 { return cold!({ $executor.cf.instr_ptr = next_instr_ptr; @@ -95,10 +134,10 @@ macro_rules! define_bounded_tail_dispatch { } let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint); + become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1); + become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1, height32); }}; } macro_rules! $dispatch_flow { @@ -114,15 +153,16 @@ macro_rules! define_bounded_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -132,16 +172,20 @@ impl Unbudgeted { // The handlers tail-call these cold paths instead of calling them: a call would make every // handler save a stack frame. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction, _: usize) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction, _: usize) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } } @@ -150,16 +194,20 @@ impl Bounded { // Tail-called like `Unbudgeted`'s. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } #[inline(always)] @@ -167,7 +215,8 @@ impl Bounded { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + let height32 = executor.value_stack.stack_32.len(); + handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1, height32) } } @@ -180,7 +229,8 @@ impl<'store> Executor<'store> { let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); - handler(&mut self, instructions, instr_ptr, instruction)?; + let height32 = self.value_stack.stack_32.len(); + handler(&mut self, instructions, instr_ptr, instruction, height32)?; if self.completed { return Ok(()); } diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 22892ebf..15932fa3 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -6,14 +6,14 @@ macro_rules! exec_op { <$ty>::stack_push(value_stack, $expr?); Ok(()) } - exec_binary_fallible(&mut $executor.store.value_stack)?; + exec_binary_fallible(&mut $executor.value_stack)?; }}; ($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ fn exec_unary(value_stack: &mut ValueStack) { let $v = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_unary(&mut $executor.store.value_stack); + exec_unary(&mut $executor.value_stack); }}; ($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr) @@ -24,7 +24,7 @@ macro_rules! exec_op { let $lhs = <$lhs_ty>::stack_pop(value_stack); <$res>::stack_push(value_stack, $expr); } - exec_binary(&mut $executor.store.value_stack); + exec_binary(&mut $executor.value_stack); }}; ($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ fn exec_ternary(value_stack: &mut ValueStack) { @@ -33,7 +33,7 @@ macro_rules! exec_op { let $a = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_ternary(&mut $executor.store.value_stack); + exec_ternary(&mut $executor.value_stack); }}; } @@ -43,15 +43,15 @@ macro_rules! instruction_handlers { ($emit:ident) => { $emit! { executor, instr_ptr, dispatch_next, dispatch_flow; Unreachable => { return cold!(Err(Trap::Unreachable.into())); }, - Drop32 => { _ = Value32::stack_pop(&mut executor.store.value_stack)}, - Drop64 => { _ = Value64::stack_pop(&mut executor.store.value_stack)}, - Drop128 => { _ = Value128::stack_pop(&mut executor.store.value_stack)}, - Select32 => Value32::stack_select(&mut executor.store.value_stack), - Select64 => Value64::stack_select(&mut executor.store.value_stack), - Select128 => Value128::stack_select(&mut executor.store.value_stack), + Drop32 => { _ = Value32::stack_pop(&mut executor.value_stack)}, + Drop64 => { _ = Value64::stack_pop(&mut executor.value_stack)}, + Drop128 => { _ = Value128::stack_pop(&mut executor.value_stack)}, + Select32 => Value32::stack_select(&mut executor.value_stack), + Select64 => Value64::stack_select(&mut executor.value_stack), + Select128 => Value128::stack_select(&mut executor.value_stack), SelectStore32(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, SelectStore64(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, - SelectMulti(counts) => executor.store.value_stack.select_multi(*counts), + SelectMulti(counts) => executor.value_stack.select_multi(*counts), Call(v) => dispatch_flow!(executor.exec_call_direct(*v, instr_ptr + 1)?), CallSelf => { executor.exec_call_self(instr_ptr + 1)?; dispatch_next!(0); }, CallIndirect(idx) => dispatch_flow!(executor.exec_call_indirect::(*idx, instr_ptr + 1)?), @@ -63,10 +63,10 @@ macro_rules! instruction_handlers { Throw(tag) => dispatch_flow!(executor.exec_throw(*tag, instr_ptr)?), ThrowRef => dispatch_flow!(executor.exec_throw_ref(instr_ptr)?), Jump(ip) => dispatch_next!(*ip as usize), - JumpIfZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, - JumpIfZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero32(ip) => if i32::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero64(ip) => if i64::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, JumpIfRefNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, JumpIfRefNonNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, BrOnCast(idx) => if let Some(ip) = executor.exec_br_on_cast::(*idx) { dispatch_next!(ip) }, @@ -91,33 +91,33 @@ macro_rules! instruction_handlers { JumpCmpLocalConst64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_const64(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal32(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local32(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local64(*packed) { dispatch_next!(ip) }, - DropKeep32 { base, keep } => executor.store.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), - DropKeep64 { base, keep } => executor.store.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), - DropKeep128 { base, keep } => executor.store.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), + DropKeep32 { base, keep } => executor.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), + DropKeep64 { base, keep } => executor.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), + DropKeep128 { base, keep } => executor.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), BranchTable(idx) => dispatch_next!(executor.exec_branch_table(*idx)), Return => dispatch_flow!(executor.exec_return()), ReturnVoid => dispatch_flow!(executor.exec_return_void()), Return32 => dispatch_flow!(executor.exec_return_32()), Return64 => dispatch_flow!(executor.exec_return_64()), Return128 => dispatch_flow!(executor.exec_return_128()), - LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet32(local_index) => Value32::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet64(local_index) => Value64::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet128(local_index) => Value128::local_push(&mut executor.value_stack, &executor.cf, *local_index), LocalSet32(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet64(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet128(local_index) => executor.exec_local_set_pop::(*local_index), - LocalCopy32(from, to) => Value32::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy64(from, to) => Value64::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy128(from, to) => Value128::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - AddConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(*c)); }, - AndConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value & *c); }, - XorConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value ^ *c); }, - ShrUConst32(c) => { u32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_shr(*c as u32)); }, - AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(rhs)); }, - BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.store.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, - BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.store.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, - IncLocal32(arg) => { i32::local_update(&mut executor.store.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, - IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.store.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, + LocalCopy32(from, to) => Value32::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy64(from, to) => Value64::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy128(from, to) => Value128::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + AddConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value.wrapping_add(*c)); }, + AndConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value & *c); }, + XorConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value ^ *c); }, + ShrUConst32(c) => { u32::stack_update(&mut executor.value_stack, |value| value.wrapping_shr(*c as u32)); }, + AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.value_stack, |value| value.wrapping_add(rhs)); }, + BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, + BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, + IncLocal32(arg) => { i32::local_update(&mut executor.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, + IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, I32Add3 => exec_op!(executor; ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(executor; ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => executor.exec_mul_acc_local::(*acc, i32::wrapping_mul, i32::wrapping_add), @@ -160,9 +160,9 @@ macro_rules! instruction_handlers { BinOpGlobalConst32(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst64(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst128(packed) => { let v = packed.index.resolve(&executor.func.data); let rhs = Value128(v.b().resolve(&executor.func.data).value()); executor.exec_binop_global_const::(v.a(), rhs, packed.op)?; }, - SetLocalConst32(arg) => i32::local_set(&mut executor.store.value_stack, &executor.cf, arg.local, arg.value), - SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, v); }, - SetLocalConst128(packed) => Value128::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), + SetLocalConst32(arg) => i32::local_set(&mut executor.value_stack, &executor.cf, arg.local, arg.value), + SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.value_stack, &executor.cf, packed.op, v); }, + SetLocalConst128(packed) => Value128::local_set(&mut executor.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), IncMemoryLocal32(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, IncMemoryLocal64(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, StoreLocalLocal32(arg) => executor.exec_store_local_local::(arg.memory_arg_idx, arg.local1, arg.local2)?, @@ -202,10 +202,10 @@ macro_rules! instruction_handlers { GlobalTee32(global_index) => executor.exec_global_tee::(*global_index), GlobalTee64(global_index) => executor.exec_global_tee::(*global_index), GlobalTee128(global_index) => executor.exec_global_tee::(*global_index), - Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val), - Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)), - Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()), - Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())), + Const32(val) => i32::stack_push(&mut executor.value_stack, *val), + Const64Imm(val) => i64::stack_push(&mut executor.value_stack, i64::from(*val)), + Const64(idx) => i64::stack_push(&mut executor.value_stack, idx.resolve(&executor.func.data).value()), + Const128Imm(val) => Value128::stack_push(&mut executor.value_stack, Value128(u128::from(*val).to_le_bytes())), I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)), @@ -290,8 +290,8 @@ macro_rules! instruction_handlers { I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())), // Reference types - RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), - RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL), + RefFunc(func_idx) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), + RefNull(_) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::NULL), RefIsNull => executor.exec_ref_is_null()?, RefAsNonNull => executor.exec_ref_as_non_null()?, RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)), @@ -460,7 +460,7 @@ macro_rules! instruction_handlers { V128Store64Lane(arg) => executor.exec_mem_store_lane::(*arg)?, V128Load32Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())), + Const128(arg) => Value128::stack_push(&mut executor.value_stack, Value128(arg.resolve(&executor.func.data).value())), I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?, I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?, I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index decc92be..dd781160 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -72,6 +72,11 @@ impl ExecFlow { } pub(crate) struct Executor<'store> { + /// The store's value stack, which the executor holds while it runs and gives back for host + /// calls and when it is dropped. Reached through the executor, which each handler receives + /// as a `&mut`, stack accesses need one load fewer, and the compiler knows that no store to a + /// slot or to memory changes a stack's height. + value_stack: ValueStack, cf: CallFrame, func: Shared, module: ModuleInstance, @@ -88,7 +93,9 @@ impl<'store> Executor<'store> { let wasm_func = store.state.funcs.wasm(cf.func_addr); let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); let mem0 = module.mem0_addr(); + let value_stack = core::mem::take(&mut store.value_stack); Self { + value_stack, module, cf, func: wasm_func.func.clone(), @@ -101,6 +108,27 @@ impl<'store> Executor<'store> { } } + /// Runs `f` with the value stack back in the store, as host functions expect it. + #[inline(always)] + fn with_store_value_stack(&mut self, f: impl FnOnce(&mut Store) -> R) -> R { + /// Takes the stack back when `f` returns or unwinds: a host function may catch a panic + /// from a nested call, and the executor must then still hold the stack. + struct Lent<'a, 'store> { + store: &'a mut &'store mut Store, + value_stack: &'a mut ValueStack, + } + impl Drop for Lent<'_, '_> { + #[inline(always)] + fn drop(&mut self) { + *self.value_stack = core::mem::take(&mut self.store.value_stack); + } + } + + self.store.value_stack = core::mem::take(&mut self.value_stack); + let lent = Lent { store: &mut self.store, value_stack: &mut self.value_stack }; + f(lent.store) + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { @@ -128,9 +156,9 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_jump_if_ref(&mut self) -> bool { - let is_null = ValueRef::stack_peek(&self.store.value_stack).is_null(); + let is_null = ValueRef::stack_peek(&self.value_stack).is_null(); if is_null { - ValueRef::stack_pop(&mut self.store.value_stack); + ValueRef::stack_pop(&mut self.value_stack); } is_null == ON_NULL } @@ -147,41 +175,41 @@ impl<'store> Executor<'store> { &self, local: LocalAddr, ) -> bool { - (T::local_get(&self.store.value_stack, &self.cf, local) == T::default()) == ON_ZERO + (T::local_get(&self.value_stack, &self.cf, local) == T::default()) == ON_ZERO } #[inline(always)] fn exec_jump_cmp_stack_const32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i32::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i32::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_const64(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i64::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i64::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::stack_pop(&mut self.store.value_stack); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i32::stack_pop(&mut self.value_stack); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local64(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::stack_pop(&mut self.store.value_stack); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i64::stack_pop(&mut self.value_stack); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_binop_local_const_jump(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.exec(value as u32, operand.value() as u32) as i32 }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -193,10 +221,10 @@ impl<'store> Executor<'store> { packed: PackedOp128<(BinOp, CmpOp), LocalUpdateCmpOperand>, ) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.0.exec(value as u32, operand.value() as u32) as i32 }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.1.cmp(lhs, rhs).then_some(operand.target() as usize) } @@ -206,10 +234,9 @@ impl<'store> Executor<'store> { packed: PackedOp128, ) -> Result, Trap> { let operand = packed.index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |lhs| { - packed.op.exec(lhs as u32, operand.value() as u32) as i32 - }); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = + i32::stack_update(&mut self.value_stack, |lhs| packed.op.exec(lhs as u32, operand.value() as u32) as i32); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -226,7 +253,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump(&mut self, index: Operand128Idx) -> Option { let operand = index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -238,8 +265,8 @@ impl<'store> Executor<'store> { index: Operand128Idx, ) -> Result, Trap> { let operand = index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |value| value.wrapping_add(operand.value())); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = i32::stack_update(&mut self.value_stack, |value| value.wrapping_add(operand.value())); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -255,84 +282,84 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump_cmp_local(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const32(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const64(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, i64::from(operand.b())).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local32(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local64(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } fn exec_i64_add128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_add(b_lo); let carry = u64::from((lo as u64) < (a_lo as u64)); let hi = a_hi.wrapping_add(b_hi).wrapping_add(carry as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_sub128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_sub(b_lo); let borrow = u64::from((a_lo as u64) < (b_lo as u64)); let hi = a_hi.wrapping_sub(b_hi).wrapping_sub(borrow as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_mul_wide_s(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as i128).wrapping_mul(rhs as i128); - i64::stack_push(&mut self.store.value_stack, product as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64); + i64::stack_push(&mut self.value_stack, product as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as i64); Ok(()) } fn exec_i64_mul_wide_u(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as u64 as u128).wrapping_mul(rhs as u64 as u128); - i64::stack_push(&mut self.store.value_stack, product as u64 as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64); + i64::stack_push(&mut self.value_stack, product as u64 as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as u64 as i64); Ok(()) } @@ -342,8 +369,8 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(Value128, u8) -> TO, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - TO::stack_push(&mut self.store.value_stack, operation(vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + TO::stack_push(&mut self.value_stack, operation(vector, lane)); Ok(()) } @@ -353,50 +380,50 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(VALUE, Value128, u8) -> Value128, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - let value = VALUE::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + let value = VALUE::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, operation(value, vector, lane)); Ok(()) } fn exec_simd_shuffle(&mut self, lanes: Value128) -> Result<(), Trap> { - let rhs = Value128::stack_pop(&mut self.store.value_stack); - let lhs = Value128::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); + let rhs = Value128::stack_pop(&mut self.value_stack); + let lhs = Value128::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); Ok(()) } #[inline(always)] fn exec_local_set_pop(&mut self, local: LocalAddr) { - let value = T::stack_pop(&mut self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_pop(&mut self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_local_tee(&mut self, local: LocalAddr) { - let value = T::stack_peek(&self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_peek(&self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_global_get(&mut self, global: GlobalAddr) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let value = T::global_get(&self.store.state.globals, addr); - T::stack_push(&mut self.store.value_stack, value); + T::stack_push(&mut self.value_stack, value); Ok(()) } #[inline(always)] fn exec_global_set(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_pop(&mut self.store.value_stack); + let value = T::stack_pop(&mut self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } #[inline(always)] fn exec_global_tee(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_peek(&self.store.value_stack); + let value = T::stack_peek(&self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } @@ -407,10 +434,10 @@ impl<'store> Executor<'store> { val: T, ) -> Result<(), Trap> { if let Some(dst) = dst { - T::local_set(&mut self.store.value_stack, &self.cf, dst, val); + T::local_set(&mut self.value_stack, &self.cf, dst, val); } if PUSH { - T::stack_push(&mut self.store.value_stack, val); + T::stack_push(&mut self.value_stack, val); } Ok(()) } @@ -423,8 +450,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -446,7 +473,7 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -458,7 +485,7 @@ impl<'store> Executor<'store> { op: impl BinOpExt, ) -> Result<(), Trap> { let lhs = T::global_get(&self.store.state.globals, self.module.resolve_global_addr(lhs)); - T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)); + T::stack_push(&mut self.value_stack, op.exec(lhs, rhs)); Ok(()) } @@ -468,9 +495,9 @@ impl<'store> Executor<'store> { T: InternalValue, CmpOp: CmpOpExt, { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); - i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); + i32::stack_push(&mut self.value_stack, i32::from(op.cmp(lhs, rhs))); Ok(()) } @@ -482,7 +509,7 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let rhs = T::global_get(&self.store.state.globals, addr); - T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); + T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); Ok(()) } @@ -493,8 +520,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let rhs = T::local_get(&self.store.value_stack, &self.cf, local); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::local_get(&self.value_stack, &self.cf, local); + let lhs = T::stack_pop(&mut self.value_stack); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -505,8 +532,8 @@ impl<'store> Executor<'store> { destination: LocalAddr, op: impl BinOpExt, ) -> Result<(), Trap> { - let value = T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); - T::local_set(&mut self.store.value_stack, &self.cf, destination, value); + let value = T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); + T::local_set(&mut self.value_stack, &self.cf, destination, value); Ok(()) } @@ -517,15 +544,15 @@ impl<'store> Executor<'store> { multiply: fn(T, T) -> T, add: fn(T, T) -> T, ) { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); let product = multiply(lhs, rhs); - T::local_update(&mut self.store.value_stack, &self.cf, accumulator, |value| add(product, value)); + T::local_update(&mut self.value_stack, &self.cf, accumulator, |value| add(product, value)); } fn exec_branch_table(&mut self, index: Operand128Idx) -> usize { let v = index.resolve(&self.func.data); - let idx = ::stack_pop(&mut self.store.value_stack); + let idx = ::stack_pop(&mut self.value_stack); let target_ip = if idx >= 0 && (idx as u32) < v.size() { self.func.data.branch_table_targets.get((v.start() + idx as u32) as usize).copied().unwrap_or(v.target()) } else { @@ -542,7 +569,7 @@ impl<'store> Executor<'store> { self.store.state.gc.check_allocation(payload_len, true)?; let mut payload = Vec::new(); cold_err!(payload.try_reserve_exact(payload_len)).map_err(|_| Trap::OutOfMemory)?; - let value_stack = &mut self.store.value_stack; + let value_stack = &mut self.value_stack; for index in (0..payload_len).rev() { let ty = self.store.state.get_canonical_func_type(type_addr).params()[index]; payload.push(match ty { @@ -553,7 +580,7 @@ impl<'store> Executor<'store> { }); } payload.reverse(); - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); self.store.state.alloc_exception(tag_addr, payload, roots) } @@ -563,7 +590,7 @@ impl<'store> Executor<'store> { } fn exec_throw_ref(&mut self, instr_ptr: usize) -> ExecResult { - let exception = ValueRef::stack_pop(&mut self.store.value_stack); + let exception = ValueRef::stack_pop(&mut self.value_stack); if exception.is_null() { return Err(Trap::NullReference.into()); } @@ -625,16 +652,15 @@ impl<'store> Executor<'store> { s64: stack_base.s64 + base.c64 as u32, s128: stack_base.s128 + base.c128 as u32, }; - self.store.value_stack.truncate_to_base(target); + self.value_stack.truncate_to_base(target); if include_payload { - let Store { state, value_stack, .. } = self.store; - let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; + let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_reserved(value); + self.value_stack.push_reserved(value); } } if with_ref { - ValueRef::stack_push(&mut self.store.value_stack, exception); + ValueRef::stack_push(&mut self.value_stack, exception); } return Ok(Some(if switched { ExecFlow::Switch(landing_pad as usize) @@ -643,7 +669,7 @@ impl<'store> Executor<'store> { })); } - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; @@ -659,8 +685,9 @@ impl<'store> Executor<'store> { type_addr: TypeAddr, return_instr_ptr: usize, ) -> ExecResult { + let module_id = self.module.id(); if let Some(host_func) = host_func.typed_callback() { - cold_err!(host_func.call_stack(self.store, self.module.id(), type_addr)) + cold_err!(self.with_store_value_stack(|store| host_func.call_stack(store, module_id, type_addr))) .map_err(|error| Trap::HostFunction(Box::new(error)))?; if TAIL { return Ok(self.exec_return()); @@ -670,11 +697,10 @@ impl<'store> Executor<'store> { let (param_count, result_count, base) = { let ty = self.store.state.get_canonical_func_type(type_addr); - (ty.params().len(), ty.results().len(), self.store.value_stack.base_before(ty.params().iter().collect())) + (ty.params().len(), ty.results().len(), self.value_stack.base_before(ty.params().iter().collect())) }; - let module_id = self.module.id(); - self.store - .with_scratch_values(param_count + result_count, |store, values| { + self.with_store_value_stack(|store| { + store.with_scratch_values(param_count + result_count, |store, values| { let host_values = store.stack_value_iter(type_addr, crate::store::FuncValueTypes::Params, base)?; for (slot, value) in values[..param_count].iter_mut().zip(host_values) { *slot = value; @@ -686,10 +712,11 @@ impl<'store> Executor<'store> { .map_err(|error| Error::Trap(Trap::HostFunction(Box::new(error))))?; store.push_wasm_values(results) }) - .map_err(|error| match error { - Error::Trap(trap) => trap, - other => Trap::HostFunction(Box::new(other)), - })?; + }) + .map_err(|error| match error { + Error::Trap(trap) => trap, + other => Trap::HostFunction(Box::new(other)), + })?; if TAIL { Ok(self.exec_return()) } else { Ok(ExecFlow::Next(return_instr_ptr)) } } @@ -721,8 +748,7 @@ impl<'store> Executor<'store> { fn exec_call_self(&mut self, return_instr_ptr: usize) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; let new = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); self.store.call_stack.push(core::mem::replace(&mut self.cf, new), return_instr_ptr)?; Ok(()) @@ -731,9 +757,8 @@ impl<'store> Executor<'store> { fn exec_return_call_self(&mut self) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -799,9 +824,9 @@ impl<'store> Executor<'store> { (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) }; if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + self.value_stack.truncate_keep_counts(self.cf.locals_base, params); } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; + let locals_base = self.value_stack.enter_locals(¶ms, &locals, &max_stack)?; if TAIL { self.cf = CallFrame::new(func_addr, locals_base, locals); } else { @@ -820,7 +845,7 @@ impl<'store> Executor<'store> { fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let func_ref = ValueRef::stack_pop(&mut self.store.value_stack); + let func_ref = ValueRef::stack_pop(&mut self.value_stack); let Some(func_addr) = func_ref.addr() else { return cold!(Err(Trap::NullFunctionReference.into())); }; @@ -829,7 +854,7 @@ impl<'store> Executor<'store> { } fn exec_return(&mut self) -> ExecFlow { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); self.finish_return() } @@ -849,28 +874,28 @@ impl<'store> Executor<'store> { } fn exec_return_void(&mut self) -> ExecFlow { - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); self.finish_return() } fn exec_return_32(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_64(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_128(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); self.finish_return() } @@ -882,18 +907,18 @@ impl<'store> Executor<'store> { value_local: u8, ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); - let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); + let value = T::local_get(&self.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + let base = u64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)); cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; value.store_at(&mut *mem, addr) @@ -911,14 +936,14 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; @@ -935,13 +960,13 @@ impl<'store> Executor<'store> { &mut self, m: CompactMemoryArg, ) -> Result<(), Trap> { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); - let acc = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); + let acc = T::stack_pop(&mut self.value_stack); let fma = acc + lhs * rhs; let mem_addr = self.mem_addr(m.mem_addr()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(fma.store_at(&mut *mem, addr)) }) @@ -966,37 +991,37 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), })? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; let value = cast(cold_err!(LOAD::load_at(&*mem, addr))?); if SET_LOCAL { - TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); + TARGET::local_set(&mut self.value_stack, &self.cf, u16::from(dst_local), value); } if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value); + TARGET::stack_push(&mut self.value_stack, value); } Ok(()) }) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { - let is_null = i32::from(ValueRef::stack_pop(&mut self.store.value_stack).is_null()); - i32::stack_push(&mut self.store.value_stack, is_null); + let is_null = i32::from(ValueRef::stack_pop(&mut self.value_stack).is_null()); + i32::stack_push(&mut self.value_stack, is_null); Ok(()) } fn exec_ref_as_non_null(&mut self) -> Result<(), Trap> { - if ValueRef::stack_peek(&self.store.value_stack).is_null() { + if ValueRef::stack_peek(&self.value_stack).is_null() { return cold!(Err(Trap::NullReference)); } Ok(()) @@ -1008,14 +1033,14 @@ impl<'store> Executor<'store> { } fn exec_ref_matches(&self, ty: RefType) -> bool { - let value = ValueRef::stack_peek(&self.store.value_stack); + let value = ValueRef::stack_peek(&self.value_stack); self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)) } fn exec_ref_test(&mut self, ty: RefType) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); let matches = self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)); - i32::stack_push(&mut self.store.value_stack, i32::from(matches)); + i32::stack_push(&mut self.value_stack, i32::from(matches)); Ok(()) } @@ -1027,7 +1052,7 @@ impl<'store> Executor<'store> { } fn exec_i31_get(&mut self, signed: bool) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); if value.is_null() { return cold!(Err(Trap::NullI31Reference)); } @@ -1036,14 +1061,14 @@ impl<'store> Executor<'store> { } else { value.i31_u().expect("validated i31.get operand") as i32 }; - i32::stack_push(&mut self.store.value_stack, value); + i32::stack_push(&mut self.value_stack, value); Ok(()) } fn push_gc_object(&mut self, type_addr: TypeAddr, values: Vec) -> Result<(), Trap> { - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); let reference = self.store.state.alloc_gc_object(type_addr, values, roots)?; - ValueRef::stack_push(&mut self.store.value_stack, reference); + ValueRef::stack_push(&mut self.value_stack, reference); Ok(()) } @@ -1068,7 +1093,7 @@ impl<'store> Executor<'store> { } else { for index in (0..field_count).rev() { let storage = self.store.state.get_type(type_addr).as_struct().unwrap().fields[index].storage; - values.push(pop_value(&mut self.store.value_stack, storage)); + values.push(pop_value(&mut self.value_stack, storage)); } values.reverse(); } @@ -1079,7 +1104,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let field_index = operand.b(); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.get type").fields [field_index as usize] @@ -1087,7 +1112,7 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(field_index as usize).expect("validated struct field index"); - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } @@ -1099,8 +1124,8 @@ impl<'store> Executor<'store> { let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.set type").fields [field_index as usize] .storage; - let value = pop_value(&mut self.store.value_stack, storage); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, field_index as usize, value).expect("live struct field"); Ok(()) @@ -1109,9 +1134,9 @@ impl<'store> Executor<'store> { fn exec_array_new(&mut self, type_index: TypeAddr, default: bool) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.new type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; self.store.state.gc.check_allocation(len, self.store.state.gc_type_has_references(type_addr))?; - let value = if default { default_value(storage) } else { pop_value(&mut self.store.value_stack, storage) }; + let value = if default { default_value(storage) } else { pop_value(&mut self.value_stack, storage) }; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; values.resize(len, value); @@ -1129,36 +1154,36 @@ impl<'store> Executor<'store> { let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; for _ in 0..len { - values.push(pop_value(&mut self.store.value_stack, arr_type.field.storage)); + values.push(pop_value(&mut self.value_stack, arr_type.field.storage)); } values.reverse(); self.push_gc_object(type_addr, values) } fn exec_array_get(&mut self, type_index: TypeAddr, signed: Option) -> Result<(), Trap> { - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.get type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(index).ok_or(Trap::ArrayOutOfBounds)?; - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } fn exec_array_set(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.set type").field.storage; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, index, value).ok_or(Trap::ArrayOutOfBounds) } fn exec_array_len(&mut self) -> Result<(), Trap> { - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); if reference.is_null() { return Err(Trap::NullArrayReference); } @@ -1169,17 +1194,17 @@ impl<'store> Executor<'store> { if self.store.state.get_type(type_addr).as_array().is_none() { return Err(Trap::Other("GC reference is not an array")); } - i32::stack_push(&mut self.store.value_stack, object.values.len() as i32); + i32::stack_push(&mut self.value_stack, object.values.len() as i32); Ok(()) } fn exec_array_fill(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.fill type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; let object_ref = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let end = index.checked_add(len).filter(|end| *end <= object_ref.values.len()).ok_or(Trap::ArrayOutOfBounds)?; @@ -1191,11 +1216,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let dst_type = operand.a(); let src_type = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = ValueRef::stack_pop(&mut self.store.value_stack); - let dst_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src_index = u32::stack_pop(&mut self.value_stack) as usize; + let src = ValueRef::stack_pop(&mut self.value_stack); + let dst_index = u32::stack_pop(&mut self.value_stack) as usize; + let dst = ValueRef::stack_pop(&mut self.value_stack); let dst_type = self.module.resolve_type_addr(dst_type); let src_type = self.module.resolve_type_addr(src_type); let dst_handle = self.store.state.gc_object(dst, dst_type)?; @@ -1221,8 +1246,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let data_addr = self.module.resolve_data_addr(data_index); @@ -1237,8 +1262,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let elem_addr = self.module.resolve_elem_addr(elem_index); let items = self.store.state.elements[elem_addr as usize].items_range(src, len)?; @@ -1253,10 +1278,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; @@ -1273,10 +1298,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let object = self.store.state.gc_object(reference, type_addr)?; let object_len = self.store.state.gc.get_handle(object).expect("validated array").values.len(); @@ -1292,8 +1317,8 @@ impl<'store> Executor<'store> { fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { match self.store.state.memory_size(self.mem_addr(addr)) { - (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), - (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), + (MemoryArch::I64, pages) => i64::stack_push(&mut self.value_stack, pages as i64), + (MemoryArch::I32, pages) => i32::stack_push(&mut self.value_stack, pages as i32), } Ok(()) } @@ -1303,14 +1328,14 @@ impl<'store> Executor<'store> { let limiter = self.store.engine.config().resource_limiter.as_deref(); let is_64bit = self.store.state.memory_type(mem_addr).arch() == MemoryArch::I64; let pages_delta = match is_64bit { - true => i64::stack_pop(&mut self.store.value_stack), - false => i64::from(i32::stack_pop(&mut self.store.value_stack)), + true => i64::stack_pop(&mut self.value_stack), + false => i64::from(i32::stack_pop(&mut self.value_stack)), }; let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { - true => i64::stack_push(&mut self.store.value_stack, size), - false => i32::stack_push(&mut self.store.value_stack, size as i32), + true => i64::stack_push(&mut self.value_stack, size), + false => i32::stack_push(&mut self.value_stack, size as i32), }; Ok(()) @@ -1326,9 +1351,9 @@ impl<'store> Executor<'store> { let src_arch = self.store.state.memory_type(src_mem_addr).arch(); let len_arch = if dst_arch == MemoryArch::I32 || src_arch == MemoryArch::I32 { MemoryArch::I32 } else { MemoryArch::I64 }; - let size = self.store.value_stack.pop_memory_operand(len_arch)?; - let src = self.store.value_stack.pop_memory_operand(src_arch)?; - let dst = self.store.value_stack.pop_memory_operand(dst_arch)?; + let size = self.value_stack.pop_memory_operand(len_arch)?; + let src = self.value_stack.pop_memory_operand(src_arch)?; + let dst = self.value_stack.pop_memory_operand(dst_arch)?; self.store.state.copy_memories(dst_mem_addr, dst, src_mem_addr, src, size) } @@ -1336,9 +1361,9 @@ impl<'store> Executor<'store> { fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); let arch = self.store.state.memory_type(mem_addr).arch(); - let size = self.store.value_stack.pop_memory_operand(arch)?; - let val = i32::stack_pop(&mut self.store.value_stack); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let size = self.value_stack.pop_memory_operand(arch)?; + let val = i32::stack_pop(&mut self.value_stack); + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, val as u8, size) } @@ -1346,7 +1371,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let mem_addr = self.mem_addr(operand.memory()); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, operand.byte(), operand.value() as u32 as usize) } @@ -1364,11 +1389,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let data_index = operand.a(); let mem_index = operand.b(); - let size = u32::stack_pop(&mut self.store.value_stack) as usize; - let offset = u32::stack_pop(&mut self.store.value_stack) as usize; + let size = u32::stack_pop(&mut self.value_stack) as usize; + let offset = u32::stack_pop(&mut self.value_stack) as usize; let mem_addr = self.mem_addr(mem_index); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); // Dropped segments behave like empty segments, including valid zero-length copies. @@ -1419,11 +1444,11 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let val = cold_err!(LOAD::load_at(&*mem, addr))?; let offset = arg.lane as usize * LOAD_SIZE; - Value128::stack_update(&mut self.store.value_stack, |value| { + Value128::stack_update(&mut self.value_stack, |value| { let mut bytes = value.to_mem_bytes(); bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); Value128(bytes) @@ -1440,10 +1465,10 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let value = cold_err!(LOAD::load_at(&*mem, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)); + TARGET::stack_push(&mut self.value_stack, cast(value)); Ok(()) }) } @@ -1458,16 +1483,16 @@ impl<'store> Executor<'store> { } fn exec_atomic_wait(&mut self, arg: Operand128Idx, op: AtomicWaitOp) -> Result<(), Trap> { - let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.store.value_stack) }; + let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.value_stack) }; let value = match op { - AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let memory = arg.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; let addr = crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; if addr % width != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1496,11 +1521,11 @@ impl<'store> Executor<'store> { AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, }; - u32::stack_push(&mut self.store.value_stack, result); + u32::stack_push(&mut self.value_stack, result); return Ok(()); } - u32::stack_push(&mut self.store.value_stack, 0); + u32::stack_push(&mut self.value_stack, 0); Ok(()) } @@ -1508,20 +1533,20 @@ impl<'store> Executor<'store> { let op = arg.op(); let value = match op { AtomicOp::Load => 0, - _ if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + _ if arg.is_64() => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let expected = match op { - AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - AtomicOp::Cmpxchg => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.value_stack), + AtomicOp::Cmpxchg => u32::stack_pop(&mut self.value_stack) as u64, _ => 0, }; let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; if addr % N != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1554,8 +1579,8 @@ impl<'store> Executor<'store> { } match arg.is_64() { - true => u64::stack_push(&mut self.store.value_stack, old), - false => u32::stack_push(&mut self.store.value_stack, old as u32), + true => u64::stack_push(&mut self.value_stack, old), + false => u32::stack_push(&mut self.value_stack, old as u32), } Ok(()) }) @@ -1563,7 +1588,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { - let bytes = Value128::stack_pop(&mut self.store.value_stack).to_mem_bytes(); + let bytes = Value128::stack_pop(&mut self.value_stack).to_mem_bytes(); let lane_offset = arg.lane as usize * N; let mut val_bytes = [0u8; N]; val_bytes.copy_from_slice(&bytes[lane_offset..lane_offset + N]); @@ -1571,7 +1596,7 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1583,7 +1608,7 @@ impl<'store> Executor<'store> { memory: Operand128, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { - let val = cast(::stack_pop(&mut self.store.value_stack)); + let val = cast(::stack_pop(&mut self.value_stack)); self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), val) } @@ -1592,9 +1617,9 @@ impl<'store> Executor<'store> { &mut self, memory: Operand128, ) -> Result<(), Trap> { - let condition = Value32::stack_pop(&mut self.store.value_stack); - let false_value = T::stack_pop(&mut self.store.value_stack); - let true_value = T::stack_pop(&mut self.store.value_stack); + let condition = Value32::stack_pop(&mut self.value_stack); + let false_value = T::stack_pop(&mut self.value_stack); + let true_value = T::stack_pop(&mut self.value_stack); let value = if condition == 0 { false_value } else { true_value }; self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), value) } @@ -1607,7 +1632,7 @@ impl<'store> Executor<'store> { val: U, ) -> Result<(), Trap> { crate::store::with_memory!(self.store.state, memory_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, offset))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1617,12 +1642,12 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let value = *self.store.state.get_table(table_addr).get(idx)?; - ValueRef::stack_push(&mut self.store.value_stack, value); + ValueRef::stack_push(&mut self.value_stack, value); Ok(()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let table = self.store.state.get_table_mut(table_addr); @@ -1632,8 +1657,8 @@ impl<'store> Executor<'store> { fn exec_table_size(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table(self.module.resolve_table_addr(table_index)); match table.kind.arch() { - MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, table.size() as i32), - MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, table.size() as i64), + MemoryArch::I32 => i32::stack_push(&mut self.value_stack, table.size() as i32), + MemoryArch::I64 => i64::stack_push(&mut self.value_stack, table.size() as i64), } Ok(()) } @@ -1656,16 +1681,16 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let limiter = self.store.engine.config().resource_limiter.as_deref(); let table = self.store.state.get_table_mut(table_addr); let sz = table.size(); let grew = table.grow(n, val, limiter)?; match (arch, grew) { - (MemoryArch::I32, true) => i32::stack_push(&mut self.store.value_stack, sz as i32), - (MemoryArch::I32, false) => i32::stack_push(&mut self.store.value_stack, -1), - (MemoryArch::I64, true) => i64::stack_push(&mut self.store.value_stack, sz as i64), - (MemoryArch::I64, false) => i64::stack_push(&mut self.store.value_stack, -1), + (MemoryArch::I32, true) => i32::stack_push(&mut self.value_stack, sz as i32), + (MemoryArch::I32, false) => i32::stack_push(&mut self.value_stack, -1), + (MemoryArch::I64, true) => i64::stack_push(&mut self.value_stack, sz as i64), + (MemoryArch::I64, false) => i64::stack_push(&mut self.value_stack, -1), } Ok(()) } @@ -1674,15 +1699,15 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let i = self.pop_table_operand(arch)?; self.store.state.get_table_mut(table_addr).fill(i, n, val) } fn pop_table_operand(&mut self, arch: MemoryArch) -> Result { let value = match arch { - MemoryArch::I32 => i32::stack_pop(&mut self.store.value_stack) as u32 as u64, - MemoryArch::I64 => i64::stack_pop(&mut self.store.value_stack) as u64, + MemoryArch::I32 => i32::stack_pop(&mut self.value_stack) as u32 as u64, + MemoryArch::I64 => i64::stack_pop(&mut self.value_stack) as u64, }; cold_err!(usize::try_from(value).map_err(|_| Trap::TableOutOfBounds { offset: usize::MAX, @@ -1691,3 +1716,9 @@ impl<'store> Executor<'store> { })) } } + +impl Drop for Executor<'_> { + fn drop(&mut self) { + self.store.value_stack = core::mem::take(&mut self.value_stack); + } +} diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 53632bd0..aa805a9b 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -25,7 +25,7 @@ macro_rules! checked_conv_float { ($from:tt, $to:tt, $self:expr) => {{ checked_conv_float!($from, $to, $to, $self) }}; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $self:expr) => {{ - let v = <$from>::stack_pop(&mut $self.store.value_stack); + let v = <$from>::stack_pop(&mut $self.value_stack); let (min, max) = float_min_max!($from, $intermediate); if v.is_nan() { core::hint::cold_path(); @@ -35,7 +35,7 @@ macro_rules! checked_conv_float { core::hint::cold_path(); return Err(crate::Trap::IntegerOverflow.into()); } - <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into()); + <$to>::stack_push(&mut $self.value_stack, (v as $intermediate as $to).into()); }}; } diff --git a/crates/tinywasm/src/interpreter/simd/instructions.rs b/crates/tinywasm/src/interpreter/simd/instructions.rs index 1ad4c25a..a3bc0789 100644 --- a/crates/tinywasm/src/interpreter/simd/instructions.rs +++ b/crates/tinywasm/src/interpreter/simd/instructions.rs @@ -30,7 +30,7 @@ impl Value128 { pub(crate) fn v128_any_true(self) -> bool { simd_impl! { wasm => { wasm::v128_any_true(self.to_wasm_v128()) } - generic => { self.0.iter().any(|&b| b != 0) } + generic => { u128::from_ne_bytes(self.0) != 0 } } } @@ -498,7 +498,11 @@ impl Value128 { "f64x2.extract_lane" => extract_lane_f64(self, lane: u8) -> f64 => f64::from_bits(self.extract_lane_i64(lane) as u64); // Truth checks, popcount, and averaging - "i8x16.all_true" => i8x16_all_true(self) -> bool => self.0.iter().all(|&b| b != 0); + "i8x16.all_true" => i8x16_all_true(self) -> bool => { + // No byte is zero: the classic zero-byte test on the whole vector. + let v = u128::from_ne_bytes(self.0); + v.wrapping_sub(u128::from_ne_bytes([1; 16])) & !v & u128::from_ne_bytes([0x80; 16]) == 0 + }; "i16x8.all_true" => i16x8_all_true(self) -> bool => self.as_i16x8().iter().all(|&x| x != 0); "i32x4.all_true" => i32x4_all_true(self) -> bool => self.as_i32x4().iter().all(|&x| x != 0); "i64x2.all_true" => i64x2_all_true(self) -> bool => self.as_i64x2().iter().all(|&x| x != 0); diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 1c40e31c..82e2e5f5 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -6,6 +6,7 @@ use crate::engine::{Config, StackConfig}; use crate::interpreter::*; use crate::{Result, Trap}; +#[derive(Default)] #[cfg_attr(feature = "debug", derive(Debug))] /// Physical value lanes used by the interpreter. /// @@ -18,31 +19,76 @@ pub(crate) struct ValueStack { } #[cfg_attr(feature = "debug", derive(Debug))] +/// One value lane: a stack of `len` values over `data`. +/// +/// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale +/// values that are written before they are read again. A large operand-stack reservation stays +/// capacity, so its slots are written, and their pages touched, only when a push first reaches them. +/// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and +/// write it back without `unsafe`. pub(crate) struct Stack { data: Vec, + len: usize, max_size: usize, dynamic: bool, } +/// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. +const WRITTEN_RESERVATION: usize = 64; + +/// An empty stack without an allocation: what a store holds while an executor runs. +impl Default for Stack { + fn default() -> Self { + Self { data: Vec::new(), len: 0, max_size: 0, dynamic: false } + } +} + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { - Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic } + Self { + data: Vec::with_capacity(config.initial_size), + len: 0, + max_size: config.max_size, + dynamic: config.dynamic, + } } pub(crate) fn clear(&mut self) { - self.data.clear(); + self.len = 0; } #[inline(always)] pub(crate) fn len(&self) -> usize { - self.data.len() + self.len + } + + /// Sets the height to one read from [`Self::len`] earlier, with nothing pushed or popped since. + /// The tail-call handlers pass the height between them and write it back on entry, so the + /// compiler can use the value in a register instead of reloading it from memory. + #[cfg(feature = "nightly-tail-calls")] + #[inline(always)] + pub(crate) fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.data.len()); + self.len = len; } /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand - /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own - /// growth path, so the instruction handlers make no calls. + /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)] pub(crate) fn push(&mut self, value: T) { + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => self.push_first(value), + } + self.len = len + 1; + } + + /// Adds the first slot at a height the stack has not reached before, within the reserved + /// capacity: it never reallocates. On the tail-call build this makes no call. + #[inline(always)] + fn push_first(&mut self, value: T) { + core::hint::cold_path(); if self.data.len() == self.data.capacity() { crate::invariant_violated("value stack push beyond the function's reservation"); } @@ -53,10 +99,12 @@ impl Stack { /// covers, so a dynamic stack grows here if needed. #[inline(always)] pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> { - if self.data.len() == self.data.capacity() { - return self.push_grow(value); + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => return self.push_grow(value), } - self.data.push(value); + self.len = len + 1; Ok(()) } @@ -69,33 +117,38 @@ impl Stack { #[cold] #[inline(never)] fn push_grow(&mut self, value: T) -> Result<(), Trap> { - // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may - // intentionally overshoot max_size. - if !self.dynamic || self.data.len() >= self.max_size { + // Only the reserved capacity is free to use. Past it, check the limit, which Vec growth + // may intentionally overshoot. + if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { return Err(Trap::ValueStackOverflow); } self.data.push(value); + self.len = self.data.len(); Ok(()) } + /// Pops the top value. On an empty stack the index wraps around, so the bounds check in + /// [`Self::get`] also catches an underflow. #[inline(always)] pub(crate) fn pop(&mut self) -> T { - match self.data.pop() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + let index = self.len.wrapping_sub(1); + let value = *self.get(index); + self.len = index; + value } #[inline(always)] pub(crate) fn last(&self) -> &T { - match self.data.last() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + self.get(self.len.wrapping_sub(1)) } + /// The slot at `index`, which validation keeps below the height. The check is against the + /// slots in use at any height, which is what memory safety needs. Builds with debug assertions + /// also check the height, and CI runs the tests optimized with them, with the release profile's + /// wrapping arithmetic. #[inline(always)] pub(crate) fn get(&self, index: usize) -> &T { + debug_assert!(index < self.len); match self.data.get(index) { Some(value) => value, None => crate::invariant_violated("value stack index out of range"), @@ -104,6 +157,7 @@ impl Stack { #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { + debug_assert!(index < self.len); match self.data.get_mut(index) { Some(slot) => *slot = value, None => crate::invariant_violated("value stack index out of range"), @@ -118,29 +172,35 @@ impl Stack { #[inline(always)] pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) { - let len = self.data.len(); + let len = self.len; debug_assert!(n <= len); if n >= len { return; } let keep = len.wrapping_sub(n).min(end_keep); - self.data.copy_within(len.wrapping_sub(keep)..len, n); - self.data.truncate(n.wrapping_add(keep)); + if keep != 0 { + // Copying to the start of the values above `n` leaves copy_within no check to fail. + match self.data.get_mut(n..len) { + Some(above) => above.copy_within(above.len().wrapping_sub(keep).., 0), + None => crate::invariant_violated("value stack index out of range"), + } + } + self.len = n.wrapping_add(keep); } #[inline(always)] pub(crate) fn truncate_to(&mut self, n: usize) { - debug_assert!(n <= self.data.len()); - self.data.truncate(n); + debug_assert!(n <= self.len); + self.len = n; } #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); + debug_assert!(n < self.len); let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + self.len = n; + self.push(last); } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -154,36 +214,75 @@ impl Stack { max_stack: usize, ) -> Result { debug_assert!(param_count <= local_count); - debug_assert!(param_count <= self.data.len()); + debug_assert!(param_count <= self.len); - let len = self.data.len(); + let len = self.len; let start = len - param_count; let end = start + local_count; let reserve = end + max_stack; - if reserve > self.data.capacity() { + if reserve > self.data.len() { core::hint::cold_path(); - if reserve > self.max_size || !self.dynamic { - return Err(Trap::ValueStackOverflow); - } - let cap = self.data.capacity(); - let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size); - if self.data.try_reserve(target - len).is_err() { - return Err(Trap::ValueStackOverflow); - } + self.reserve_slots(end, reserve)?; } - self.data.resize(end, T::default()); + // Most functions have no or few locals in a lane. Store the first and last directly and + // fill only what lies between: `fill` becomes a memset call even for one value. + if end > len { + match self.data.get_mut(len..end) { + Some([a]) => *a = T::default(), + Some([a, middle @ .., b]) => { + if !middle.is_empty() { + middle.fill(T::default()); + } + (*a, *b) = (T::default(), T::default()); + } + _ => crate::invariant_violated("value stack index out of range"), + } + } + self.len = end; Ok(start as u32) } + /// Makes slots for a function's locals and reserves its operand stack when they reach past the + /// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out + /// with the locals, so later entries at this height skip this. A larger one stays capacity and + /// its slots are written as they are pushed, so a function whose deep branch rarely runs does + /// not keep pages for it; its entries come back here. + #[cold] + #[inline(never)] + fn reserve_slots(&mut self, end: usize, reserve: usize) -> Result<(), Trap> { + if reserve > self.data.capacity() { + self.grow_to(reserve)?; + } + let slots = if reserve - self.data.len() <= WRITTEN_RESERVATION { reserve } else { end }; + if slots > self.data.len() { + self.data.resize(slots, T::default()); + } + Ok(()) + } + + /// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it. + #[cold] + #[inline(never)] + fn grow_to(&mut self, reserve: usize) -> Result<(), Trap> { + if reserve > self.max_size || !self.dynamic { + return Err(Trap::ValueStackOverflow); + } + let target = reserve.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size); + if self.data.try_reserve(target - self.data.len()).is_err() { + return Err(Trap::ValueStackOverflow); + } + Ok(()) + } + #[inline(always)] pub(crate) fn select_many(&mut self, count: usize, condition: bool) { if count == 0 { return; } - let len = self.data.len(); + let len = self.len; let needed = count.wrapping_mul(2); if len < needed { @@ -193,10 +292,13 @@ impl Stack { if !condition { let dst = len.wrapping_sub(needed); let src = len.wrapping_sub(count); - self.data.copy_within(src..len, dst); + match self.data.get_mut(..len) { + Some(live) => live.copy_within(src..len, dst), + None => crate::invariant_violated("value stack index out of range"), + } } - self.data.truncate(len.wrapping_sub(count)); + self.len = len.wrapping_sub(count); } } @@ -205,7 +307,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack { type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { - self.data.iter() + self.data[..self.len].iter() } } impl ValueStack { @@ -317,3 +419,60 @@ impl ValueStack { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A stack of height 2 whose third slot still holds a popped value. + #[cfg(debug_assertions)] + fn popped() -> Stack { + let mut stack = Stack::new(StackConfig::fixed(8)); + for value in [1, 2, 3] { + stack.push_or_grow(value).unwrap(); + } + assert_eq!(stack.pop(), 3); + stack + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn get_above_the_height() { + popped().get(2); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn set_above_the_height() { + popped().set(2, 0); + } + + /// A large operand-stack reservation is not written: only the locals and the values pushed + /// become slots. A small one is written with the locals. + #[test] + fn large_reservations_are_written_as_they_are_pushed() { + let mut stack = Stack::::new(StackConfig::fixed(1024)); + assert!(matches!(stack.enter_locals(0, 2, 500), Ok(0))); + assert_eq!(stack.data.len(), 2); + for value in 0..500 { + stack.push(value); + } + assert_eq!((stack.len(), stack.data.len()), (502, 502)); + + // Below the slots the stack already has, entering a function writes only its locals. + stack.truncate_to(2); + stack.push(7); + assert!(matches!(stack.enter_locals(0, 2, 10), Ok(3))); + assert_eq!((stack.len(), stack.data.len()), (5, 502)); + + stack.truncate_to(0); + for value in 0..500 { + stack.push_or_grow(value).unwrap(); + } + assert!(matches!(stack.enter_locals(0, 1, 8), Ok(500))); + assert_eq!(stack.data.len(), 509); + assert!(matches!(stack.enter_locals(0, 2, 1000), Err(Trap::ValueStackOverflow))); + } +}