Skip to content

perf: keep calls out of the tail-call instruction handlers #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
explodingcamera merged 4 commits into from
Sep 29, 2026
Merged

perf: keep calls out of the tail-call instruction handlers #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fbf5cb2
perf: keep calls out of the tail-call instruction handlers
matthargett Sep 27, 2026
ced212b
perf: inline the remaining memory helpers
matthargett Sep 27, 2026
bc19bf4
chore: stop on invariant violations through `abort_immediate`
matthargett Sep 28, 2026
763e7b0
chore: panic directly in the `handler_mismatch` cold paths
matthargett Sep 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 43 additions & 16 deletions crates/tinywasm/src/interpreter/executor/dispatch_become.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,6 @@ struct Bounded;
type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>;
type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>;

#[cold]
#[inline(never)]
fn instruction_handler_mismatch() -> ! {
unreachable!("instruction handler mismatch")
}

macro_rules! define_unbudgeted_tail_dispatch {
($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident;
$($variant:ident $(($($arg:pat),*))? $({ $($field:ident),* })? => $body:expr),* $(,)?) => {
Expand All @@ -36,9 +30,11 @@ macro_rules! define_unbudgeted_tail_dispatch {
macro_rules! $dispatch_next {
($next_instr_ptr:expr) => {{
let next_instr_ptr = $next_instr_ptr;
let instruction = instructions[next_instr_ptr];
let handler = Self::handler_for(instruction.opcode());
become handler($executor, instructions, next_instr_ptr, instruction);
let Some(&next) = instructions.get(next_instr_ptr) else {
become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction);
};
let handler = Self::handler_for(next.opcode());
become handler($executor, instructions, next_instr_ptr, next);
}};
}
macro_rules! $dispatch_flow {
Expand All @@ -55,10 +51,10 @@ macro_rules! define_unbudgeted_tail_dispatch {
}
use tinywasm_types::Instruction::*;
$(let $variant($($arg),*) = &instruction else {
cold!(instruction_handler_mismatch())
become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction);
};)?
$(let $variant { $($field),* } = &instruction else {
cold!(instruction_handler_mismatch())
become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction);
};)?
$body;
$dispatch_next!($instr_ptr + 1)
Expand Down Expand Up @@ -98,9 +94,11 @@ macro_rules! define_bounded_tail_dispatch {
});
}

let instruction = $executor.func.instructions[next_instr_ptr];
let handler = Self::handler_for(instruction.opcode());
become handler($executor, next_instr_ptr, instruction, instructions_until_checkpoint - 1);
let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else {
become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint);
};
let handler = Self::handler_for(next.opcode());
become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1);
}};
}
macro_rules! $dispatch_flow {
Expand All @@ -116,10 +114,10 @@ macro_rules! define_bounded_tail_dispatch {
}
use tinywasm_types::Instruction::*;
$(let $variant($($arg),*) = &instruction else {
cold!(instruction_handler_mismatch())
become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint);
};)?
$(let $variant { $($field),* } = &instruction else {
cold!(instruction_handler_mismatch())
become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint);
};)?
$body;
$dispatch_next!($instr_ptr + 1)
Expand All @@ -130,11 +128,40 @@ macro_rules! define_bounded_tail_dispatch {

impl Unbudgeted {
instruction_handlers!(define_unbudgeted_tail_dispatch);

// The handlers tail-call these cold paths instead of calling them: a call would make every
// handler save a stack frame.

#[cold]
#[inline(never)]
fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> {
unreachable!("instruction handler mismatch")
}

#[cold]
#[inline(never)]
fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> {
unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
}
}

impl Bounded {
instruction_handlers!(define_bounded_tail_dispatch);

// Tail-called like `Unbudgeted`'s.

#[cold]
#[inline(never)]
fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> {
unreachable!("instruction handler mismatch")
}
Comment thread
explodingcamera marked this conversation as resolved.

#[cold]
#[inline(never)]
fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> {
unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
}

#[inline(always)]
fn run(executor: &mut Executor<'_>) -> ExecResult<()> {
let instr_ptr = executor.cf.instr_ptr;
Expand Down
41 changes: 21 additions & 20 deletions crates/tinywasm/src/interpreter/executor/instructions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 +3,37 @@ macro_rules! exec_op {
fn exec_binary_fallible(value_stack: &mut ValueStack) -> Result<(), Trap> {
let $rhs = <$ty>::stack_pop(value_stack);
let $lhs = <$ty>::stack_pop(value_stack);
<$ty>::stack_push(value_stack, $expr?)
<$ty>::stack_push(value_stack, $expr?);
Ok(())
}
exec_binary_fallible(&mut $executor.store.value_stack)?;
}};
($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{
fn exec_unary(value_stack: &mut ValueStack) -> Result<(), Trap> {
fn exec_unary(value_stack: &mut ValueStack) {
let $v = <$from>::stack_pop(value_stack);
<$to>::stack_push(value_stack, $expr)
<$to>::stack_push(value_stack, $expr);
}
exec_unary(&mut $executor.store.value_stack)?;
exec_unary(&mut $executor.store.value_stack);
}};
($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{
exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr)
}};
($executor:ident; binary $lhs_ty:ty, $rhs_ty:ty => $res:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{
fn exec_binary(value_stack: &mut ValueStack) -> Result<(), Trap> {
fn exec_binary(value_stack: &mut ValueStack) {
let $rhs = <$rhs_ty>::stack_pop(value_stack);
let $lhs = <$lhs_ty>::stack_pop(value_stack);
<$res>::stack_push(value_stack, $expr)
<$res>::stack_push(value_stack, $expr);
}
exec_binary(&mut $executor.store.value_stack)?;
exec_binary(&mut $executor.store.value_stack);
}};
($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{
fn exec_ternary(value_stack: &mut ValueStack) -> Result<(), Trap> {
fn exec_ternary(value_stack: &mut ValueStack) {
let $c = <$from>::stack_pop(value_stack);
let $b = <$from>::stack_pop(value_stack);
let $a = <$from>::stack_pop(value_stack);
<$to>::stack_push(value_stack, $expr)
<$to>::stack_push(value_stack, $expr);
}
exec_ternary(&mut $executor.store.value_stack)?;
exec_ternary(&mut $executor.store.value_stack);
}};
}

Expand Down Expand Up @@ -99,9 +100,9 @@ macro_rules! instruction_handlers {
Return32 => dispatch_flow!(executor.exec_return_32()),
Return64 => dispatch_flow!(executor.exec_return_64()),
Return128 => dispatch_flow!(executor.exec_return_128()),
LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?,
LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?,
LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?,
LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index),
LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index),
LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index),
LocalSet32(local_index) => executor.exec_local_set_pop::<Value32>(*local_index),
LocalSet64(local_index) => executor.exec_local_set_pop::<Value64>(*local_index),
LocalSet128(local_index) => executor.exec_local_set_pop::<Value128>(*local_index),
Expand Down Expand Up @@ -201,10 +202,10 @@ macro_rules! instruction_handlers {
GlobalTee32(global_index) => executor.exec_global_tee::<Value32>(*global_index),
GlobalTee64(global_index) => executor.exec_global_tee::<Value64>(*global_index),
GlobalTee128(global_index) => executor.exec_global_tee::<Value128>(*global_index),
Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val)?,
Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val))?,
Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value())?,
Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes()))?,
Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val),
Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)),
Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()),
Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())),
I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)),
I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)),
I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)),
Expand Down Expand Up @@ -289,8 +290,8 @@ macro_rules! instruction_handlers {
I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())),

// Reference types
RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx)))?,
RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL)?,
RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))),
RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL),
RefIsNull => executor.exec_ref_is_null()?,
RefAsNonNull => executor.exec_ref_as_non_null()?,
RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)),
Expand Down Expand Up @@ -459,7 +460,7 @@ macro_rules! instruction_handlers {
V128Store64Lane(arg) => executor.exec_mem_store_lane::<i64, 8>(*arg)?,
V128Load32Zero(idx) => executor.exec_mem_load::<i32, 4, Value128>(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?,
V128Load64Zero(idx) => executor.exec_mem_load::<i64, 8, Value128>(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?,
Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value()))?,
Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())),
I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::<i32>(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?,
I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::<i32>(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?,
I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::<i32>(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?,
Expand Down
Loading
Loading