Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch: stackmaps and value refs (#14066)

Daniel Macoveic2bf66bb9ad8
diff --git a/cranelift/codegen/src/ir/user_stack_maps.rs b/cranelift/codegen/src/ir/user_stack_maps.rs
index 6ad9b697cd..fb0382cfb0 100644
--- a/cranelift/codegen/src/ir/user_stack_maps.rs
+++ b/cranelift/codegen/src/ir/user_stack_maps.rs
@@ -171,6 +171,23 @@ impl UserStackMap {
}
}
+ /// Construct a stack map from offsets that are already relative to the
+ /// stack pointer at the safepoint.
+ pub fn from_sp_offsets(offsets: impl IntoIterator<Item = u32>) -> Self {
+ let offsets: alloc::vec::Vec<u32> = offsets.into_iter().collect();
+ let max = offsets.iter().max().copied().unwrap_or(0);
+ let mut bitset = CompoundBitSet::with_capacity(usize::try_from(max).unwrap() + 1);
+ for offset in offsets {
+ bitset.insert(usize::try_from(offset).unwrap());
+ }
+ let mut by_type = SmallVec::<[(ir::Type, CompoundBitSet); 1]>::default();
+ by_type.push((ir::types::I32, bitset));
+ UserStackMap {
+ by_type,
+ sp_to_sized_stack_slots: Some(0),
+ }
+ }
+
/// Finalize this stack map by filling in the SP-to-stack-slots offset.
pub(crate) fn finalize(&mut self, sp_to_sized_stack_slots: u32) {
debug_assert!(self.sp_to_sized_stack_slots.is_none());
@@ -197,3 +214,31 @@ impl UserStackMap {
})
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn from_sp_offsets_roundtrip() {
+ let map = UserStackMap::from_sp_offsets([8, 16, 40]);
+ let entries: alloc::vec::Vec<_> = map.entries().collect();
+ assert_eq!(entries.len(), 3);
+ assert_eq!(entries[0], (ir::types::I32, 8));
+ assert_eq!(entries[1], (ir::types::I32, 16));
+ assert_eq!(entries[2], (ir::types::I32, 40));
+ }
+
+ #[test]
+ fn from_sp_offsets_empty() {
+ let map = UserStackMap::from_sp_offsets([]);
+ assert_eq!(map.entries().count(), 0);
+ }
+
+ #[test]
+ fn from_sp_offsets_dedups_and_sorts() {
+ let map = UserStackMap::from_sp_offsets([40, 8, 8, 16]);
+ let offsets: alloc::vec::Vec<u32> = map.entries().map(|(_, o)| o).collect();
+ assert_eq!(offsets, [8, 16, 40]);
+ }
+}
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index fba4ee3d81..6f6e8564e7 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -1844,6 +1844,23 @@ impl<I: VCodeInst> MachBuffer<I> {
self.user_stack_maps.push((return_addr, span, stack_map));
}
+ /// Push a user stack map whose offsets are already relative to the stack
+ /// pointer at the safepoint, with an explicitly provided frame size.
+ pub fn push_user_stack_map_sp_relative(
+ &mut self,
+ return_addr: CodeOffset,
+ frame_size: u32,
+ stack_map: ir::UserStackMap,
+ ) {
+ debug_assert!(
+ self.user_stack_maps
+ .last()
+ .map_or(true, |(prev_addr, _, _)| *prev_addr < return_addr),
+ );
+ self.user_stack_maps
+ .push((return_addr, frame_size, stack_map));
+ }
+
/// Push a debug tag associated with the current buffer offset.
pub fn push_debug_tags(&mut self, pos: MachDebugTagPos, tags: &[DebugTag]) {
trace!("debug tags at offset {}: {tags:?}", self.cur_offset());
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index b6ec104358..835c41e4f2 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -454,7 +454,6 @@ impl Compiler {
|| config.tail_call()
|| config.function_references()
|| config.relaxed_simd()
- || config.gc_types()
|| config.exceptions()
|| config.legacy_exceptions()
|| config.stack_switching()
@@ -563,6 +562,16 @@ impl WastTest {
return true;
}
+ // Winch supports GC types only under the barrier-free collectors;
+ // the deferred reference-counting collector is refused until Winch
+ // emits GC barriers.
+ if config.compiler == Compiler::Winch
+ && config.collector == Collector::DeferredReferenceCounting
+ && self.config.gc_types()
+ {
+ return true;
+ }
+
// Disable spec tests per target for proposals that Winch does not implement yet.
if config.compiler == Compiler::Winch {
// Common list for tests that fail in all targets supported by Winch.
@@ -570,16 +579,10 @@ impl WastTest {
"extended-const/elem.wast",
"extended-const/global.wast",
"misc_testsuite/component-model/modules.wast",
- "misc_testsuite/externref-id-function.wast",
- "misc_testsuite/externref-segment.wast",
"misc_testsuite/externref-segments.wast",
"misc_testsuite/externref-table-dropped-segment-issue-8281.wast",
- "misc_testsuite/linking-errors.wast",
"misc_testsuite/many_table_gets_lead_to_gc.wast",
- "misc_testsuite/mutable_externref_globals.wast",
- "misc_testsuite/no-mixup-stack-maps.wast",
"misc_testsuite/no-panic.wast",
- "misc_testsuite/simple_ref_is_null.wast",
];
if unsupported.iter().any(|part| self.path.ends_with(part)) {
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 6534da5b5e..98090ecfc4 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -2454,10 +2454,17 @@ impl Config {
| WasmFeatures::FUNCTION_REFERENCES
| WasmFeatures::RELAXED_SIMD
| WasmFeatures::TAIL_CALL
- | WasmFeatures::GC_TYPES
| WasmFeatures::EXCEPTIONS
| WasmFeatures::LEGACY_EXCEPTIONS
| WasmFeatures::STACK_SWITCHING;
+
+ // Winch supports GC types only under the barrier-free
+ // collectors; the deferred reference-counting collector
+ // requires GC barriers that Winch does not emit yet.
+ #[cfg(feature = "gc")]
+ if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) {
+ unsupported |= WasmFeatures::GC_TYPES;
+ }
match self.compiler_target().architecture {
target_lexicon::Architecture::Aarch64(_) => {
unsupported |= WasmFeatures::THREADS;
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index 3058855e33..ac381877f7 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -3788,3 +3788,107 @@ fn initial_size_larger_than_reservation() -> Result<()> {
Ok(())
}
+
+/// Under Winch, a frame holding the only reference to an `externref` across a
+/// collection must keep it alive. Runs under the barrier-free collectors that Winch supports.
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_externref_survives_gc_in_frame() -> Result<()> {
+ for collector in [Collector::Null, Collector::Copying] {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(collector);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (import "" "make" (func $make (result externref)))
+ (import "" "gc" (func $gc))
+ (func (export "hold") (result externref)
+ ;; the only reference to the object is on this frame's value
+ ;; stack while the collection runs
+ (call $make)
+ (call $gc)))
+ "#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ let make = Func::wrap(
+ &mut store,
+ |mut cx: Caller<'_, ()>| -> Result<Option<Rooted<ExternRef>>> {
+ Ok(Some(ExternRef::new(&mut cx, 0xDECAFu32)?))
+ },
+ );
+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {
+ let _ = cx.gc(None);
+ });
+ let instance = Instance::new(&mut store, &module, &[make.into(), gc.into()])?;
+ let hold = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "hold")?;
+ let out = hold.call(&mut store, ())?.expect("must not be null");
+ let got = out
+ .data(&store)?
+ .and_then(|d| d.downcast_ref::<u32>().copied());
+ assert_eq!(
+ got,
+ Some(0xDECAF),
+ "externref did not survive GC under {collector:?}"
+ );
+ }
+ Ok(())
+}
+
+/// Reference values crossing the ABI boundary in every position: stack-passed
+/// externref params and multi-value externref results (more than fit in registers)
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_ref_params_and_results_across_gc() -> Result<()> {
+ for (nparams, nresults) in [(12, 12), (2, 12), (12, 3), (12, 1)] {
+ let params = "externref ".repeat(nparams);
+ let results = "externref ".repeat(nresults);
+ let gets: String = (0..nresults)
+ .map(|i| format!("(local.get {})", i % nparams))
+ .collect();
+ let wat = format!(
+ r#"(module
+ (import "" "gc" (func $gc))
+ (func (export "hold") (param {params}) (result {results})
+ (call $gc)
+ {gets}))"#
+ );
+ for collector in [Collector::Null, Collector::Copying] {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(collector);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+ let module = Module::new(&engine, &wat)?;
+ let mut store = Store::new(&engine, ());
+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {
+ let _ = cx.gc(None);
+ });
+ let instance = Instance::new(&mut store, &module, &[gc.into()])?;
+ let hold = instance.get_func(&mut store, "hold").unwrap();
+ let args: Vec<Val> = (0..nparams as u32)
+ .map(|i| Ok(Val::ExternRef(Some(ExternRef::new(&mut store, i)?))))
+ .collect::<Result<_>>()?;
+ let mut outs = vec![Val::null_extern_ref(); nresults];
+ hold.call(&mut store, &args, &mut outs)?;
+ for (i, v) in outs.iter().enumerate() {
+ let id = v
+ .unwrap_externref()
+ .expect("result became null")
+ .data(&store)?
+ .and_then(|d| d.downcast_ref::<u32>().copied());
+ assert_eq!(
+ id,
+ Some((i % nparams) as u32),
+ "result {i} corrupted under {collector:?} ({nparams}p/{nresults}r)"
+ );
+ }
+ }
+ }
+ Ok(())
+}
diff --git a/tests/all/tags.rs b/tests/all/tags.rs
index 0db6793f18..b820a27687 100644
--- a/tests/all/tags.rs
+++ b/tests/all/tags.rs
@@ -201,6 +201,13 @@ fn stack_switching_disallows_inlining() -> Result<()> {
fn issue_13474_create_tag_without_gc_runtime_configured() -> Result<()> {
let mut config = Config::new();
config.strategy(Strategy::Winch);
+ // Disable the GC runtime explicitly, along with the proposals that
+ // require it, to exercise creating a tag without a GC runtime; this
+ // previously relied on Winch stripping these features by default.
+ config.gc_support(false);
+ config.wasm_exceptions(false);
+ config.wasm_gc(false);
+ config.wasm_function_references(false);
// Ignore targets that don't have support for Winch just yet
let Ok(engine) = Engine::new(&config) else {
return Ok(());
diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rs
index fe252842a1..966588eb07 100644
--- a/tests/all/winch_engine_features.rs
+++ b/tests/all/winch_engine_features.rs
@@ -64,3 +64,41 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> R
Ok(())
}
+
+#[wasmtime_test(strategies(only(Winch)))]
+#[cfg_attr(miri, ignore)]
+fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> {
+ config.collector(Collector::DeferredReferenceCounting);
+ config.gc_support(true);
+ let result = Engine::new(&config);
+ match result {
+ Ok(_) => {
+ wasmtime::bail!(
+ "Expected incompatibility between the deferred reference-counting \
+ collector and Winch"
+ )
+ }
+ Err(e) => {
+ assert_eq!(
+ e.to_string(),
+ "the wasm_gc_types feature is not supported on this compiler configuration"
+ );
+ }
+ }
+
+ Ok(())
+}
+
+#[wasmtime_test(strategies(only(Winch)))]
+#[cfg_attr(miri, ignore)]
+fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> {
+ config.collector(Collector::DeferredReferenceCounting);
+ let engine = Engine::new(&config)?;
+ let result = Module::new(
+ &engine,
+ r#"(module (global (mut externref) (ref.null extern)))"#,
+ );
+ assert!(result.is_err());
+
+ Ok(())
+}
diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wast
new file mode 100644
index 0000000000..4f242b0223
--- /dev/null
+++ b/tests/misc_testsuite/winch/gc-refs.wast
@@ -0,0 +1,28 @@
+;;! reference_types = true
+;;! gc_types = true
+
+;; Winch reference-value support: externref flows through params, results,
+;; locals, and globals, and survives calls (stack maps). Runs under the
+;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers.
+
+(module
+ (global $g (mut externref) (ref.null extern))
+
+ (func $nop)
+
+ (func (export "roundtrip") (param externref) (result externref)
+ (local $l externref)
+ (local.set $l (local.get 0))
+ (call $nop)
+ (local.get $l))
+
+ (func (export "via_global") (param externref) (result externref)
+ (global.set $g (local.get 0))
+ (global.get $g))
+
+ (func (export "null_is_null") (result i32)
+ (ref.is_null (global.get $g))))
+
+(assert_return (invoke "null_is_null") (i32.const 1))
+(assert_return (invoke "roundtrip" (ref.null extern)) (ref.null extern))
+(assert_return (invoke "via_global" (ref.null extern)) (ref.null extern))
diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs
index 0f54c9254c..e0de2888bb 100644
--- a/winch/codegen/src/codegen/call.rs
+++ b/winch/codegen/src/codegen/call.rs
@@ -96,10 +96,22 @@ impl FnCall {
context.spill(masm)?;
let ret_area = Self::make_ret_area(&sig, masm)?;
let arg_stack_space = sig.params_stack_size();
- let reserved_stack = masm.call(arg_stack_space, |masm| {
- Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;
- Ok((kind, sig.call_conv))
- })?;
+ let reserved_stack = masm.call(
+ arg_stack_space,
+ context,
+ |masm, context| {
+ Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;
+ Ok((kind, sig.call_conv))
+ },
+ |masm, context| {
+ let sp = masm.sp_offset()?;
+ let offsets = context.calculate_stack_map_offsets(sp)?;
+ if !offsets.is_empty() {
+ masm.emit_stack_map(sp, &offsets)?;
+ }
+ Ok(())
+ },
+ )?;
Self::cleanup(
sig,
diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs
index 41a4fc3ed7..f31c336b6e 100644
--- a/winch/codegen/src/codegen/context.rs
+++ b/winch/codegen/src/codegen/context.rs
@@ -4,7 +4,7 @@ use crate::{
abi::{ABIOperand, ABIResults, RetArea, vmctx},
bail,
codegen::{BranchState, CodeGenError, CodeGenPhase, Emission, Prologue},
- ensure,
+ ensure, format_err,
frame::Frame,
isa::reg::RegClass,
masm::{
@@ -15,6 +15,7 @@ use crate::{
regalloc::RegAlloc,
stack::{Stack, TypedReg, Val},
};
+use smallvec::SmallVec;
use wasmparser::{Ieee32, Ieee64};
use wasmtime_environ::{VMOffsets, WasmHeapType, WasmValType};
@@ -636,18 +637,55 @@ impl<'a> CodeGenContext<'a, Emission> {
let len = self.stack.len();
ensure!(last <= len, CodeGenError::unexpected_value_stack_index(),);
let truncate = self.stack.len() - last;
- let stack_mut = self.stack.inner_mut();
// Invoke the callback in top-to-bottom order.
- for v in stack_mut[truncate..].into_iter().rev() {
+ for v in self.stack.inner()[truncate..].iter().rev() {
f(&mut self.regalloc, v)?
}
- stack_mut.truncate(truncate);
+ self.stack.truncate(truncate);
}
Ok(())
}
+ /// Calculate the stack map offsets for a call site at the given stack
+ /// pointer offset: the distance from the stack pointer to every slot
+ /// holding a live GC reference, in the frame's locals and in the value
+ /// stack.
+ pub fn calculate_stack_map_offsets(&self, sp: SPOffset) -> Result<SmallVec<[SPOffset; 8]>> {
+ if self.frame.gc_ref_local_offsets().is_empty() && self.stack.gc_ref_count() == 0 {
+ return Ok(SmallVec::new());
+ }
+
+ let mut offsets: SmallVec<[SPOffset; 8]> = SmallVec::new();
+
+ for offset in self.frame.gc_ref_local_offsets() {
+ offsets.push(
+ sp.checked_sub(*offset)
+ .ok_or_else(|| format_err!(CodeGenError::invalid_local_offset()))?,
+ );
+ }
+
+ let mut remaining = self.stack.gc_ref_count();
+ if remaining != 0 {
+ for v in self.stack.inner() {
+ if v.needs_stack_map() {
+ ensure!(v.is_mem(), CodeGenError::unexpected_value_in_value_stack());
+ offsets.push(
+ sp.checked_sub(v.unwrap_mem().slot.offset)
+ .ok_or_else(|| format_err!(CodeGenError::invalid_sp_offset()))?,
+ );
+ remaining -= 1;
+ if remaining == 0 {
+ break;
+ }
+ }
+ }
+ }
+
+ Ok(offsets)
+ }
+
/// Convenience wrapper around [`Self::spill_callback`].
///
/// This function exists for cases in which triggering an unconditional
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index ac0fd0b1c1..0df13d76d9 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -1,6 +1,6 @@
use crate::{
Result,
- abi::{ABIOperand, ABISig, RetArea, vmctx},
+ abi::{ABI, ABIOperand, ABISig, LocalSlot, RetArea, vmctx},
bail,
codegen::BlockSig,
ensure, format_err,
@@ -168,6 +168,7 @@ where
self.masm.reserve_stack(self.context.frame.locals_size)?;
self.spill_register_arguments()?;
+ self.copy_stack_gc_refs_to_frame()?;
let defined_locals_range = &self.context.frame.defined_locals_range;
self.masm.zero_mem_range(defined_locals_range.as_range())?;
@@ -222,9 +223,12 @@ where
self.masm.store((*reg).into(), addr, (*ty).try_into()?)?;
}
Ref(rt) => match rt.heap_type {
- WasmHeapType::Func | WasmHeapType::Extern => {
+ WasmHeapType::Func => {
self.masm.store_ptr(*reg, addr)?;
}
+ WasmHeapType::Extern => {
+ self.masm.store((*reg).into(), addr, (*ty).try_into()?)?;
+ }
_ => bail!(CodeGenError::unsupported_wasm_type()),
},
}
@@ -235,6 +239,40 @@ where
}
Ok(())
}
+
+ /// Copy GC references passed on the stack into frame slots so that
+ /// stack maps cover them: the caller's argument area is not visited by
+ /// the collector, so a reference left there goes stale across a
+ /// collection. Everything else stays in the caller's argument area.
+ fn copy_stack_gc_refs_to_frame(&mut self) -> Result<()> {
+ for (operand, slot) in self
+ .sig
+ .params_without_retptr()
+ .iter()
+ .zip(self.context.frame.locals())
+ {
+ match (operand, slot) {
+ (ABIOperand::Stack { ty, offset, .. }, slot)
+ if ty.is_vmgcref_type_and_not_i31() =>
+ {
+ ensure!(
+ slot.addressed_from_sp(),
+ CodeGenError::sp_addressing_expected(),
+ );
+ let arg_base = u32::from(<M::ABI as ABI>::arg_base_offset());
+ let src = LocalSlot::stack_arg(*ty, offset + arg_base);
+ let src_addr = self.masm.local_address(&src)?;
+ let dst_addr = self.masm.local_address(slot)?;
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load(src_addr, scratch.writable(), (*ty).try_into()?)?;
+ masm.store(scratch.inner().into(), dst_addr, (*ty).try_into()?)
+ })?;
+ }
+ _ => {}
+ }
+ }
+ Ok(())
+ }
}
impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>
diff --git a/winch/codegen/src/frame/mod.rs b/winch/codegen/src/frame/mod.rs
index 8846b3bd70..9b2e907c38 100644
--- a/winch/codegen/src/frame/mod.rs
+++ b/winch/codegen/src/frame/mod.rs
@@ -2,7 +2,8 @@ use crate::{
Result,
abi::{ABI, ABIOperand, ABISig, LocalSlot, align_to},
codegen::{CodeGenPhase, Emission, Prologue},
- masm::MacroAssembler,
+ masm::{MacroAssembler, SPOffset},
+ stack::needs_stack_map,
};
use smallvec::SmallVec;
use std::marker::PhantomData;
@@ -94,22 +95,27 @@ pub(crate) struct Frame<P: CodeGenPhase> {
/// The slot holding the address of the results area.
pub results_base_slot: Option<LocalSlot>,
marker: PhantomData<P>,
+
+ /// Frame offsets of SP-addressed locals that stack maps must cover,
+ /// precomputed so that call sites don't scan every local.
+ gc_ref_local_offsets: SmallVec<[SPOffset; 4]>,
}
impl Frame<Prologue> {
/// Allocate a new [`Frame`].
pub fn new<A: ABI>(sig: &ABISig, defined_locals: &DefinedLocals) -> Result<Frame<Prologue>> {
- let (special_locals, mut wasm_locals, defined_locals_start) =
+ let (special_locals, mut wasm_locals, mut gc_ref_local_offsets, defined_locals_start) =
Self::compute_arg_slots::<A>(sig)?;
// The defined locals have a zero-based offset by default
// so we need to add the defined locals start to the offset.
- wasm_locals.extend(
- defined_locals
- .defined_locals
- .iter()
- .map(|l| LocalSlot::new(l.ty, l.offset + defined_locals_start)),
- );
+ wasm_locals.extend(defined_locals.defined_locals.iter().map(|l| {
+ let slot = LocalSlot::new(l.ty, l.offset + defined_locals_start);
+ if needs_stack_map(&slot.ty) {
+ gc_ref_local_offsets.push(SPOffset::from_u32(slot.offset));
+ }
+ slot
+ }));
let stack_align = <A as ABI>::stack_align();
let defined_locals_end = align_to(
@@ -157,6 +163,7 @@ impl Frame<Prologue> {
),
results_base_slot,
marker: PhantomData,
+ gc_ref_local_offsets,
})
}
@@ -175,10 +182,13 @@ impl Frame<Prologue> {
defined_locals_range: self.defined_locals_range,
results_base_slot: self.results_base_slot,
marker: PhantomData,
+ gc_ref_local_offsets: self.gc_ref_local_offsets,
}
}
- fn compute_arg_slots<A: ABI>(sig: &ABISig) -> Result<(SpecialLocals, WasmLocals, u32)> {
+ fn compute_arg_slots<A: ABI>(
+ sig: &ABISig,
+ ) -> Result<(SpecialLocals, WasmLocals, SmallVec<[SPOffset; 4]>, u32)> {
// Go over the function ABI-signature and
// calculate the stack slots.
//
@@ -225,11 +235,23 @@ impl Frame<Prologue> {
.map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset))
.expect("Slot for VMContext");
+ let mut gc_ref_local_offsets = SmallVec::new();
let slots: WasmLocals = params_iter
- .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset))
+ .map(|arg| {
+ let slot = Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset);
+ if slot.addressed_from_sp() && needs_stack_map(&slot.ty) {
+ gc_ref_local_offsets.push(SPOffset::from_u32(slot.offset));
+ }
+ slot
+ })
.collect();
- Ok(([callee_vmctx, caller_vmctx], slots, next_stack))
+ Ok((
+ [callee_vmctx, caller_vmctx],
+ slots,
+ gc_ref_local_offsets,
+ next_stack,
+ ))
}
fn abi_arg_slot(arg: &ABIOperand, next_stack: &mut u32, arg_base_offset: u32) -> LocalSlot {
@@ -241,7 +263,15 @@ impl Frame<Prologue> {
LocalSlot::new(*ty, *next_stack)
}
// Create a local slot, with an offset from the arguments base in
- // the stack; which is the frame pointer + return address.
+ // the stack; which is the frame pointer + return address. GC
+ // references are re-homed into a frame slot by the prologue so
+ // that stack maps cover them: the caller's outgoing argument
+ // area is not visited by the collector, so a reference left
+ // there goes stale across a collection.
+ ABIOperand::Stack { ty, size, .. } if ty.is_vmgcref_type_and_not_i31() => {
+ *next_stack = align_to(*next_stack, *size) + *size;
+ LocalSlot::new(*ty, *next_stack)
+ }
ABIOperand::Stack { ty, offset, .. } => {
LocalSlot::stack_arg(*ty, offset + arg_base_offset)
}
@@ -279,6 +309,11 @@ impl Frame<Emission> {
self.get_special_local(0)
}
+ /// Frame offsets of the SP-addressed locals that stack maps must cover.
+ pub fn gc_ref_local_offsets(&self) -> &[SPOffset] {
+ &self.gc_ref_local_offsets
+ }
+
/// Returns the address of the local at the given index.
///
/// # Panics
diff --git a/winch/codegen/src/isa/aarch64/abi.rs b/winch/codegen/src/isa/aarch64/abi.rs
index ad933eea20..a65b12badd 100644
--- a/winch/codegen/src/isa/aarch64/abi.rs
+++ b/winch/codegen/src/isa/aarch64/abi.rs
@@ -131,6 +131,7 @@ impl ABI for Aarch64ABI {
match ty {
WasmValType::Ref(rt) => match rt.heap_type {
WasmHeapType::Func => Self::word_bytes(),
+ WasmHeapType::Extern => Self::word_bytes() / 2,
ht => unimplemented!("Support for WasmHeapType: {ht}"),
},
WasmValType::F64 | WasmValType::I64 => Self::word_bytes(),
@@ -212,6 +213,22 @@ mod tests {
WasmValType::{self, *},
};
+ #[test]
+ fn ref_sizes_match_cranelift_representation() {
+ use wasmtime_environ::{WasmHeapType, WasmRefType};
+ let ty = |heap_type| {
+ WasmValType::Ref(WasmRefType {
+ nullable: true,
+ heap_type,
+ })
+ };
+ assert_eq!(
+ Aarch64ABI::sizeof(&ty(WasmHeapType::Func)),
+ Aarch64ABI::word_bytes()
+ );
+ assert_eq!(Aarch64ABI::sizeof(&ty(WasmHeapType::Extern)), 4);
+ }
+
#[test]
fn xreg_abi_sig() -> Result<()> {
let wasm_sig = WasmFuncType::new([I32, I64, I32, I64, I32, I32, I64, I32, I64], [])?;
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 9ac7842106..276bbfd4dc 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -419,7 +419,12 @@ impl Masm for MacroAssembler {
fn call(
&mut self,
stack_args_size: u32,
- mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,
+ context: &mut CodeGenContext<Emission>,
+ mut load_callee: impl FnMut(
+ &mut Self,
+ &mut CodeGenContext<Emission>,
+ ) -> Result<(CalleeKind, CallingConvention)>,
+ mut finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,
) -> Result<u32> {
let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();
let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into();
@@ -427,11 +432,12 @@ impl Masm for MacroAssembler {
let aligned_args_size = align_to(stack_args_size, alignment);
let total_stack = delta + aligned_args_size;
self.reserve_stack(total_stack)?;
- let (callee, call_conv) = load_callee(self)?;
+ let (callee, call_conv) = load_callee(self, context)?;
match callee {
CalleeKind::Indirect(reg) => self.asm.call_with_reg(reg, call_conv),
CalleeKind::Direct(idx) => self.asm.call_with_name(idx, call_conv),
}
+ finalize(self, context)?;
Ok(total_stack)
}
@@ -1344,6 +1350,18 @@ impl Masm for MacroAssembler {
Ok(())
}
+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()> {
+ let frame_size = sp_offset.as_u32() + u32::from(SHADOW_STACK_POINTER_SLOT_SIZE);
+ let return_addr = self.asm.buffer().cur_offset();
+ let map = cranelift_codegen::ir::UserStackMap::from_sp_offsets(
+ offsets.iter().map(SPOffset::as_u32),
+ );
+ self.asm
+ .buffer_mut()
+ .push_user_stack_map_sp_relative(return_addr, frame_size, map);
+ Ok(())
+ }
+
fn current_code_offset(&self) -> Result<CodeOffset> {
Ok(self.asm.buffer().cur_offset())
}
diff --git a/winch/codegen/src/isa/x64/abi.rs b/winch/codegen/src/isa/x64/abi.rs
index 7ec1433ca0..83c1c81606 100644
--- a/winch/codegen/src/isa/x64/abi.rs
+++ b/winch/codegen/src/isa/x64/abi.rs
@@ -115,7 +115,8 @@ impl ABI for X64ABI {
fn sizeof(ty: &WasmValType) -> u8 {
match ty {
WasmValType::Ref(rt) => match rt.heap_type {
- WasmHeapType::Func | WasmHeapType::Extern => Self::word_bytes(),
+ WasmHeapType::Func => Self::word_bytes(),
+ WasmHeapType::Extern => Self::word_bytes() / 2,
ht => unimplemented!("Support for WasmHeapType: {ht}"),
},
WasmValType::F64 | WasmValType::I64 => Self::word_bytes(),
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 0044af0222..a455f828dd 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -326,7 +326,12 @@ impl Masm for MacroAssembler {
fn call(
&mut self,
stack_args_size: u32,
- mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,
+ context: &mut CodeGenContext<Emission>,
+ mut load_callee: impl FnMut(
+ &mut Self,
+ &mut CodeGenContext<Emission>,
+ ) -> Result<(CalleeKind, CallingConvention)>,
+ mut finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,
) -> Result<u32> {
let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();
let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into();
@@ -334,11 +339,13 @@ impl Masm for MacroAssembler {
let aligned_args_size = align_to(stack_args_size, alignment);
let total_stack = delta + aligned_args_size;
self.reserve_stack(total_stack)?;
- let (callee, cc) = load_callee(self)?;
+ let (callee, cc) = load_callee(self, context)?;
match callee {
CalleeKind::Indirect(reg) => self.asm.call_with_reg(cc, reg),
CalleeKind::Direct(idx) => self.asm.call_with_name(cc, idx),
};
+ finalize(self, context)?;
+
Ok(total_stack)
}
@@ -1400,6 +1407,18 @@ impl Masm for MacroAssembler {
Ok(())
}
+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()> {
+ let frame_size = sp_offset.as_u32();
+ let return_addr = self.asm.buffer().cur_offset();
+ let map = cranelift_codegen::ir::UserStackMap::from_sp_offsets(
+ offsets.iter().map(SPOffset::as_u32),
+ );
+ self.asm
+ .buffer_mut()
+ .push_user_stack_map_sp_relative(return_addr, frame_size, map);
+ Ok(())
+ }
+
fn current_code_offset(&self) -> Result<CodeOffset> {
Ok(self.asm.buffer().cur_offset())
}
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 62c8318c15..b64cd66551 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -199,6 +199,10 @@ impl SPOffset {
pub fn as_u32(&self) -> u32 {
self.0
}
+
+ pub fn checked_sub(self, rhs: Self) -> Option<Self> {
+ self.0.checked_sub(rhs.0).map(Self)
+ }
}
/// A stack slot.
@@ -1453,9 +1457,20 @@ pub(crate) trait MacroAssembler {
fn call(
&mut self,
stack_args_size: u32,
- f: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,
+ context: &mut CodeGenContext<Emission>,
+ f: impl FnMut(
+ &mut Self,
+ &mut CodeGenContext<Emission>,
+ ) -> Result<(CalleeKind, CallingConvention)>,
+ finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,
) -> Result<u32>;
+ /// Record a GC stack map at the current code offset, which must be the
+ /// return address of the call emitted immediately before. Each offset is
+ /// the distance from the stack pointer at the call site to a slot holding
+ /// a live GC reference.
+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()>;
+
/// Acquire a scratch register and execute the given callback.
fn with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R;
@@ -1470,7 +1485,7 @@ pub(crate) trait MacroAssembler {
WasmValType::I32
| WasmValType::I64
| WasmValType::Ref(WasmRefType {
- heap_type: WasmHeapType::Func,
+ heap_type: WasmHeapType::Func | WasmHeapType::Extern,
..
}) => self.with_scratch::<IntScratch, _>(f),
WasmValType::F32 | WasmValType::F64 | WasmValType::V128 => {
diff --git a/winch/codegen/src/stack.rs b/winch/codegen/src/stack.rs
index 424d05f663..af051d19ad 100644
--- a/winch/codegen/src/stack.rs
+++ b/winch/codegen/src/stack.rs
@@ -310,6 +310,20 @@ impl Val {
Val::Local(l) => l.ty,
}
}
+
+ /// Whether this value is a GC reference that must be covered by stack maps.
+ pub fn needs_stack_map(&self) -> bool {
+ needs_stack_map(&self.ty())
+ }
+}
+
+/// Whether a value of this type must be covered by stack maps.
+pub(crate) fn needs_stack_map(ty: &WasmValType) -> bool {
+ ty.is_vmgcref_type_and_not_i31()
+ && match ty {
+ WasmValType::Ref(r) => !r.heap_type.is_bottom(),
+ _ => false,
+ }
}
/// The shadow stack used for compilation.
@@ -317,6 +331,7 @@ impl Val {
pub(crate) struct Stack {
// NB: The 64 is chosen arbitrarily. We can adjust as we see fit.
inner: SmallVec<[Val; 64]>,
+ gc_ref_count: usize,
}
impl Stack {
@@ -324,6 +339,7 @@ impl Stack {
pub fn new() -> Self {
Self {
inner: Default::default(),
+ gc_ref_count: Default::default(),
}
}
@@ -356,13 +372,16 @@ impl Stack {
/// Extend the stack with the given elements.
pub fn extend(&mut self, values: impl IntoIterator<Item = Val>) {
- self.inner.extend(values);
+ for val in values {
+ self.push(val);
+ }
}
/// Inserts many values at the given index.
pub fn insert_many(&mut self, at: usize, values: &[Val]) {
debug_assert!(at <= self.len());
+ self.gc_ref_count += values.iter().filter(|v| v.needs_stack_map()).count();
if at == self.len() {
self.inner.extend_from_slice(values);
} else {
@@ -377,6 +396,9 @@ impl Stack {
/// Push a value to the stack.
pub fn push(&mut self, val: Val) {
+ if val.needs_stack_map() {
+ self.gc_ref_count += 1;
+ }
self.inner.push(val);
}
@@ -397,7 +419,13 @@ impl Stack {
/// Pops the top element of the stack, if any.
pub fn pop(&mut self) -> Option<Val> {
- self.inner.pop()
+ let val = self.inner.pop();
+ if let Some(v) = val
+ && v.needs_stack_map()
+ {
+ self.gc_ref_count -= 1;
+ }
+ val
}
/// Pops the element at the top of the stack if it is an i32 const;
@@ -457,10 +485,19 @@ impl Stack {
}
/// Get a mutable reference to the inner stack representation.
+ ///
+ /// Mutations through this reference must not add or remove GC
+ /// references or the gc ref count will go stale; changing a value's
+ /// representation (e.g. spilling a register to memory) is fine.
pub fn inner_mut(&mut self) -> &mut SmallVec<[Val; 64]> {
&mut self.inner
}
+ /// The number of values on the stack that stack maps must cover.
+ pub fn gc_ref_count(&self) -> usize {
+ self.gc_ref_count
+ }
+
/// Get a reference to the inner stack representation.
pub fn inner(&self) -> &SmallVec<[Val; 64]> {
&self.inner
@@ -477,6 +514,13 @@ impl Stack {
}
})
}
+
+ /// Truncate the stack to `len`, keeping the gc ref count in sync.
+ pub fn truncate(&mut self, len: usize) {
+ while self.inner.len() > len {
+ self.pop();
+ }
+ }
}
#[cfg(test)]
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 2d6228c58c..791f7354ee 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -1663,7 +1663,9 @@ where
match slot.ty {
I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)),
Ref(rt) => match rt.heap_type {
- WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)),
+ WasmHeapType::Func | WasmHeapType::Extern => {
+ context.stack.push(Val::local(index, slot.ty))
+ }
_ => bail!(CodeGenError::unsupported_wasm_type()),
},
}
@@ -2130,6 +2132,10 @@ where
}
Ok(())
}
+ HeapType::EXTERN => {
+ self.context.stack.push(Val::i32(0));
+ Ok(())
+ }
_ => Err(format_err!(CodeGenError::unsupported_wasm_type())),
}
}
@@ -4620,7 +4626,7 @@ impl TryFrom<WasmValType> for OperandSize {
// to be updated in such a way that the calculation of the
// OperandSize will depend on the target's pointer size.
WasmHeapType::Func => OperandSize::S64,
- WasmHeapType::Extern => OperandSize::S64,
+ WasmHeapType::Extern => OperandSize::S32,
_ => bail!(CodeGenError::unsupported_wasm_type()),
}
}