A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch: stackmaps and value refs (#14066)
Daniel Macoveic2bf66bb9ad8
diff --git a/cranelift/codegen/src/ir/user_stack_maps.rs b/cranelift/codegen/src/ir/user_stack_maps.rsindex 6ad9b697cd..fb0382cfb0 100644--- a/cranelift/codegen/src/ir/user_stack_maps.rs+++ b/cranelift/codegen/src/ir/user_stack_maps.rs@@ -171,6 +171,23 @@ impl UserStackMap {}}+ /// Construct a stack map from offsets that are already relative to the+ /// stack pointer at the safepoint.+ pub fn from_sp_offsets(offsets: impl IntoIterator<Item = u32>) -> Self {+ let offsets: alloc::vec::Vec<u32> = offsets.into_iter().collect();+ let max = offsets.iter().max().copied().unwrap_or(0);+ let mut bitset = CompoundBitSet::with_capacity(usize::try_from(max).unwrap() + 1);+ for offset in offsets {+ bitset.insert(usize::try_from(offset).unwrap());+ }+ let mut by_type = SmallVec::<[(ir::Type, CompoundBitSet); 1]>::default();+ by_type.push((ir::types::I32, bitset));+ UserStackMap {+ by_type,+ sp_to_sized_stack_slots: Some(0),+ }+ }+/// Finalize this stack map by filling in the SP-to-stack-slots offset.pub(crate) fn finalize(&mut self, sp_to_sized_stack_slots: u32) {debug_assert!(self.sp_to_sized_stack_slots.is_none());@@ -197,3 +214,31 @@ impl UserStackMap {})}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn from_sp_offsets_roundtrip() {+ let map = UserStackMap::from_sp_offsets([8, 16, 40]);+ let entries: alloc::vec::Vec<_> = map.entries().collect();+ assert_eq!(entries.len(), 3);+ assert_eq!(entries[0], (ir::types::I32, 8));+ assert_eq!(entries[1], (ir::types::I32, 16));+ assert_eq!(entries[2], (ir::types::I32, 40));+ }++ #[test]+ fn from_sp_offsets_empty() {+ let map = UserStackMap::from_sp_offsets([]);+ assert_eq!(map.entries().count(), 0);+ }++ #[test]+ fn from_sp_offsets_dedups_and_sorts() {+ let map = UserStackMap::from_sp_offsets([40, 8, 8, 16]);+ let offsets: alloc::vec::Vec<u32> = map.entries().map(|(_, o)| o).collect();+ assert_eq!(offsets, [8, 16, 40]);+ }+}diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rsindex fba4ee3d81..6f6e8564e7 100644--- a/cranelift/codegen/src/machinst/buffer.rs+++ b/cranelift/codegen/src/machinst/buffer.rs@@ -1844,6 +1844,23 @@ impl<I: VCodeInst> MachBuffer<I> {self.user_stack_maps.push((return_addr, span, stack_map));}+ /// Push a user stack map whose offsets are already relative to the stack+ /// pointer at the safepoint, with an explicitly provided frame size.+ pub fn push_user_stack_map_sp_relative(+ &mut self,+ return_addr: CodeOffset,+ frame_size: u32,+ stack_map: ir::UserStackMap,+ ) {+ debug_assert!(+ self.user_stack_maps+ .last()+ .map_or(true, |(prev_addr, _, _)| *prev_addr < return_addr),+ );+ self.user_stack_maps+ .push((return_addr, frame_size, stack_map));+ }+/// Push a debug tag associated with the current buffer offset.pub fn push_debug_tags(&mut self, pos: MachDebugTagPos, tags: &[DebugTag]) {trace!("debug tags at offset {}: {tags:?}", self.cur_offset());diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex b6ec104358..835c41e4f2 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -454,7 +454,6 @@ impl Compiler {|| config.tail_call()|| config.function_references()|| config.relaxed_simd()- || config.gc_types()|| config.exceptions()|| config.legacy_exceptions()|| config.stack_switching()@@ -563,6 +562,16 @@ impl WastTest {return true;}+ // Winch supports GC types only under the barrier-free collectors;+ // the deferred reference-counting collector is refused until Winch+ // emits GC barriers.+ if config.compiler == Compiler::Winch+ && config.collector == Collector::DeferredReferenceCounting+ && self.config.gc_types()+ {+ return true;+ }+// Disable spec tests per target for proposals that Winch does not implement yet.if config.compiler == Compiler::Winch {// Common list for tests that fail in all targets supported by Winch.@@ -570,16 +579,10 @@ impl WastTest {"extended-const/elem.wast","extended-const/global.wast","misc_testsuite/component-model/modules.wast",- "misc_testsuite/externref-id-function.wast",- "misc_testsuite/externref-segment.wast","misc_testsuite/externref-segments.wast","misc_testsuite/externref-table-dropped-segment-issue-8281.wast",- "misc_testsuite/linking-errors.wast","misc_testsuite/many_table_gets_lead_to_gc.wast",- "misc_testsuite/mutable_externref_globals.wast",- "misc_testsuite/no-mixup-stack-maps.wast","misc_testsuite/no-panic.wast",- "misc_testsuite/simple_ref_is_null.wast",];if unsupported.iter().any(|part| self.path.ends_with(part)) {diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 6534da5b5e..98090ecfc4 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -2454,10 +2454,17 @@ impl Config {| WasmFeatures::FUNCTION_REFERENCES| WasmFeatures::RELAXED_SIMD| WasmFeatures::TAIL_CALL- | WasmFeatures::GC_TYPES| WasmFeatures::EXCEPTIONS| WasmFeatures::LEGACY_EXCEPTIONS| WasmFeatures::STACK_SWITCHING;++ // Winch supports GC types only under the barrier-free+ // collectors; the deferred reference-counting collector+ // requires GC barriers that Winch does not emit yet.+ #[cfg(feature = "gc")]+ if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) {+ unsupported |= WasmFeatures::GC_TYPES;+ }match self.compiler_target().architecture {target_lexicon::Architecture::Aarch64(_) => {unsupported |= WasmFeatures::THREADS;diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex 3058855e33..ac381877f7 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -3788,3 +3788,107 @@ fn initial_size_larger_than_reservation() -> Result<()> {Ok(())}++/// Under Winch, a frame holding the only reference to an `externref` across a+/// collection must keep it alive. Runs under the barrier-free collectors that Winch supports.+#[test]+#[cfg_attr(miri, ignore)]+fn winch_externref_survives_gc_in_frame() -> Result<()> {+ for collector in [Collector::Null, Collector::Copying] {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(collector);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };+ let module = Module::new(+ &engine,+ r#"+ (module+ (import "" "make" (func $make (result externref)))+ (import "" "gc" (func $gc))+ (func (export "hold") (result externref)+ ;; the only reference to the object is on this frame's value+ ;; stack while the collection runs+ (call $make)+ (call $gc)))+ "#,+ )?;+ let mut store = Store::new(&engine, ());+ let make = Func::wrap(+ &mut store,+ |mut cx: Caller<'_, ()>| -> Result<Option<Rooted<ExternRef>>> {+ Ok(Some(ExternRef::new(&mut cx, 0xDECAFu32)?))+ },+ );+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {+ let _ = cx.gc(None);+ });+ let instance = Instance::new(&mut store, &module, &[make.into(), gc.into()])?;+ let hold = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "hold")?;+ let out = hold.call(&mut store, ())?.expect("must not be null");+ let got = out+ .data(&store)?+ .and_then(|d| d.downcast_ref::<u32>().copied());+ assert_eq!(+ got,+ Some(0xDECAF),+ "externref did not survive GC under {collector:?}"+ );+ }+ Ok(())+}++/// Reference values crossing the ABI boundary in every position: stack-passed+/// externref params and multi-value externref results (more than fit in registers)+#[test]+#[cfg_attr(miri, ignore)]+fn winch_ref_params_and_results_across_gc() -> Result<()> {+ for (nparams, nresults) in [(12, 12), (2, 12), (12, 3), (12, 1)] {+ let params = "externref ".repeat(nparams);+ let results = "externref ".repeat(nresults);+ let gets: String = (0..nresults)+ .map(|i| format!("(local.get {})", i % nparams))+ .collect();+ let wat = format!(+ r#"(module+ (import "" "gc" (func $gc))+ (func (export "hold") (param {params}) (result {results})+ (call $gc)+ {gets}))"#+ );+ for collector in [Collector::Null, Collector::Copying] {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(collector);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };+ let module = Module::new(&engine, &wat)?;+ let mut store = Store::new(&engine, ());+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {+ let _ = cx.gc(None);+ });+ let instance = Instance::new(&mut store, &module, &[gc.into()])?;+ let hold = instance.get_func(&mut store, "hold").unwrap();+ let args: Vec<Val> = (0..nparams as u32)+ .map(|i| Ok(Val::ExternRef(Some(ExternRef::new(&mut store, i)?))))+ .collect::<Result<_>>()?;+ let mut outs = vec![Val::null_extern_ref(); nresults];+ hold.call(&mut store, &args, &mut outs)?;+ for (i, v) in outs.iter().enumerate() {+ let id = v+ .unwrap_externref()+ .expect("result became null")+ .data(&store)?+ .and_then(|d| d.downcast_ref::<u32>().copied());+ assert_eq!(+ id,+ Some((i % nparams) as u32),+ "result {i} corrupted under {collector:?} ({nparams}p/{nresults}r)"+ );+ }+ }+ }+ Ok(())+}diff --git a/tests/all/tags.rs b/tests/all/tags.rsindex 0db6793f18..b820a27687 100644--- a/tests/all/tags.rs+++ b/tests/all/tags.rs@@ -201,6 +201,13 @@ fn stack_switching_disallows_inlining() -> Result<()> {fn issue_13474_create_tag_without_gc_runtime_configured() -> Result<()> {let mut config = Config::new();config.strategy(Strategy::Winch);+ // Disable the GC runtime explicitly, along with the proposals that+ // require it, to exercise creating a tag without a GC runtime; this+ // previously relied on Winch stripping these features by default.+ config.gc_support(false);+ config.wasm_exceptions(false);+ config.wasm_gc(false);+ config.wasm_function_references(false);// Ignore targets that don't have support for Winch just yetlet Ok(engine) = Engine::new(&config) else {return Ok(());diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rsindex fe252842a1..966588eb07 100644--- a/tests/all/winch_engine_features.rs+++ b/tests/all/winch_engine_features.rs@@ -64,3 +64,41 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> ROk(())}++#[wasmtime_test(strategies(only(Winch)))]+#[cfg_attr(miri, ignore)]+fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> {+ config.collector(Collector::DeferredReferenceCounting);+ config.gc_support(true);+ let result = Engine::new(&config);+ match result {+ Ok(_) => {+ wasmtime::bail!(+ "Expected incompatibility between the deferred reference-counting \+ collector and Winch"+ )+ }+ Err(e) => {+ assert_eq!(+ e.to_string(),+ "the wasm_gc_types feature is not supported on this compiler configuration"+ );+ }+ }++ Ok(())+}++#[wasmtime_test(strategies(only(Winch)))]+#[cfg_attr(miri, ignore)]+fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> {+ config.collector(Collector::DeferredReferenceCounting);+ let engine = Engine::new(&config)?;+ let result = Module::new(+ &engine,+ r#"(module (global (mut externref) (ref.null extern)))"#,+ );+ assert!(result.is_err());++ Ok(())+}diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wastnew file mode 100644index 0000000000..4f242b0223--- /dev/null+++ b/tests/misc_testsuite/winch/gc-refs.wast@@ -0,0 +1,28 @@+;;! reference_types = true+;;! gc_types = true++;; Winch reference-value support: externref flows through params, results,+;; locals, and globals, and survives calls (stack maps). Runs under the+;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers.++(module+ (global $g (mut externref) (ref.null extern))++ (func $nop)++ (func (export "roundtrip") (param externref) (result externref)+ (local $l externref)+ (local.set $l (local.get 0))+ (call $nop)+ (local.get $l))++ (func (export "via_global") (param externref) (result externref)+ (global.set $g (local.get 0))+ (global.get $g))++ (func (export "null_is_null") (result i32)+ (ref.is_null (global.get $g))))++(assert_return (invoke "null_is_null") (i32.const 1))+(assert_return (invoke "roundtrip" (ref.null extern)) (ref.null extern))+(assert_return (invoke "via_global" (ref.null extern)) (ref.null extern))diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rsindex 0f54c9254c..e0de2888bb 100644--- a/winch/codegen/src/codegen/call.rs+++ b/winch/codegen/src/codegen/call.rs@@ -96,10 +96,22 @@ impl FnCall {context.spill(masm)?;let ret_area = Self::make_ret_area(&sig, masm)?;let arg_stack_space = sig.params_stack_size();- let reserved_stack = masm.call(arg_stack_space, |masm| {- Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;- Ok((kind, sig.call_conv))- })?;+ let reserved_stack = masm.call(+ arg_stack_space,+ context,+ |masm, context| {+ Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;+ Ok((kind, sig.call_conv))+ },+ |masm, context| {+ let sp = masm.sp_offset()?;+ let offsets = context.calculate_stack_map_offsets(sp)?;+ if !offsets.is_empty() {+ masm.emit_stack_map(sp, &offsets)?;+ }+ Ok(())+ },+ )?;Self::cleanup(sig,diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rsindex 41a4fc3ed7..f31c336b6e 100644--- a/winch/codegen/src/codegen/context.rs+++ b/winch/codegen/src/codegen/context.rs@@ -4,7 +4,7 @@ use crate::{abi::{ABIOperand, ABIResults, RetArea, vmctx},bail,codegen::{BranchState, CodeGenError, CodeGenPhase, Emission, Prologue},- ensure,+ ensure, format_err,frame::Frame,isa::reg::RegClass,masm::{@@ -15,6 +15,7 @@ use crate::{regalloc::RegAlloc,stack::{Stack, TypedReg, Val},};+use smallvec::SmallVec;use wasmparser::{Ieee32, Ieee64};use wasmtime_environ::{VMOffsets, WasmHeapType, WasmValType};@@ -636,18 +637,55 @@ impl<'a> CodeGenContext<'a, Emission> {let len = self.stack.len();ensure!(last <= len, CodeGenError::unexpected_value_stack_index(),);let truncate = self.stack.len() - last;- let stack_mut = self.stack.inner_mut();// Invoke the callback in top-to-bottom order.- for v in stack_mut[truncate..].into_iter().rev() {+ for v in self.stack.inner()[truncate..].iter().rev() {f(&mut self.regalloc, v)?}- stack_mut.truncate(truncate);+ self.stack.truncate(truncate);}Ok(())}+ /// Calculate the stack map offsets for a call site at the given stack+ /// pointer offset: the distance from the stack pointer to every slot+ /// holding a live GC reference, in the frame's locals and in the value+ /// stack.+ pub fn calculate_stack_map_offsets(&self, sp: SPOffset) -> Result<SmallVec<[SPOffset; 8]>> {+ if self.frame.gc_ref_local_offsets().is_empty() && self.stack.gc_ref_count() == 0 {+ return Ok(SmallVec::new());+ }++ let mut offsets: SmallVec<[SPOffset; 8]> = SmallVec::new();++ for offset in self.frame.gc_ref_local_offsets() {+ offsets.push(+ sp.checked_sub(*offset)+ .ok_or_else(|| format_err!(CodeGenError::invalid_local_offset()))?,+ );+ }++ let mut remaining = self.stack.gc_ref_count();+ if remaining != 0 {+ for v in self.stack.inner() {+ if v.needs_stack_map() {+ ensure!(v.is_mem(), CodeGenError::unexpected_value_in_value_stack());+ offsets.push(+ sp.checked_sub(v.unwrap_mem().slot.offset)+ .ok_or_else(|| format_err!(CodeGenError::invalid_sp_offset()))?,+ );+ remaining -= 1;+ if remaining == 0 {+ break;+ }+ }+ }+ }++ Ok(offsets)+ }+/// Convenience wrapper around [`Self::spill_callback`].////// This function exists for cases in which triggering an unconditionaldiff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex ac0fd0b1c1..0df13d76d9 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -1,6 +1,6 @@use crate::{Result,- abi::{ABIOperand, ABISig, RetArea, vmctx},+ abi::{ABI, ABIOperand, ABISig, LocalSlot, RetArea, vmctx},bail,codegen::BlockSig,ensure, format_err,@@ -168,6 +168,7 @@ whereself.masm.reserve_stack(self.context.frame.locals_size)?;self.spill_register_arguments()?;+ self.copy_stack_gc_refs_to_frame()?;let defined_locals_range = &self.context.frame.defined_locals_range;self.masm.zero_mem_range(defined_locals_range.as_range())?;@@ -222,9 +223,12 @@ whereself.masm.store((*reg).into(), addr, (*ty).try_into()?)?;}Ref(rt) => match rt.heap_type {- WasmHeapType::Func | WasmHeapType::Extern => {+ WasmHeapType::Func => {self.masm.store_ptr(*reg, addr)?;}+ WasmHeapType::Extern => {+ self.masm.store((*reg).into(), addr, (*ty).try_into()?)?;+ }_ => bail!(CodeGenError::unsupported_wasm_type()),},}@@ -235,6 +239,40 @@ where}Ok(())}++ /// Copy GC references passed on the stack into frame slots so that+ /// stack maps cover them: the caller's argument area is not visited by+ /// the collector, so a reference left there goes stale across a+ /// collection. Everything else stays in the caller's argument area.+ fn copy_stack_gc_refs_to_frame(&mut self) -> Result<()> {+ for (operand, slot) in self+ .sig+ .params_without_retptr()+ .iter()+ .zip(self.context.frame.locals())+ {+ match (operand, slot) {+ (ABIOperand::Stack { ty, offset, .. }, slot)+ if ty.is_vmgcref_type_and_not_i31() =>+ {+ ensure!(+ slot.addressed_from_sp(),+ CodeGenError::sp_addressing_expected(),+ );+ let arg_base = u32::from(<M::ABI as ABI>::arg_base_offset());+ let src = LocalSlot::stack_arg(*ty, offset + arg_base);+ let src_addr = self.masm.local_address(&src)?;+ let dst_addr = self.masm.local_address(slot)?;+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.load(src_addr, scratch.writable(), (*ty).try_into()?)?;+ masm.store(scratch.inner().into(), dst_addr, (*ty).try_into()?)+ })?;+ }+ _ => {}+ }+ }+ Ok(())+ }}impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>diff --git a/winch/codegen/src/frame/mod.rs b/winch/codegen/src/frame/mod.rsindex 8846b3bd70..9b2e907c38 100644--- a/winch/codegen/src/frame/mod.rs+++ b/winch/codegen/src/frame/mod.rs@@ -2,7 +2,8 @@ use crate::{Result,abi::{ABI, ABIOperand, ABISig, LocalSlot, align_to},codegen::{CodeGenPhase, Emission, Prologue},- masm::MacroAssembler,+ masm::{MacroAssembler, SPOffset},+ stack::needs_stack_map,};use smallvec::SmallVec;use std::marker::PhantomData;@@ -94,22 +95,27 @@ pub(crate) struct Frame<P: CodeGenPhase> {/// The slot holding the address of the results area.pub results_base_slot: Option<LocalSlot>,marker: PhantomData<P>,++ /// Frame offsets of SP-addressed locals that stack maps must cover,+ /// precomputed so that call sites don't scan every local.+ gc_ref_local_offsets: SmallVec<[SPOffset; 4]>,}impl Frame<Prologue> {/// Allocate a new [`Frame`].pub fn new<A: ABI>(sig: &ABISig, defined_locals: &DefinedLocals) -> Result<Frame<Prologue>> {- let (special_locals, mut wasm_locals, defined_locals_start) =+ let (special_locals, mut wasm_locals, mut gc_ref_local_offsets, defined_locals_start) =Self::compute_arg_slots::<A>(sig)?;// The defined locals have a zero-based offset by default// so we need to add the defined locals start to the offset.- wasm_locals.extend(- defined_locals- .defined_locals- .iter()- .map(|l| LocalSlot::new(l.ty, l.offset + defined_locals_start)),- );+ wasm_locals.extend(defined_locals.defined_locals.iter().map(|l| {+ let slot = LocalSlot::new(l.ty, l.offset + defined_locals_start);+ if needs_stack_map(&slot.ty) {+ gc_ref_local_offsets.push(SPOffset::from_u32(slot.offset));+ }+ slot+ }));let stack_align = <A as ABI>::stack_align();let defined_locals_end = align_to(@@ -157,6 +163,7 @@ impl Frame<Prologue> {),results_base_slot,marker: PhantomData,+ gc_ref_local_offsets,})}@@ -175,10 +182,13 @@ impl Frame<Prologue> {defined_locals_range: self.defined_locals_range,results_base_slot: self.results_base_slot,marker: PhantomData,+ gc_ref_local_offsets: self.gc_ref_local_offsets,}}- fn compute_arg_slots<A: ABI>(sig: &ABISig) -> Result<(SpecialLocals, WasmLocals, u32)> {+ fn compute_arg_slots<A: ABI>(+ sig: &ABISig,+ ) -> Result<(SpecialLocals, WasmLocals, SmallVec<[SPOffset; 4]>, u32)> {// Go over the function ABI-signature and// calculate the stack slots.//@@ -225,11 +235,23 @@ impl Frame<Prologue> {.map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)).expect("Slot for VMContext");+ let mut gc_ref_local_offsets = SmallVec::new();let slots: WasmLocals = params_iter- .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset))+ .map(|arg| {+ let slot = Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset);+ if slot.addressed_from_sp() && needs_stack_map(&slot.ty) {+ gc_ref_local_offsets.push(SPOffset::from_u32(slot.offset));+ }+ slot+ }).collect();- Ok(([callee_vmctx, caller_vmctx], slots, next_stack))+ Ok((+ [callee_vmctx, caller_vmctx],+ slots,+ gc_ref_local_offsets,+ next_stack,+ ))}fn abi_arg_slot(arg: &ABIOperand, next_stack: &mut u32, arg_base_offset: u32) -> LocalSlot {@@ -241,7 +263,15 @@ impl Frame<Prologue> {LocalSlot::new(*ty, *next_stack)}// Create a local slot, with an offset from the arguments base in- // the stack; which is the frame pointer + return address.+ // the stack; which is the frame pointer + return address. GC+ // references are re-homed into a frame slot by the prologue so+ // that stack maps cover them: the caller's outgoing argument+ // area is not visited by the collector, so a reference left+ // there goes stale across a collection.+ ABIOperand::Stack { ty, size, .. } if ty.is_vmgcref_type_and_not_i31() => {+ *next_stack = align_to(*next_stack, *size) + *size;+ LocalSlot::new(*ty, *next_stack)+ }ABIOperand::Stack { ty, offset, .. } => {LocalSlot::stack_arg(*ty, offset + arg_base_offset)}@@ -279,6 +309,11 @@ impl Frame<Emission> {self.get_special_local(0)}+ /// Frame offsets of the SP-addressed locals that stack maps must cover.+ pub fn gc_ref_local_offsets(&self) -> &[SPOffset] {+ &self.gc_ref_local_offsets+ }+/// Returns the address of the local at the given index.////// # Panicsdiff --git a/winch/codegen/src/isa/aarch64/abi.rs b/winch/codegen/src/isa/aarch64/abi.rsindex ad933eea20..a65b12badd 100644--- a/winch/codegen/src/isa/aarch64/abi.rs+++ b/winch/codegen/src/isa/aarch64/abi.rs@@ -131,6 +131,7 @@ impl ABI for Aarch64ABI {match ty {WasmValType::Ref(rt) => match rt.heap_type {WasmHeapType::Func => Self::word_bytes(),+ WasmHeapType::Extern => Self::word_bytes() / 2,ht => unimplemented!("Support for WasmHeapType: {ht}"),},WasmValType::F64 | WasmValType::I64 => Self::word_bytes(),@@ -212,6 +213,22 @@ mod tests {WasmValType::{self, *},};+ #[test]+ fn ref_sizes_match_cranelift_representation() {+ use wasmtime_environ::{WasmHeapType, WasmRefType};+ let ty = |heap_type| {+ WasmValType::Ref(WasmRefType {+ nullable: true,+ heap_type,+ })+ };+ assert_eq!(+ Aarch64ABI::sizeof(&ty(WasmHeapType::Func)),+ Aarch64ABI::word_bytes()+ );+ assert_eq!(Aarch64ABI::sizeof(&ty(WasmHeapType::Extern)), 4);+ }+#[test]fn xreg_abi_sig() -> Result<()> {let wasm_sig = WasmFuncType::new([I32, I64, I32, I64, I32, I32, I64, I32, I64], [])?;diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 9ac7842106..276bbfd4dc 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -419,7 +419,12 @@ impl Masm for MacroAssembler {fn call(&mut self,stack_args_size: u32,- mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,+ context: &mut CodeGenContext<Emission>,+ mut load_callee: impl FnMut(+ &mut Self,+ &mut CodeGenContext<Emission>,+ ) -> Result<(CalleeKind, CallingConvention)>,+ mut finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,) -> Result<u32> {let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into();@@ -427,11 +432,12 @@ impl Masm for MacroAssembler {let aligned_args_size = align_to(stack_args_size, alignment);let total_stack = delta + aligned_args_size;self.reserve_stack(total_stack)?;- let (callee, call_conv) = load_callee(self)?;+ let (callee, call_conv) = load_callee(self, context)?;match callee {CalleeKind::Indirect(reg) => self.asm.call_with_reg(reg, call_conv),CalleeKind::Direct(idx) => self.asm.call_with_name(idx, call_conv),}+ finalize(self, context)?;Ok(total_stack)}@@ -1344,6 +1350,18 @@ impl Masm for MacroAssembler {Ok(())}+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()> {+ let frame_size = sp_offset.as_u32() + u32::from(SHADOW_STACK_POINTER_SLOT_SIZE);+ let return_addr = self.asm.buffer().cur_offset();+ let map = cranelift_codegen::ir::UserStackMap::from_sp_offsets(+ offsets.iter().map(SPOffset::as_u32),+ );+ self.asm+ .buffer_mut()+ .push_user_stack_map_sp_relative(return_addr, frame_size, map);+ Ok(())+ }+fn current_code_offset(&self) -> Result<CodeOffset> {Ok(self.asm.buffer().cur_offset())}diff --git a/winch/codegen/src/isa/x64/abi.rs b/winch/codegen/src/isa/x64/abi.rsindex 7ec1433ca0..83c1c81606 100644--- a/winch/codegen/src/isa/x64/abi.rs+++ b/winch/codegen/src/isa/x64/abi.rs@@ -115,7 +115,8 @@ impl ABI for X64ABI {fn sizeof(ty: &WasmValType) -> u8 {match ty {WasmValType::Ref(rt) => match rt.heap_type {- WasmHeapType::Func | WasmHeapType::Extern => Self::word_bytes(),+ WasmHeapType::Func => Self::word_bytes(),+ WasmHeapType::Extern => Self::word_bytes() / 2,ht => unimplemented!("Support for WasmHeapType: {ht}"),},WasmValType::F64 | WasmValType::I64 => Self::word_bytes(),diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex 0044af0222..a455f828dd 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -326,7 +326,12 @@ impl Masm for MacroAssembler {fn call(&mut self,stack_args_size: u32,- mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,+ context: &mut CodeGenContext<Emission>,+ mut load_callee: impl FnMut(+ &mut Self,+ &mut CodeGenContext<Emission>,+ ) -> Result<(CalleeKind, CallingConvention)>,+ mut finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,) -> Result<u32> {let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into();@@ -334,11 +339,13 @@ impl Masm for MacroAssembler {let aligned_args_size = align_to(stack_args_size, alignment);let total_stack = delta + aligned_args_size;self.reserve_stack(total_stack)?;- let (callee, cc) = load_callee(self)?;+ let (callee, cc) = load_callee(self, context)?;match callee {CalleeKind::Indirect(reg) => self.asm.call_with_reg(cc, reg),CalleeKind::Direct(idx) => self.asm.call_with_name(cc, idx),};+ finalize(self, context)?;+Ok(total_stack)}@@ -1400,6 +1407,18 @@ impl Masm for MacroAssembler {Ok(())}+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()> {+ let frame_size = sp_offset.as_u32();+ let return_addr = self.asm.buffer().cur_offset();+ let map = cranelift_codegen::ir::UserStackMap::from_sp_offsets(+ offsets.iter().map(SPOffset::as_u32),+ );+ self.asm+ .buffer_mut()+ .push_user_stack_map_sp_relative(return_addr, frame_size, map);+ Ok(())+ }+fn current_code_offset(&self) -> Result<CodeOffset> {Ok(self.asm.buffer().cur_offset())}diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rsindex 62c8318c15..b64cd66551 100644--- a/winch/codegen/src/masm.rs+++ b/winch/codegen/src/masm.rs@@ -199,6 +199,10 @@ impl SPOffset {pub fn as_u32(&self) -> u32 {self.0}++ pub fn checked_sub(self, rhs: Self) -> Option<Self> {+ self.0.checked_sub(rhs.0).map(Self)+ }}/// A stack slot.@@ -1453,9 +1457,20 @@ pub(crate) trait MacroAssembler {fn call(&mut self,stack_args_size: u32,- f: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,+ context: &mut CodeGenContext<Emission>,+ f: impl FnMut(+ &mut Self,+ &mut CodeGenContext<Emission>,+ ) -> Result<(CalleeKind, CallingConvention)>,+ finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,) -> Result<u32>;+ /// Record a GC stack map at the current code offset, which must be the+ /// return address of the call emitted immediately before. Each offset is+ /// the distance from the stack pointer at the call site to a slot holding+ /// a live GC reference.+ fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()>;+/// Acquire a scratch register and execute the given callback.fn with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R;@@ -1470,7 +1485,7 @@ pub(crate) trait MacroAssembler {WasmValType::I32| WasmValType::I64| WasmValType::Ref(WasmRefType {- heap_type: WasmHeapType::Func,+ heap_type: WasmHeapType::Func | WasmHeapType::Extern,..}) => self.with_scratch::<IntScratch, _>(f),WasmValType::F32 | WasmValType::F64 | WasmValType::V128 => {diff --git a/winch/codegen/src/stack.rs b/winch/codegen/src/stack.rsindex 424d05f663..af051d19ad 100644--- a/winch/codegen/src/stack.rs+++ b/winch/codegen/src/stack.rs@@ -310,6 +310,20 @@ impl Val {Val::Local(l) => l.ty,}}++ /// Whether this value is a GC reference that must be covered by stack maps.+ pub fn needs_stack_map(&self) -> bool {+ needs_stack_map(&self.ty())+ }+}++/// Whether a value of this type must be covered by stack maps.+pub(crate) fn needs_stack_map(ty: &WasmValType) -> bool {+ ty.is_vmgcref_type_and_not_i31()+ && match ty {+ WasmValType::Ref(r) => !r.heap_type.is_bottom(),+ _ => false,+ }}/// The shadow stack used for compilation.@@ -317,6 +331,7 @@ impl Val {pub(crate) struct Stack {// NB: The 64 is chosen arbitrarily. We can adjust as we see fit.inner: SmallVec<[Val; 64]>,+ gc_ref_count: usize,}impl Stack {@@ -324,6 +339,7 @@ impl Stack {pub fn new() -> Self {Self {inner: Default::default(),+ gc_ref_count: Default::default(),}}@@ -356,13 +372,16 @@ impl Stack {/// Extend the stack with the given elements.pub fn extend(&mut self, values: impl IntoIterator<Item = Val>) {- self.inner.extend(values);+ for val in values {+ self.push(val);+ }}/// Inserts many values at the given index.pub fn insert_many(&mut self, at: usize, values: &[Val]) {debug_assert!(at <= self.len());+ self.gc_ref_count += values.iter().filter(|v| v.needs_stack_map()).count();if at == self.len() {self.inner.extend_from_slice(values);} else {@@ -377,6 +396,9 @@ impl Stack {/// Push a value to the stack.pub fn push(&mut self, val: Val) {+ if val.needs_stack_map() {+ self.gc_ref_count += 1;+ }self.inner.push(val);}@@ -397,7 +419,13 @@ impl Stack {/// Pops the top element of the stack, if any.pub fn pop(&mut self) -> Option<Val> {- self.inner.pop()+ let val = self.inner.pop();+ if let Some(v) = val+ && v.needs_stack_map()+ {+ self.gc_ref_count -= 1;+ }+ val}/// Pops the element at the top of the stack if it is an i32 const;@@ -457,10 +485,19 @@ impl Stack {}/// Get a mutable reference to the inner stack representation.+ ///+ /// Mutations through this reference must not add or remove GC+ /// references or the gc ref count will go stale; changing a value's+ /// representation (e.g. spilling a register to memory) is fine.pub fn inner_mut(&mut self) -> &mut SmallVec<[Val; 64]> {&mut self.inner}+ /// The number of values on the stack that stack maps must cover.+ pub fn gc_ref_count(&self) -> usize {+ self.gc_ref_count+ }+/// Get a reference to the inner stack representation.pub fn inner(&self) -> &SmallVec<[Val; 64]> {&self.inner@@ -477,6 +514,13 @@ impl Stack {}})}++ /// Truncate the stack to `len`, keeping the gc ref count in sync.+ pub fn truncate(&mut self, len: usize) {+ while self.inner.len() > len {+ self.pop();+ }+ }}#[cfg(test)]diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rsindex 2d6228c58c..791f7354ee 100644--- a/winch/codegen/src/visitor.rs+++ b/winch/codegen/src/visitor.rs@@ -1663,7 +1663,9 @@ wherematch slot.ty {I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)),Ref(rt) => match rt.heap_type {- WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)),+ WasmHeapType::Func | WasmHeapType::Extern => {+ context.stack.push(Val::local(index, slot.ty))+ }_ => bail!(CodeGenError::unsupported_wasm_type()),},}@@ -2130,6 +2132,10 @@ where}Ok(())}+ HeapType::EXTERN => {+ self.context.stack.push(Val::i32(0));+ Ok(())+ }_ => Err(format_err!(CodeGenError::unsupported_wasm_type())),}}@@ -4620,7 +4626,7 @@ impl TryFrom<WasmValType> for OperandSize {// to be updated in such a way that the calculation of the// OperandSize will depend on the target's pointer size.WasmHeapType::Func => OperandSize::S64,- WasmHeapType::Extern => OperandSize::S64,+ WasmHeapType::Extern => OperandSize::S32,_ => bail!(CodeGenError::unsupported_wasm_type()),}}