Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch: drc barriers (#14081)

Daniel Macoveif08de396d2bd
diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs
index fe75676554..8a9de06f63 100644
--- a/crates/cranelift/src/func_environ/gc/drc.rs
+++ b/crates/cranelift/src/func_environ/gc/drc.rs
@@ -15,10 +15,6 @@ use wasmtime_environ::{
WasmValType, drc::DrcTypeLayouts,
};
-// The minimum over-approximated stack roots list size for which we will trigger
-// a GC.
-const MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024;
-
#[derive(Default)]
pub struct DrcCompiler {
layouts: DrcTypeLayouts,
@@ -190,9 +186,10 @@ impl DrcCompiler {
);
let doubled_last_len = builder.ins().iadd(last_len, last_len);
- let min_threshold = builder
- .ins()
- .iconst(ir::types::I32, MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD);
+ let min_threshold = builder.ins().iconst(
+ ir::types::I32,
+ wasmtime_environ::drc::MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD,
+ );
let threshold = builder.ins().umax(doubled_last_len, min_threshold);
let should_gc =
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index 50c0b26cbe..150726f70c 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -49,7 +49,7 @@ mod trap;
use self::compiler::Compiler;
const TRAP_INTERNAL_ASSERT: TrapCode = TrapCode::unwrap_user(1);
-const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2);
+pub const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2);
const TRAP_OFFSET: u8 = 3;
pub const TRAP_CANNOT_LEAVE_COMPONENT: TrapCode =
TrapCode::unwrap_user(Trap::CannotLeaveComponent as u8 + TRAP_OFFSET);
diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs
index 5a5c9d64bf..a560e7f918 100644
--- a/crates/environ/src/gc.rs
+++ b/crates/environ/src/gc.rs
@@ -29,6 +29,19 @@ use core::alloc::Layout;
/// enabled.
pub const POISON: u8 = 0b00001111;
+/// The bit within a `VMDrcHeader`'s reserved bits that is the mark
+/// bit. Collectively, this bit in all the heap's objects' headers implements
+/// the precise-stack-roots set.
+pub const DRC_HEADER_MARK_BIT: u32 = 1 << 0;
+
+/// The bit within a `VMDrcHeader`'s reserved bits that is the
+/// in-the-over-approximated-stack-roots list bit.
+pub const DRC_HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1;
+
+/// The minimum length the over-approximated-stack-roots list must reach
+/// before a read barrier considers forcing a GC.
+pub const DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024;
+
/// Assert a condition, but only when `gc_zeal` is enabled.
#[macro_export]
macro_rules! gc_assert {
diff --git a/crates/environ/src/gc/drc.rs b/crates/environ/src/gc/drc.rs
index 9f55750630..a24a21f668 100644
--- a/crates/environ/src/gc/drc.rs
+++ b/crates/environ/src/gc/drc.rs
@@ -17,14 +17,9 @@ pub const EXCEPTION_TAG_INSTANCE_OFFSET: u32 = HEADER_SIZE;
/// The offset of the tag-defined-index field in an exception header.
pub const EXCEPTION_TAG_DEFINED_OFFSET: u32 = HEADER_SIZE + 4;
-/// The bit within a `VMDrcHeader`'s reserved bits that is the mark
-/// bit. Collectively, this bit in all the heap's objects' headers implements
-/// the precise-stack-roots set.
-pub const HEADER_MARK_BIT: u32 = 1 << 0;
-
-/// The bit within a `VMDrcHeader`'s reserved bits that is the
-/// in-the-over-approximated-stack-roots list bit.
-pub const HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1;
+pub use super::DRC_HEADER_IN_OVER_APPROX_LIST_BIT as HEADER_IN_OVER_APPROX_LIST_BIT;
+pub use super::DRC_HEADER_MARK_BIT as HEADER_MARK_BIT;
+pub use super::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD;
/// The layout of Wasm GC objects in the deferred reference-counting collector.
#[derive(Default)]
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index e8ce5360b5..576df405cd 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -558,16 +558,6 @@ impl WastTest {
return true;
}
- // Winch supports GC types only under the barrier-free collectors;
- // the deferred reference-counting collector is refused until Winch
- // emits GC barriers.
- if config.compiler == Compiler::Winch
- && config.collector == Collector::DeferredReferenceCounting
- && self.config.gc_types()
- {
- return true;
- }
-
// Disable spec tests per target for proposals that Winch does not implement yet.
if config.compiler == Compiler::Winch {
// Common list for tests that fail in all targets supported by Winch.
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index c8060d2f72..948c7cf890 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -2475,13 +2475,6 @@ impl Config {
| WasmFeatures::LEGACY_EXCEPTIONS
| WasmFeatures::STACK_SWITCHING;
- // Winch supports GC types only under the barrier-free
- // collectors; the deferred reference-counting collector
- // requires GC barriers that Winch does not emit yet.
- #[cfg(feature = "gc")]
- if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) {
- unsupported |= WasmFeatures::GC_TYPES;
- }
match self.compiler_target().architecture {
target_lexicon::Architecture::Aarch64(_) => {
unsupported |= WasmFeatures::THREADS;
diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md
index c9c8f59bf3..455b370595 100644
--- a/docs/stability-tiers.md
+++ b/docs/stability-tiers.md
@@ -340,10 +340,11 @@ emitting Pulley bytecode.
| [`exception-handling`] | ✅ | ❌ |
| [`stack-switching`] | ❌ | ❌ |
-[^a]: Winch supports some features of the [`reference-types`] proposal such as
- the change to support multiple tables and LEB-encoding table indices in
- instructions, but it does not support GC types such as `externref` or the
- new table opcodes in the [`reference-types`] proposal.
+[^a]: Winch supports GC reference values such as `externref` through parameters,
+ results, locals, globals, and calls, including the required stack maps and
+ collector barriers. It also supports multiple tables and LEB-encoded table
+ indices, but does not yet support every table and element-segment case in the
+ [`reference-types`] proposal.
[^b]: Pulley does not support the [`threads`] proposal because there is no known
safe way to implement this with Rust's memory model.
[^c]: Winch's support for aarch64 is complete for Core Wasm.
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index ff32f09fc4..c432c48fbf 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -3839,6 +3839,217 @@ fn winch_externref_survives_gc_in_frame() -> Result<()> {
Ok(())
}
+/// The write barrier's decrement chain releases an object once a global stops
+/// holding the last reference to it.
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_drc_write_barrier_drops_old_global_value() -> Result<()> {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(Collector::DeferredReferenceCounting);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (global $g (mut externref) (ref.null extern))
+ (func (export "set") (param externref)
+ (global.set $g (local.get 0))))
+ "#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let set = instance.get_func(&mut store, "set").unwrap();
+
+ let dropped = Arc::new(AtomicBool::new(false));
+ {
+ let mut scope = RootScope::new(&mut store);
+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?;
+ set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?;
+ }
+
+ // The global holds the only reference; nothing may be dropped yet.
+ store.gc(None)?;
+ assert!(!dropped.load(SeqCst));
+
+ // Overwriting the global decrements the count to zero and releases the
+ // old value.
+ set.call(&mut store, &[Val::ExternRef(None)], &mut [])?;
+ store.gc(None)?;
+ assert!(dropped.load(SeqCst));
+
+ Ok(())
+}
+
+/// The read barrier holds a count for references entering the stack, so
+/// overwriting their last long-lived home cannot free them out from under
+/// the frame that loaded them.
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_drc_read_barrier_keeps_loaded_ref_alive() -> Result<()> {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(Collector::DeferredReferenceCounting);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (import "" "gc" (func $gc))
+ (global $g (mut externref) (ref.null extern))
+ (func (export "set") (param externref)
+ (global.set $g (local.get 0)))
+ (func (export "swap") (result externref)
+ (local $tmp externref)
+ (local.set $tmp (global.get $g))
+ (global.set $g (ref.null extern))
+ (call $gc)
+ (local.get $tmp)))
+ "#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {
+ let _ = cx.gc(None);
+ });
+ let instance = Instance::new(&mut store, &module, &[gc.into()])?;
+ let set = instance.get_func(&mut store, "set").unwrap();
+ let swap = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "swap")?;
+
+ {
+ let mut scope = RootScope::new(&mut store);
+ let r = ExternRef::new(&mut scope, 0xDECAFu32)?;
+ set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?;
+ }
+
+ // Settle the deferred unroot so the global truly holds the last count.
+ store.gc(None)?;
+
+ // `swap` loads the reference onto the stack, overwrites the global, and
+ // collects while the stack copy is live.
+ let out = swap.call(&mut store, ())?.expect("must not be null");
+ let got = out
+ .data(&store)?
+ .and_then(|d| d.downcast_ref::<u32>().copied());
+ assert_eq!(got, Some(0xDECAF));
+
+ Ok(())
+}
+
+/// An `externref` may wrap an unboxed i31 created through the host API. Such
+/// values do not have reference counts and must bypass both DRC barriers.
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_drc_i31_wrapped_as_externref_skips_global_barriers() -> Result<()> {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(Collector::DeferredReferenceCounting);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (global $g (mut externref) (ref.null extern))
+ (func (export "set") (param externref)
+ local.get 0
+ global.set $g)
+ (func (export "get") (result externref)
+ global.get $g))
+ "#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let set = instance.get_func(&mut store, "set").unwrap();
+ let get = instance.get_func(&mut store, "get").unwrap();
+
+ let anyref = AnyRef::from_i31(&mut store, I31::wrapping_u32(0x1234));
+ let externref = ExternRef::convert_any(&mut store, anyref)?;
+ set.call(&mut store, &[Val::ExternRef(Some(externref))], &mut [])?;
+
+ let mut results = [Val::null_extern_ref()];
+ get.call(&mut store, &[], &mut results)?;
+ let externref = results[0]
+ .unwrap_externref()
+ .expect("global.get returned null");
+ let anyref = AnyRef::convert_extern(&mut store, *externref)?;
+ assert_eq!(anyref.unwrap_i31(&store)?.get_u32(), 0x1234);
+
+ // Replacing the i31-backed externref exercises the old-value side of the
+ // write barrier as well.
+ set.call(&mut store, &[Val::null_extern_ref()], &mut [])?;
+ Ok(())
+}
+
+/// Growing the over-approximated-stack-roots list to its threshold forces a
+/// collection from the read barrier and preserves the reference whose load
+/// triggered that collection.
+#[test]
+#[cfg_attr(miri, ignore)]
+fn winch_drc_read_barrier_forces_gc_at_threshold() -> Result<()> {
+ let mut config = Config::new();
+ config.strategy(Strategy::Winch);
+ config.collector(Collector::DeferredReferenceCounting);
+ let Ok(engine) = Engine::new(&config) else {
+ return Ok(());
+ };
+
+ let num_refs = wasmtime_environ::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as usize;
+ let mut wat = "(module\n".to_string();
+ for i in 0..num_refs {
+ wat.push_str(&format!(
+ r#"(global $g{i} (export "g{i}") (mut externref) (ref.null extern))
+"#,
+ ));
+ }
+ wat.push_str("(func (export \"drain\")\n");
+ for i in 0..num_refs {
+ wat.push_str(&format!(
+ "(drop (global.get $g{i}))\n(global.set $g{i} (ref.null extern))\n"
+ ));
+ }
+ wat.push_str("))");
+
+ let module = Module::new(&engine, &wat)?;
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let dropped = Arc::new(AtomicUsize::new(0));
+
+ {
+ let mut scope = RootScope::new(&mut store);
+ for i in 0..num_refs {
+ let gc_ref = ExternRef::new(&mut scope, CountDrops(dropped.clone()))?;
+ let global = instance
+ .get_global(&mut scope, &format!("g{i}"))
+ .expect("global must be exported");
+ global.set(&mut scope, Val::ExternRef(Some(gc_ref)))?;
+ }
+ }
+
+ // Settle the host roots. Each global is now the only long-lived home for
+ // its reference.
+ store.gc(None)?;
+ assert_eq!(dropped.load(SeqCst), 0);
+
+ let drain = instance.get_typed_func::<(), ()>(&mut store, "drain")?;
+ drain.call(&mut store, ())?;
+
+ // The final global.get reaches the threshold and forces a collection.
+ // The preceding references are no longer on the stack or in globals, but
+ // the triggering reference is still live in the global.get result slot.
+ assert_eq!(dropped.load(SeqCst), num_refs - 1);
+
+ // The triggering reference becomes collectible after `drain` returns.
+ store.gc(None)?;
+ assert_eq!(dropped.load(SeqCst), num_refs);
+
+ Ok(())
+}
+
/// Reference values crossing the ABI boundary in every position: stack-passed
/// externref params and multi-value externref results (more than fit in registers)
#[test]
diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rs
index 966588eb07..fe252842a1 100644
--- a/tests/all/winch_engine_features.rs
+++ b/tests/all/winch_engine_features.rs
@@ -64,41 +64,3 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> R
Ok(())
}
-
-#[wasmtime_test(strategies(only(Winch)))]
-#[cfg_attr(miri, ignore)]
-fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> {
- config.collector(Collector::DeferredReferenceCounting);
- config.gc_support(true);
- let result = Engine::new(&config);
- match result {
- Ok(_) => {
- wasmtime::bail!(
- "Expected incompatibility between the deferred reference-counting \
- collector and Winch"
- )
- }
- Err(e) => {
- assert_eq!(
- e.to_string(),
- "the wasm_gc_types feature is not supported on this compiler configuration"
- );
- }
- }
-
- Ok(())
-}
-
-#[wasmtime_test(strategies(only(Winch)))]
-#[cfg_attr(miri, ignore)]
-fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> {
- config.collector(Collector::DeferredReferenceCounting);
- let engine = Engine::new(&config)?;
- let result = Module::new(
- &engine,
- r#"(module (global (mut externref) (ref.null extern)))"#,
- );
- assert!(result.is_err());
-
- Ok(())
-}
diff --git a/tests/disas/winch/aarch64/global-get-externref-drc.wat b/tests/disas/winch/aarch64/global-get-externref-drc.wat
new file mode 100644
index 0000000000..9ec469e213
--- /dev/null
+++ b/tests/disas/winch/aarch64/global-get-externref-drc.wat
@@ -0,0 +1,94 @@
+;;! target = "aarch64"
+;;! test = "winch"
+;;! flags = "-Ccollector=drc"
+
+(module
+ (global $g (mut externref) (ref.null extern))
+ (func (export "get") (result externref)
+ (global.get $g)))
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x148
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x10
+;; mov sp, x28
+;; stur x0, [x28, #8]
+;; stur x1, [x28]
+;; ldur w0, [x9, #0x30]
+;; sub x28, x28, #4
+;; mov sp, x28
+;; stur w0, [x28]
+;; ldur w0, [x28]
+;; tst w0, w0
+;; b.eq #0x124
+;; b #0x60
+;; 60: mov w16, w0
+;; and w16, w16, #1
+;; tst w16, w16
+;; b.ne #0x124
+;; b #0x74
+;; 74: ldur x1, [x9, #8]
+;; ldur x2, [x1, #0x28]
+;; ldur x1, [x1, #0x20]
+;; mov x16, x0
+;; add x16, x16, #0x14
+;; cmp x16, x2, uxtx
+;; sub sp, x28, #0xc
+;; b.hi #0x14c
+;; 94: mov sp, x28
+;; mov x2, x1
+;; add x2, x2, x0, uxtx
+;; ldur w16, [x2]
+;; and w16, w16, #2
+;; tst w16, w16
+;; b.ne #0x124
+;; b #0xb4
+;; b4: ldur x3, [x9, #0x20]
+;; ldur w16, [x3]
+;; stur w16, [x2, #0x10]
+;; ldur w16, [x2]
+;; orr w16, w16, #2
+;; stur w16, [x2]
+;; ldur x4, [x2, #8]
+;; add x4, x4, #1
+;; stur x4, [x2, #8]
+;; stur w0, [x3]
+;; ldur w4, [x3, #4]
+;; add w4, w4, #1
+;; stur w4, [x3, #4]
+;; ldur w16, [x3, #8]
+;; add w16, w16, w16, uxtx
+;; cmp w4, w16, uxtx
+;; b.lo #0x124
+;; b #0xfc
+;; fc: cmp w4, #0x400
+;; b.lo #0x124
+;; b #0x108
+;; 108: sub x28, x28, #0xc
+;; mov sp, x28
+;; mov x0, x9
+;; bl #0x29c
+;; 118: add x28, x28, #0xc
+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
+;; mov sp, x28
+;; ldur x9, [x28, #0xc]
+;; ldur w0, [x28]
+;; add x28, x28, #4
+;; mov sp, x28
+;; add x28, x28, #0x10
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 148: udf #0xc11f
+;; 14c: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/global-set-externref-drc.wat b/tests/disas/winch/aarch64/global-set-externref-drc.wat
new file mode 100644
index 0000000000..6e5875a579
--- /dev/null
+++ b/tests/disas/winch/aarch64/global-set-externref-drc.wat
@@ -0,0 +1,103 @@
+;;! target = "aarch64"
+;;! test = "winch"
+;;! flags = "-Ccollector=drc"
+
+(module
+ (global $g (mut externref) (ref.null extern))
+ (func (param externref)
+ (global.set $g (local.get 0))))
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x168
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x0, [x28, #0x10]
+;; stur x1, [x28, #8]
+;; stur w2, [x28, #4]
+;; ldur w16, [x28, #4]
+;; sub x28, x28, #4
+;; mov sp, x28
+;; stur w16, [x28]
+;; ldur w0, [x28]
+;; add x28, x28, #4
+;; mov sp, x28
+;; ldur x1, [x9, #8]
+;; ldur x2, [x1, #0x28]
+;; ldur x1, [x1, #0x20]
+;; ldur w3, [x9, #0x30]
+;; tst w0, w0
+;; b.eq #0xbc
+;; b #0x7c
+;; 7c: mov w16, w0
+;; and w16, w16, #1
+;; tst w16, w16
+;; b.ne #0xbc
+;; b #0x90
+;; 90: mov x16, x0
+;; add x16, x16, #0x10
+;; cmp x16, x2, uxtx
+;; sub sp, x28, #8
+;; b.hi #0x16c
+;; a4: mov sp, x28
+;; mov x4, x1
+;; add x4, x4, x0, uxtx
+;; ldur x5, [x4, #8]
+;; add x5, x5, #1
+;; stur x5, [x4, #8]
+;; stur w0, [x9, #0x30]
+;; tst w3, w3
+;; b.eq #0x150
+;; b #0xcc
+;; cc: mov w16, w3
+;; and w16, w16, #1
+;; tst w16, w16
+;; b.ne #0x150
+;; b #0xe0
+;; e0: mov x16, x3
+;; add x16, x16, #0x10
+;; cmp x16, x2, uxtx
+;; sub sp, x28, #8
+;; b.hi #0x170
+;; f4: mov sp, x28
+;; mov x4, x1
+;; add x4, x4, x3, uxtx
+;; ldur x5, [x4, #8]
+;; sub x5, x5, #1
+;; cmp x5, #0
+;; b.eq #0x11c
+;; b #0x114
+;; 114: stur x5, [x4, #8]
+;; b #0x150
+;; 11c: sub x28, x28, #4
+;; mov sp, x28
+;; stur w3, [x28]
+;; sub x28, x28, #4
+;; mov sp, x28
+;; mov x0, x9
+;; ldur w1, [x28, #4]
+;; bl #0x1ec
+;; 13c: add x28, x28, #4
+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
+;; mov sp, x28
+;; add x28, x28, #4
+;; mov sp, x28
+;; ldur x9, [x28, #0x10]
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 168: udf #0xc11f
+;; 16c: udf #0xc11f
+;; 170: udf #0xc11f
diff --git a/tests/disas/winch/x64/global-get-externref-drc.wat b/tests/disas/winch/x64/global-get-externref-drc.wat
new file mode 100644
index 0000000000..4d04893636
--- /dev/null
+++ b/tests/disas/winch/x64/global-get-externref-drc.wat
@@ -0,0 +1,75 @@
+;;! target = "x86_64"
+;;! test = "winch"
+;;! flags = "-Ccollector=drc"
+
+(module
+ (global $g (mut externref) (ref.null extern))
+ (func (export "get") (result externref)
+ (global.get $g)))
+;; wasm[0]::function[0]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x20, %r11
+;; cmpq %rsp, %r11
+;; ja 0x112
+;; 1c: movq %rdi, %r14
+;; subq $0x10, %rsp
+;; movq %rdi, 8(%rsp)
+;; movq %rsi, (%rsp)
+;; movl 0x30(%r14), %eax
+;; subq $4, %rsp
+;; movl %eax, (%rsp)
+;; movl (%rsp), %eax
+;; testl %eax, %eax
+;; je 0xff
+;; 48: movl %eax, %r11d
+;; andl $1, %r11d
+;; testl %r11d, %r11d
+;; jne 0xff
+;; 5b: movq 8(%r14), %rcx
+;; movq 0x28(%rcx), %rdx
+;; movq 0x20(%rcx), %rcx
+;; movq %rax, %r11
+;; addq $0x14, %r11
+;; cmpq %rdx, %r11
+;; ja 0x114
+;; 7a: movq %rcx, %rdx
+;; addq %rax, %rdx
+;; movl (%rdx), %r11d
+;; andl $2, %r11d
+;; testl %r11d, %r11d
+;; jne 0xff
+;; 93: movq 0x20(%r14), %rbx
+;; movl (%rbx), %r11d
+;; movl %r11d, 0x10(%rdx)
+;; movl (%rdx), %r11d
+;; orl $2, %r11d
+;; movl %r11d, (%rdx)
+;; movq 8(%rdx), %rsi
+;; addq $1, %rsi
+;; movq %rsi, 8(%rdx)
+;; movl %eax, (%rbx)
+;; movl 4(%rbx), %esi
+;; addl $1, %esi
+;; movl %esi, 4(%rbx)
+;; movl 8(%rbx), %r11d
+;; addl %r11d, %r11d
+;; cmpl %r11d, %esi
+;; jb 0xff
+;; d8: cmpl $0x400, %esi
+;; jb 0xff
+;; e4: subq $0xc, %rsp
+;; movq %r14, %rdi
+;; callq 0x21a
+;; addq $0xc, %rsp
+;; ╰─╼ stack_map: frame_size=32, frame_offsets=[12]
+;; movq 0xc(%rsp), %r14
+;; movl (%rsp), %eax
+;; addq $4, %rsp
+;; addq $0x10, %rsp
+;; popq %rbp
+;; retq
+;; 112: ud2
+;; 114: ud2
diff --git a/tests/disas/winch/x64/global-set-externref-drc.wat b/tests/disas/winch/x64/global-set-externref-drc.wat
new file mode 100644
index 0000000000..d272b62c6c
--- /dev/null
+++ b/tests/disas/winch/x64/global-set-externref-drc.wat
@@ -0,0 +1,80 @@
+;;! target = "x86_64"
+;;! test = "winch"
+;;! flags = "-Ccollector=drc"
+
+(module
+ (global $g (mut externref) (ref.null extern))
+ (func (param externref)
+ (global.set $g (local.get 0))))
+;; wasm[0]::function[0]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x30, %r11
+;; cmpq %rsp, %r11
+;; ja 0x130
+;; 1c: movq %rdi, %r14
+;; subq $0x20, %rsp
+;; movq %rdi, 0x18(%rsp)
+;; movq %rsi, 0x10(%rsp)
+;; movl %edx, 0xc(%rsp)
+;; movl 0xc(%rsp), %r11d
+;; subq $4, %rsp
+;; movl %r11d, (%rsp)
+;; movl (%rsp), %eax
+;; addq $4, %rsp
+;; movq 8(%r14), %rcx
+;; movq 0x28(%rcx), %rdx
+;; movq 0x20(%rcx), %rcx
+;; movl 0x30(%r14), %ebx
+;; testl %eax, %eax
+;; je 0xa1
+;; 66: movl %eax, %r11d
+;; andl $1, %r11d
+;; testl %r11d, %r11d
+;; jne 0xa1
+;; 79: movq %rax, %r11
+;; addq $0x10, %r11
+;; cmpq %rdx, %r11
+;; ja 0x132
+;; 8c: movq %rcx, %rsi
+;; addq %rax, %rsi
+;; movq 8(%rsi), %rdi
+;; addq $1, %rdi
+;; movq %rdi, 8(%rsi)
+;; movl %eax, 0x30(%r14)
+;; testl %ebx, %ebx
+;; je 0x127
+;; ad: movl %ebx, %r11d
+;; andl $1, %r11d
+;; testl %r11d, %r11d
+;; jne 0x127
+;; c0: movq %rbx, %r11
+;; addq $0x10, %r11
+;; cmpq %rdx, %r11
+;; ja 0x134
+;; d3: movq %rcx, %rsi
+;; addq %rbx, %rsi
+;; movq 8(%rsi), %rdi
+;; subq $1, %rdi
+;; cmpq $0, %rdi
+;; je 0xf7
+;; ee: movq %rdi, 8(%rsi)
+;; jmp 0x127
+;; f7: subq $4, %rsp
+;; movl %ebx, (%rsp)
+;; subq $0xc, %rsp
+;; movq %r14, %rdi
+;; movl 0xc(%rsp), %esi
+;; callq 0x193
+;; addq $0xc, %rsp
+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]
+;; addq $4, %rsp
+;; movq 0x18(%rsp), %r14
+;; addq $0x20, %rsp
+;; popq %rbp
+;; retq
+;; 130: ud2
+;; 132: ud2
+;; 134: ud2
diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wast
index 4f242b0223..d9ba79a13a 100644
--- a/tests/misc_testsuite/winch/gc-refs.wast
+++ b/tests/misc_testsuite/winch/gc-refs.wast
@@ -2,8 +2,8 @@
;;! gc_types = true
;; Winch reference-value support: externref flows through params, results,
-;; locals, and globals, and survives calls (stack maps). Runs under the
-;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers.
+;; locals, and globals, and survives calls (stack maps). Global reads and
+;; writes use barriers when running under the DRC collector.
(module
(global $g (mut externref) (ref.null extern))
diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rs
new file mode 100644
index 0000000000..d3eb52d321
--- /dev/null
+++ b/winch/codegen/src/codegen/drc.rs
@@ -0,0 +1,420 @@
+use super::{Callee, CodeGen, CodeGenError, Emission, FnCall};
+use crate::{
+ Result,
+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm},
+ reg::{Reg, writable},
+ stack::{TypedReg, Val},
+};
+use cranelift_codegen::MachLabel;
+use wasmtime_environ::{
+ DRC_HEADER_IN_OVER_APPROX_LIST_BIT, DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD, PtrSize,
+ WasmValType,
+};
+
+#[derive(Clone, Copy)]
+enum RefCountMutation {
+ Increment,
+ Decrement,
+}
+
+impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>
+where
+ M: MacroAssembler,
+{
+ /// Emits a DRC read barrier for a value loaded from `addr`.
+ ///
+ /// The loaded reference is first pushed and spilled so it is represented
+ /// in a stack map if this barrier calls `force_gc`. Null and i31 references
+ /// do not point into the GC heap, so they skip the heap access and barrier
+ /// bookkeeping. For a heap reference, the barrier bounds-checks the header
+ /// access and, unless the object is already present, links it into the
+ /// over-approximated stack-roots list, marks it as linked, and retains it.
+ /// Finally it forces a collection when the roots list reaches both the
+ /// proportional and absolute thresholds.
+ ///
+ /// Leaves the loaded reference on the value stack and marks
+ /// `storage_base` available for register reuse before returning.
+ pub(crate) fn emit_drc_read_barrier(
+ &mut self,
+ ty: WasmValType,
+ storage_base: Reg,
+ addr: M::Address,
+ ) -> Result<()> {
+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;
+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;
+ self.context.stack.push(Val::reg(gc_ref, ty));
+
+ // Spill the loaded result into a stack-map-visible slot before the
+ // possible `force_gc` call, then reload a temporary copy for the
+ // barrier calculations.
+ self.context.spill(self.masm)?;
+ let slot = self
+ .context
+ .stack
+ .peek()
+ .ok_or_else(|| CodeGenError::missing_values_in_stack())?
+ .unwrap_mem()
+ .slot;
+ let ref_reg = self.context.any_gpr(self.masm)?;
+ self.masm.load(
+ self.masm.address_from_sp(slot.offset)?,
+ writable!(ref_reg),
+ OperandSize::S32,
+ )?;
+
+ let skip_barrier = self.masm.get_label()?;
+ self.emit_skip_if_gc_ref_is_null_or_i31(ref_reg, skip_barrier)?;
+
+ let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?;
+
+ // The read barrier accesses through the header's `next` field.
+ let header_extent = i64::from(
+ self.env
+ .vmoffsets
+ .vm_drc_header_next_over_approximated_stack_root(),
+ ) + 4;
+ self.emit_gc_ref_bounds_check(ref_reg, bound_reg, header_extent)?;
+
+ // The bound is dead after the check. Free it before requesting the
+ // object address so the allocator can hand the same register back and
+ // keep register pressure low.
+ self.context.free_reg(bound_reg);
+ let object_addr = self.emit_gc_ref_addr(ref_reg, heap_reg)?;
+
+ self.emit_skip_if_in_over_approx_stack_roots(object_addr, skip_barrier)?;
+
+ let (heap_data_reg, roots_len) =
+ self.emit_push_over_approx_stack_root(ref_reg, object_addr)?;
+
+ // Only the roots-list length and the heap data pointer are still live;
+ // both are consumed and freed by `emit_maybe_force_gc`.
+ self.context.free_reg(ref_reg);
+ self.context.free_reg(heap_reg);
+ self.context.free_reg(object_addr);
+ self.emit_maybe_force_gc(roots_len, heap_data_reg)?;
+
+ self.masm.bind(skip_barrier)?;
+ self.context.free_reg(storage_base);
+ Ok(())
+ }
+
+ /// Emits a DRC write barrier for replacing the reference at `addr`.
+ ///
+ /// The barrier increments the new reference's count before decrementing the
+ /// old reference's count. It stores the new reference before a possible
+ /// out-of-line `drop_gc_ref` call. Null and i31 references skip their
+ /// respective retain and release operations.
+ ///
+ /// Consumes the new reference from the value stack and marks
+ /// `storage_base` available for register reuse before a possible runtime
+ /// call.
+ pub(crate) fn emit_drc_write_barrier(
+ &mut self,
+ ty: WasmValType,
+ storage_base: Reg,
+ addr: M::Address,
+ ) -> Result<()> {
+ self.context.spill(self.masm)?;
+ let new_ref = self.context.pop_to_reg(self.masm, None)?;
+ let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?;
+
+ let old_reg = self.context.any_gpr(self.masm)?;
+ self.masm.load(addr, writable!(old_reg), OperandSize::S32)?;
+
+ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();
+ let header_extent = i64::from(ref_count_offset) + 8;
+
+ // Retain the new heap reference before publishing it. This ordering
+ // keeps self-assignment from temporarily dropping the final owner.
+ let skip_inc = self.masm.get_label()?;
+ self.emit_skip_if_gc_ref_is_null_or_i31(new_ref.reg, skip_inc)?;
+ self.emit_gc_ref_bounds_check(new_ref.reg, bound_reg, header_extent)?;
+ let new_addr = self.emit_gc_ref_addr(new_ref.reg, heap_reg)?;
+ let count = self.emit_mutate_ref_count(new_addr, RefCountMutation::Increment)?;
+ self.emit_store_ref_count(new_addr, count)?;
+ self.context.free_reg(count);
+ self.context.free_reg(new_addr);
+
+ self.masm.bind(skip_inc)?;
+ // Publish the new value before releasing the old one because the
+ // zero-count path below can make an out-of-line runtime call.
+ self.masm.store(new_ref.reg.into(), addr, ty.try_into()?)?;
+
+ // Release the old heap reference, storing a nonzero count inline and
+ // delegating the zero-count case to `drop_gc_ref`.
+ let skip_dec = self.masm.get_label()?;
+ self.emit_skip_if_gc_ref_is_null_or_i31(old_reg, skip_dec)?;
+ self.emit_gc_ref_bounds_check(old_reg, bound_reg, header_extent)?;
+ let old_addr = self.emit_gc_ref_addr(old_reg, heap_reg)?;
+ let count = self.emit_mutate_ref_count(old_addr, RefCountMutation::Decrement)?;
+
+ let drop_old = self.masm.get_label()?;
+ self.masm.branch(
+ IntCmpKind::Eq,
+ count,
+ RegImm::i64(0),
+ drop_old,
+ OperandSize::S64,
+ )?;
+ self.emit_store_ref_count(old_addr, count)?;
+ self.masm.jmp(skip_dec)?;
+
+ self.context.free_reg(count);
+ self.context.free_reg(old_addr);
+ self.context.free_reg(bound_reg);
+ self.context.free_reg(heap_reg);
+ self.context.free_reg(new_ref.reg);
+ self.context.free_reg(storage_base);
+
+ self.masm.bind(drop_old)?;
+ let drop_gc_ref = self.env.builtins.drop_gc_ref::<M::ABI>()?;
+ self.context.stack.push(TypedReg::i32(old_reg).into());
+ FnCall::emit::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ Callee::Builtin(drop_gc_ref),
+ )?;
+
+ self.masm.bind(skip_dec)
+ }
+
+ /// Branches to `skip` when `object_addr` is already linked into the
+ /// over-approximated stack-roots list.
+ ///
+ /// Uses the dedicated scratch register, so it must not be called from
+ /// inside another `with_scratch` scope.
+ fn emit_skip_if_in_over_approx_stack_roots(
+ &mut self,
+ object_addr: Reg,
+ skip: MachLabel,
+ ) -> Result<()> {
+ let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits();
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load(
+ masm.address_at_reg(object_addr, reserved_offset)?,
+ scratch.writable(),
+ OperandSize::S32,
+ )?;
+ masm.and(
+ scratch.writable(),
+ scratch.inner(),
+ RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32),
+ OperandSize::S32,
+ )?;
+ masm.branch(
+ IntCmpKind::Ne,
+ scratch.inner(),
+ scratch.inner().into(),
+ skip,
+ OperandSize::S32,
+ )
+ })
+ }
+
+ /// Adds `gc_ref` to the DRC heap's over-approximated stack-roots list.
+ ///
+ /// The object is linked at the head, marked as linked, and retained for
+ /// the list's ownership.
+ ///
+ /// Returns the DRC heap data pointer and the updated roots-list length,
+ /// both allocated here and owned by the caller, which must eventually free
+ /// them. The length feeds the collection-threshold check.
+ fn emit_push_over_approx_stack_root(
+ &mut self,
+ gc_ref: Reg,
+ object_addr: Reg,
+ ) -> Result<(Reg, Reg)> {
+ let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data();
+ let roots_head_offset = u32::from(
+ self.env
+ .vmoffsets
+ .ptr
+ .vmdrc_heap_data_over_approximated_stack_roots(),
+ );
+ let roots_len_offset = u32::from(
+ self.env
+ .vmoffsets
+ .ptr
+ .vmdrc_heap_data_current_over_approximated_stack_roots_len(),
+ );
+ let next_offset = self
+ .env
+ .vmoffsets
+ .vm_drc_header_next_over_approximated_stack_root();
+ let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits();
+
+ let heap_data_reg = self.context.any_gpr(self.masm)?;
+ self.masm.load_ptr(
+ self.masm.address_at_vmctx(u32::from(heap_data_offset))?,
+ writable!(heap_data_reg),
+ )?;
+
+ // Link this object to the old head of the over-approximated list, then
+ // mark it as present so subsequent reads can take the fast path. Both
+ // steps only need a temporary, so they share the scratch register.
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load(
+ masm.address_at_reg(heap_data_reg, roots_head_offset)?,
+ scratch.writable(),
+ OperandSize::S32,
+ )?;
+ masm.store(
+ scratch.inner().into(),
+ masm.address_at_reg(object_addr, next_offset)?,
+ OperandSize::S32,
+ )?;
+
+ masm.load(
+ masm.address_at_reg(object_addr, reserved_offset)?,
+ scratch.writable(),
+ OperandSize::S32,
+ )?;
+ masm.or(
+ scratch.writable(),
+ scratch.inner(),
+ RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32),
+ OperandSize::S32,
+ )?;
+ masm.store(
+ scratch.inner().into(),
+ masm.address_at_reg(object_addr, reserved_offset)?,
+ OperandSize::S32,
+ )
+ })?;
+
+ // Retain the object for the list's ownership.
+ let count = self.emit_mutate_ref_count(object_addr, RefCountMutation::Increment)?;
+ self.emit_store_ref_count(object_addr, count)?;
+ self.context.free_reg(count);
+
+ // Publish the new head, then the updated length. The length outlives
+ // this helper, so it gets its own allocated register.
+ self.masm.store(
+ gc_ref.into(),
+ self.masm.address_at_reg(heap_data_reg, roots_head_offset)?,
+ OperandSize::S32,
+ )?;
+ let roots_len = self.context.any_gpr(self.masm)?;
+ self.masm.load(
+ self.masm.address_at_reg(heap_data_reg, roots_len_offset)?,
+ writable!(roots_len),
+ OperandSize::S32,
+ )?;
+ self.masm.add(
+ writable!(roots_len),
+ roots_len,
+ RegImm::i32(1),
+ OperandSize::S32,
+ )?;
+ self.masm.store(
+ roots_len.into(),
+ self.masm.address_at_reg(heap_data_reg, roots_len_offset)?,
+ OperandSize::S32,
+ )?;
+
+ Ok((heap_data_reg, roots_len))
+ }
+
+ /// Forces a collection when the over-approximated roots list has reached
+ /// both twice its post-GC length and the absolute minimum threshold.
+ ///
+ /// This method consumes and frees both register arguments before a
+ /// possible call to `force_gc`.
+ ///
+ /// The doubled post-GC length is only needed for the first comparison, so
+ /// it lives in the dedicated scratch register. This helper must therefore
+ /// not be called from inside another `with_scratch` scope.
+ fn emit_maybe_force_gc(&mut self, current_len: Reg, heap_data: Reg) -> Result<()> {
+ let last_len_offset = u32::from(
+ self.env
+ .vmoffsets
+ .ptr
+ .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(),
+ );
+ let skip_gc = self.masm.get_label()?;
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load(
+ masm.address_at_reg(heap_data, last_len_offset)?,
+ scratch.writable(),
+ OperandSize::S32,
+ )?;
+ masm.add(
+ scratch.writable(),
+ scratch.inner(),
+ scratch.inner().into(),
+ OperandSize::S32,
+ )?;
+ masm.branch(
+ IntCmpKind::LtU,
+ current_len,
+ scratch.inner().into(),
+ skip_gc,
+ OperandSize::S32,
+ )
+ })?;
+ let min_threshold = i32::try_from(DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD).unwrap();
+ self.masm.branch(
+ IntCmpKind::LtU,
+ current_len,
+ RegImm::i32(min_threshold),
+ skip_gc,
+ OperandSize::S32,
+ )?;
+
+ self.context.free_reg(heap_data);
+ self.context.free_reg(current_len);
+
+ let force_gc = self.env.builtins.force_gc::<M::ABI>()?;
+ FnCall::emit::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ Callee::Builtin(force_gc),
+ )?;
+ self.context.pop_and_free(self.masm)?;
+
+ self.masm.bind(skip_gc)
+ }
+
+ /// Loads a reference count and applies `mutation`, returning the register
+ /// holding the updated value. The caller decides whether and when to store
+ /// it, and must eventually free the returned register.
+ ///
+ /// The count is allocated rather than taken from the caller because the
+ /// decrement path inspects it after this helper returns, to decide between
+ /// storing it and calling `drop_gc_ref`.
+ fn emit_mutate_ref_count(
+ &mut self,
+ object_addr: Reg,
+ mutation: RefCountMutation,
+ ) -> Result<Reg> {
+ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();
+ let count = self.context.any_gpr(self.masm)?;
+ self.masm.load(
+ self.masm.address_at_reg(object_addr, ref_count_offset)?,
+ writable!(count),
+ OperandSize::S64,
+ )?;
+ match mutation {
+ RefCountMutation::Increment => {
+ self.masm
+ .add(writable!(count), count, RegImm::i64(1), OperandSize::S64)?
+ }
+ RefCountMutation::Decrement => {
+ self.masm
+ .sub(writable!(count), count, RegImm::i64(1), OperandSize::S64)?
+ }
+ }
+ Ok(count)
+ }
+
+ fn emit_store_ref_count(&mut self, object_addr: Reg, count: Reg) -> Result<()> {
+ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();
+ self.masm.store(
+ count.into(),
+ self.masm.address_at_reg(object_addr, ref_count_offset)?,
+ OperandSize::S64,
+ )
+ }
+}
diff --git a/winch/codegen/src/codegen/gc.rs b/winch/codegen/src/codegen/gc.rs
new file mode 100644
index 0000000000..a7f7ad86d5
--- /dev/null
+++ b/winch/codegen/src/codegen/gc.rs
@@ -0,0 +1,140 @@
+use super::{CodeGen, Emission};
+use crate::{
+ Result,
+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm},
+ reg::{Reg, writable},
+};
+use cranelift_codegen::MachLabel;
+use wasmtime_cranelift::TRAP_GC_HEAP_CORRUPT;
+use wasmtime_environ::{I31_DISCRIMINANT, PtrSize};
+
+impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>
+where
+ M: MacroAssembler,
+{
+ /// Branches to `skip` when `gc_ref` is null or an unboxed i31 reference.
+ ///
+ /// A `VMGcRef` is null when all its bits are zero. The first branch tests
+ /// this by comparing the register with itself. The second test masks off
+ /// every bit except the i31 discriminant and skips the heap access when that bit is set.
+ ///
+ /// The discriminant test uses the dedicated scratch register rather than an
+ /// allocated temporary: the scratch register is non-allocatable, so it can
+ /// never alias `gc_ref`, which masking in place would otherwise destroy on
+ /// the fall-through path. Note that this means the helper must not be
+ /// called from inside another `with_scratch` scope.
+ pub(super) fn emit_skip_if_gc_ref_is_null_or_i31(
+ &mut self,
+ gc_ref: Reg,
+ skip: MachLabel,
+ ) -> Result<()> {
+ self.masm.branch(
+ IntCmpKind::Eq,
+ gc_ref,
+ gc_ref.into(),
+ skip,
+ OperandSize::S32,
+ )?;
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S32)?;
+ masm.and(
+ scratch.writable(),
+ scratch.inner(),
+ RegImm::i32(I31_DISCRIMINANT as i32),
+ OperandSize::S32,
+ )?;
+ masm.branch(
+ IntCmpKind::Ne,
+ scratch.inner(),
+ scratch.inner().into(),
+ skip,
+ OperandSize::S32,
+ )
+ })
+ }
+
+ /// Loads the GC heap's base address and current length.
+ ///
+ /// Both returned registers are allocated from the code-generation context
+ /// and are owned by the caller. The caller must eventually free them.
+ pub(super) fn emit_load_gc_heap_base_and_bound(&mut self) -> Result<(Reg, Reg)> {
+ let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
+ let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base();
+ let gc_heap_len_offset = self
+ .env
+ .vmoffsets
+ .ptr
+ .vm_store_context()
+ .gc_heap_current_length();
+ let heap_base = self.context.any_gpr(self.masm)?;
+ let heap_bound = self.context.any_gpr(self.masm)?;
+
+ self.masm.load_ptr(
+ self.masm
+ .address_at_vmctx(u32::from(store_context_offset))?,
+ writable!(heap_base),
+ )?;
+ self.masm.load(
+ self.masm
+ .address_at_reg(heap_base, u32::from(gc_heap_len_offset))?,
+ writable!(heap_bound),
+ OperandSize::S64,
+ )?;
+ self.masm.load_ptr(
+ self.masm
+ .address_at_reg(heap_base, u32::from(gc_heap_base_offset))?,
+ writable!(heap_base),
+ )?;
+
+ Ok((heap_base, heap_bound))
+ }
+
+ /// Checks that accessing `access_extent` bytes starting at `gc_ref` stays
+ /// within the GC heap.
+ ///
+ /// Callers must perform this check before converting a non-null, non-i31
+ /// reference into a native address and accessing its object header.
+ ///
+ /// The end of the access is computed in the dedicated scratch register. It
+ /// is non-allocatable, so it can alias neither `gc_ref` — which would leave
+ /// the reference offset by `access_extent` — nor `heap_bound`, which would
+ /// turn the comparison into `cmp scratch, scratch` and silently disable the
+ /// check. Using it also guarantees nothing is emitted between `cmp` and
+ /// `trapif`. Note that this means the helper must not be called from inside
+ /// another `with_scratch` scope.
+ pub(super) fn emit_gc_ref_bounds_check(
+ &mut self,
+ gc_ref: Reg,
+ heap_bound: Reg,
+ access_extent: i64,
+ ) -> Result<()> {
+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S64)?;
+ masm.add(
+ scratch.writable(),
+ scratch.inner(),
+ RegImm::i64(access_extent),
+ OperandSize::S64,
+ )?;
+ masm.cmp(scratch.inner(), heap_bound.into(), OperandSize::S64)?;
+ masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT)
+ })
+ }
+
+ /// Converts a bounds-checked `VMGcRef` heap offset into a native address.
+ ///
+ /// The returned register is allocated from the code-generation context and
+ /// is owned by the caller, which must eventually free it. Allocating here
+ /// keeps the result from aliasing `gc_ref`, which the addition would
+ /// otherwise clobber. Callers that want the address to reuse a register
+ /// that is dead by this point should free that register first and let the
+ /// allocator hand it back.
+ pub(super) fn emit_gc_ref_addr(&mut self, gc_ref: Reg, heap_base: Reg) -> Result<Reg> {
+ let dst = self.context.any_gpr(self.masm)?;
+ self.masm
+ .mov(writable!(dst), heap_base.into(), OperandSize::S64)?;
+ self.masm
+ .add(writable!(dst), dst, gc_ref.into(), OperandSize::S64)?;
+ Ok(dst)
+ }
+}
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 0df13d76d9..1577c091ae 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -40,6 +40,8 @@ pub(crate) use control::*;
mod builtin;
pub use builtin::*;
pub(crate) mod bounds;
+mod drc;
+mod gc;
use bounds::{Bounds, ImmOffset, Index};
@@ -490,6 +492,14 @@ where
}
}
+ /// Whether a GC barrier must be emitted when writing or reading a
+ /// reference of the given type through a collector-visible location.
+ pub fn gc_barrier_needed(&self, ty: &WasmValType) -> bool {
+ ty.is_vmgcref_type_and_not_i31()
+ && self.tunables.collector
+ == Some(wasmtime_environ::Collector::DeferredReferenceCounting)
+ }
+
/// Emits a a series of instructions that will type check a function reference call.
pub fn emit_typecheck_funcref(
&mut self,
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 791f7354ee..90deb3d394 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -2066,11 +2066,15 @@ where
let index = GlobalIndex::from_u32(global_index);
let (ty, base, offset) = self.emit_get_global_addr(index)?;
let addr = self.masm.address_at_reg(base, offset)?;
- let dst = self.context.reg_for_type(ty, self.masm)?;
- self.masm.load(addr, writable!(dst), ty.try_into()?)?;
- self.context.stack.push(Val::reg(dst, ty));
+ if self.gc_barrier_needed(&ty) {
+ self.emit_drc_read_barrier(ty, base, addr)?;
+ } else {
+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;
+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;
+ self.context.stack.push(Val::reg(gc_ref, ty));
- self.context.free_reg(base);
+ self.context.free_reg(base);
+ }
Ok(())
}
@@ -2080,11 +2084,15 @@ where
let (ty, base, offset) = self.emit_get_global_addr(index)?;
let addr = self.masm.address_at_reg(base, offset)?;
- let typed_reg = self.context.pop_to_reg(self.masm, None)?;
- self.masm
- .store(typed_reg.reg.into(), addr, ty.try_into()?)?;
- self.context.free_reg(typed_reg.reg);
- self.context.free_reg(base);
+ if self.gc_barrier_needed(&ty) {
+ self.emit_drc_write_barrier(ty, base, addr)?;
+ } else {
+ let typed_reg = self.context.pop_to_reg(self.masm, None)?;
+ self.masm
+ .store(typed_reg.reg.into(), addr, ty.try_into()?)?;
+ self.context.free_reg(typed_reg.reg);
+ self.context.free_reg(base);
+ }
Ok(())
}