A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch: drc barriers (#14081)
Daniel Macoveif08de396d2bd
diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rsindex fe75676554..8a9de06f63 100644--- a/crates/cranelift/src/func_environ/gc/drc.rs+++ b/crates/cranelift/src/func_environ/gc/drc.rs@@ -15,10 +15,6 @@ use wasmtime_environ::{WasmValType, drc::DrcTypeLayouts,};-// The minimum over-approximated stack roots list size for which we will trigger-// a GC.-const MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024;-#[derive(Default)]pub struct DrcCompiler {layouts: DrcTypeLayouts,@@ -190,9 +186,10 @@ impl DrcCompiler {);let doubled_last_len = builder.ins().iadd(last_len, last_len);- let min_threshold = builder- .ins()- .iconst(ir::types::I32, MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD);+ let min_threshold = builder.ins().iconst(+ ir::types::I32,+ wasmtime_environ::drc::MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD,+ );let threshold = builder.ins().umax(doubled_last_len, min_threshold);let should_gc =diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rsindex 50c0b26cbe..150726f70c 100644--- a/crates/cranelift/src/lib.rs+++ b/crates/cranelift/src/lib.rs@@ -49,7 +49,7 @@ mod trap;use self::compiler::Compiler;const TRAP_INTERNAL_ASSERT: TrapCode = TrapCode::unwrap_user(1);-const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2);+pub const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2);const TRAP_OFFSET: u8 = 3;pub const TRAP_CANNOT_LEAVE_COMPONENT: TrapCode =TrapCode::unwrap_user(Trap::CannotLeaveComponent as u8 + TRAP_OFFSET);diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rsindex 5a5c9d64bf..a560e7f918 100644--- a/crates/environ/src/gc.rs+++ b/crates/environ/src/gc.rs@@ -29,6 +29,19 @@ use core::alloc::Layout;/// enabled.pub const POISON: u8 = 0b00001111;+/// The bit within a `VMDrcHeader`'s reserved bits that is the mark+/// bit. Collectively, this bit in all the heap's objects' headers implements+/// the precise-stack-roots set.+pub const DRC_HEADER_MARK_BIT: u32 = 1 << 0;++/// The bit within a `VMDrcHeader`'s reserved bits that is the+/// in-the-over-approximated-stack-roots list bit.+pub const DRC_HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1;++/// The minimum length the over-approximated-stack-roots list must reach+/// before a read barrier considers forcing a GC.+pub const DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024;+/// Assert a condition, but only when `gc_zeal` is enabled.#[macro_export]macro_rules! gc_assert {diff --git a/crates/environ/src/gc/drc.rs b/crates/environ/src/gc/drc.rsindex 9f55750630..a24a21f668 100644--- a/crates/environ/src/gc/drc.rs+++ b/crates/environ/src/gc/drc.rs@@ -17,14 +17,9 @@ pub const EXCEPTION_TAG_INSTANCE_OFFSET: u32 = HEADER_SIZE;/// The offset of the tag-defined-index field in an exception header.pub const EXCEPTION_TAG_DEFINED_OFFSET: u32 = HEADER_SIZE + 4;-/// The bit within a `VMDrcHeader`'s reserved bits that is the mark-/// bit. Collectively, this bit in all the heap's objects' headers implements-/// the precise-stack-roots set.-pub const HEADER_MARK_BIT: u32 = 1 << 0;--/// The bit within a `VMDrcHeader`'s reserved bits that is the-/// in-the-over-approximated-stack-roots list bit.-pub const HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1;+pub use super::DRC_HEADER_IN_OVER_APPROX_LIST_BIT as HEADER_IN_OVER_APPROX_LIST_BIT;+pub use super::DRC_HEADER_MARK_BIT as HEADER_MARK_BIT;+pub use super::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD;/// The layout of Wasm GC objects in the deferred reference-counting collector.#[derive(Default)]diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex e8ce5360b5..576df405cd 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -558,16 +558,6 @@ impl WastTest {return true;}- // Winch supports GC types only under the barrier-free collectors;- // the deferred reference-counting collector is refused until Winch- // emits GC barriers.- if config.compiler == Compiler::Winch- && config.collector == Collector::DeferredReferenceCounting- && self.config.gc_types()- {- return true;- }-// Disable spec tests per target for proposals that Winch does not implement yet.if config.compiler == Compiler::Winch {// Common list for tests that fail in all targets supported by Winch.diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex c8060d2f72..948c7cf890 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -2475,13 +2475,6 @@ impl Config {| WasmFeatures::LEGACY_EXCEPTIONS| WasmFeatures::STACK_SWITCHING;- // Winch supports GC types only under the barrier-free- // collectors; the deferred reference-counting collector- // requires GC barriers that Winch does not emit yet.- #[cfg(feature = "gc")]- if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) {- unsupported |= WasmFeatures::GC_TYPES;- }match self.compiler_target().architecture {target_lexicon::Architecture::Aarch64(_) => {unsupported |= WasmFeatures::THREADS;diff --git a/docs/stability-tiers.md b/docs/stability-tiers.mdindex c9c8f59bf3..455b370595 100644--- a/docs/stability-tiers.md+++ b/docs/stability-tiers.md@@ -340,10 +340,11 @@ emitting Pulley bytecode.| [`exception-handling`] | ✅ | ❌ || [`stack-switching`] | ❌ | ❌ |-[^a]: Winch supports some features of the [`reference-types`] proposal such as- the change to support multiple tables and LEB-encoding table indices in- instructions, but it does not support GC types such as `externref` or the- new table opcodes in the [`reference-types`] proposal.+[^a]: Winch supports GC reference values such as `externref` through parameters,+ results, locals, globals, and calls, including the required stack maps and+ collector barriers. It also supports multiple tables and LEB-encoded table+ indices, but does not yet support every table and element-segment case in the+ [`reference-types`] proposal.[^b]: Pulley does not support the [`threads`] proposal because there is no knownsafe way to implement this with Rust's memory model.[^c]: Winch's support for aarch64 is complete for Core Wasm.diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex ff32f09fc4..c432c48fbf 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -3839,6 +3839,217 @@ fn winch_externref_survives_gc_in_frame() -> Result<()> {Ok(())}+/// The write barrier's decrement chain releases an object once a global stops+/// holding the last reference to it.+#[test]+#[cfg_attr(miri, ignore)]+fn winch_drc_write_barrier_drops_old_global_value() -> Result<()> {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(Collector::DeferredReferenceCounting);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };+ let module = Module::new(+ &engine,+ r#"+ (module+ (global $g (mut externref) (ref.null extern))+ (func (export "set") (param externref)+ (global.set $g (local.get 0))))+ "#,+ )?;+ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;+ let set = instance.get_func(&mut store, "set").unwrap();++ let dropped = Arc::new(AtomicBool::new(false));+ {+ let mut scope = RootScope::new(&mut store);+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?;+ set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?;+ }++ // The global holds the only reference; nothing may be dropped yet.+ store.gc(None)?;+ assert!(!dropped.load(SeqCst));++ // Overwriting the global decrements the count to zero and releases the+ // old value.+ set.call(&mut store, &[Val::ExternRef(None)], &mut [])?;+ store.gc(None)?;+ assert!(dropped.load(SeqCst));++ Ok(())+}++/// The read barrier holds a count for references entering the stack, so+/// overwriting their last long-lived home cannot free them out from under+/// the frame that loaded them.+#[test]+#[cfg_attr(miri, ignore)]+fn winch_drc_read_barrier_keeps_loaded_ref_alive() -> Result<()> {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(Collector::DeferredReferenceCounting);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };+ let module = Module::new(+ &engine,+ r#"+ (module+ (import "" "gc" (func $gc))+ (global $g (mut externref) (ref.null extern))+ (func (export "set") (param externref)+ (global.set $g (local.get 0)))+ (func (export "swap") (result externref)+ (local $tmp externref)+ (local.set $tmp (global.get $g))+ (global.set $g (ref.null extern))+ (call $gc)+ (local.get $tmp)))+ "#,+ )?;+ let mut store = Store::new(&engine, ());+ let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| {+ let _ = cx.gc(None);+ });+ let instance = Instance::new(&mut store, &module, &[gc.into()])?;+ let set = instance.get_func(&mut store, "set").unwrap();+ let swap = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "swap")?;++ {+ let mut scope = RootScope::new(&mut store);+ let r = ExternRef::new(&mut scope, 0xDECAFu32)?;+ set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?;+ }++ // Settle the deferred unroot so the global truly holds the last count.+ store.gc(None)?;++ // `swap` loads the reference onto the stack, overwrites the global, and+ // collects while the stack copy is live.+ let out = swap.call(&mut store, ())?.expect("must not be null");+ let got = out+ .data(&store)?+ .and_then(|d| d.downcast_ref::<u32>().copied());+ assert_eq!(got, Some(0xDECAF));++ Ok(())+}++/// An `externref` may wrap an unboxed i31 created through the host API. Such+/// values do not have reference counts and must bypass both DRC barriers.+#[test]+#[cfg_attr(miri, ignore)]+fn winch_drc_i31_wrapped_as_externref_skips_global_barriers() -> Result<()> {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(Collector::DeferredReferenceCounting);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };+ let module = Module::new(+ &engine,+ r#"+ (module+ (global $g (mut externref) (ref.null extern))+ (func (export "set") (param externref)+ local.get 0+ global.set $g)+ (func (export "get") (result externref)+ global.get $g))+ "#,+ )?;+ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;+ let set = instance.get_func(&mut store, "set").unwrap();+ let get = instance.get_func(&mut store, "get").unwrap();++ let anyref = AnyRef::from_i31(&mut store, I31::wrapping_u32(0x1234));+ let externref = ExternRef::convert_any(&mut store, anyref)?;+ set.call(&mut store, &[Val::ExternRef(Some(externref))], &mut [])?;++ let mut results = [Val::null_extern_ref()];+ get.call(&mut store, &[], &mut results)?;+ let externref = results[0]+ .unwrap_externref()+ .expect("global.get returned null");+ let anyref = AnyRef::convert_extern(&mut store, *externref)?;+ assert_eq!(anyref.unwrap_i31(&store)?.get_u32(), 0x1234);++ // Replacing the i31-backed externref exercises the old-value side of the+ // write barrier as well.+ set.call(&mut store, &[Val::null_extern_ref()], &mut [])?;+ Ok(())+}++/// Growing the over-approximated-stack-roots list to its threshold forces a+/// collection from the read barrier and preserves the reference whose load+/// triggered that collection.+#[test]+#[cfg_attr(miri, ignore)]+fn winch_drc_read_barrier_forces_gc_at_threshold() -> Result<()> {+ let mut config = Config::new();+ config.strategy(Strategy::Winch);+ config.collector(Collector::DeferredReferenceCounting);+ let Ok(engine) = Engine::new(&config) else {+ return Ok(());+ };++ let num_refs = wasmtime_environ::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as usize;+ let mut wat = "(module\n".to_string();+ for i in 0..num_refs {+ wat.push_str(&format!(+ r#"(global $g{i} (export "g{i}") (mut externref) (ref.null extern))+"#,+ ));+ }+ wat.push_str("(func (export \"drain\")\n");+ for i in 0..num_refs {+ wat.push_str(&format!(+ "(drop (global.get $g{i}))\n(global.set $g{i} (ref.null extern))\n"+ ));+ }+ wat.push_str("))");++ let module = Module::new(&engine, &wat)?;+ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;+ let dropped = Arc::new(AtomicUsize::new(0));++ {+ let mut scope = RootScope::new(&mut store);+ for i in 0..num_refs {+ let gc_ref = ExternRef::new(&mut scope, CountDrops(dropped.clone()))?;+ let global = instance+ .get_global(&mut scope, &format!("g{i}"))+ .expect("global must be exported");+ global.set(&mut scope, Val::ExternRef(Some(gc_ref)))?;+ }+ }++ // Settle the host roots. Each global is now the only long-lived home for+ // its reference.+ store.gc(None)?;+ assert_eq!(dropped.load(SeqCst), 0);++ let drain = instance.get_typed_func::<(), ()>(&mut store, "drain")?;+ drain.call(&mut store, ())?;++ // The final global.get reaches the threshold and forces a collection.+ // The preceding references are no longer on the stack or in globals, but+ // the triggering reference is still live in the global.get result slot.+ assert_eq!(dropped.load(SeqCst), num_refs - 1);++ // The triggering reference becomes collectible after `drain` returns.+ store.gc(None)?;+ assert_eq!(dropped.load(SeqCst), num_refs);++ Ok(())+}+/// Reference values crossing the ABI boundary in every position: stack-passed/// externref params and multi-value externref results (more than fit in registers)#[test]diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rsindex 966588eb07..fe252842a1 100644--- a/tests/all/winch_engine_features.rs+++ b/tests/all/winch_engine_features.rs@@ -64,41 +64,3 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> ROk(())}--#[wasmtime_test(strategies(only(Winch)))]-#[cfg_attr(miri, ignore)]-fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> {- config.collector(Collector::DeferredReferenceCounting);- config.gc_support(true);- let result = Engine::new(&config);- match result {- Ok(_) => {- wasmtime::bail!(- "Expected incompatibility between the deferred reference-counting \- collector and Winch"- )- }- Err(e) => {- assert_eq!(- e.to_string(),- "the wasm_gc_types feature is not supported on this compiler configuration"- );- }- }-- Ok(())-}--#[wasmtime_test(strategies(only(Winch)))]-#[cfg_attr(miri, ignore)]-fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> {- config.collector(Collector::DeferredReferenceCounting);- let engine = Engine::new(&config)?;- let result = Module::new(- &engine,- r#"(module (global (mut externref) (ref.null extern)))"#,- );- assert!(result.is_err());-- Ok(())-}diff --git a/tests/disas/winch/aarch64/global-get-externref-drc.wat b/tests/disas/winch/aarch64/global-get-externref-drc.watnew file mode 100644index 0000000000..9ec469e213--- /dev/null+++ b/tests/disas/winch/aarch64/global-get-externref-drc.wat@@ -0,0 +1,94 @@+;;! target = "aarch64"+;;! test = "winch"+;;! flags = "-Ccollector=drc"++(module+ (global $g (mut externref) (ref.null extern))+ (func (export "get") (result externref)+ (global.get $g)))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x148+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldur w0, [x9, #0x30]+;; sub x28, x28, #4+;; mov sp, x28+;; stur w0, [x28]+;; ldur w0, [x28]+;; tst w0, w0+;; b.eq #0x124+;; b #0x60+;; 60: mov w16, w0+;; and w16, w16, #1+;; tst w16, w16+;; b.ne #0x124+;; b #0x74+;; 74: ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; mov x16, x0+;; add x16, x16, #0x14+;; cmp x16, x2, uxtx+;; sub sp, x28, #0xc+;; b.hi #0x14c+;; 94: mov sp, x28+;; mov x2, x1+;; add x2, x2, x0, uxtx+;; ldur w16, [x2]+;; and w16, w16, #2+;; tst w16, w16+;; b.ne #0x124+;; b #0xb4+;; b4: ldur x3, [x9, #0x20]+;; ldur w16, [x3]+;; stur w16, [x2, #0x10]+;; ldur w16, [x2]+;; orr w16, w16, #2+;; stur w16, [x2]+;; ldur x4, [x2, #8]+;; add x4, x4, #1+;; stur x4, [x2, #8]+;; stur w0, [x3]+;; ldur w4, [x3, #4]+;; add w4, w4, #1+;; stur w4, [x3, #4]+;; ldur w16, [x3, #8]+;; add w16, w16, w16, uxtx+;; cmp w4, w16, uxtx+;; b.lo #0x124+;; b #0xfc+;; fc: cmp w4, #0x400+;; b.lo #0x124+;; b #0x108+;; 108: sub x28, x28, #0xc+;; mov sp, x28+;; mov x0, x9+;; bl #0x29c+;; 118: add x28, x28, #0xc+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]+;; mov sp, x28+;; ldur x9, [x28, #0xc]+;; ldur w0, [x28]+;; add x28, x28, #4+;; mov sp, x28+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 148: udf #0xc11f+;; 14c: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/global-set-externref-drc.wat b/tests/disas/winch/aarch64/global-set-externref-drc.watnew file mode 100644index 0000000000..6e5875a579--- /dev/null+++ b/tests/disas/winch/aarch64/global-set-externref-drc.wat@@ -0,0 +1,103 @@+;;! target = "aarch64"+;;! test = "winch"+;;! flags = "-Ccollector=drc"++(module+ (global $g (mut externref) (ref.null extern))+ (func (param externref)+ (global.set $g (local.get 0))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x168+;; 2c: mov x9, x0+;; sub x28, x28, #0x18+;; mov sp, x28+;; stur x0, [x28, #0x10]+;; stur x1, [x28, #8]+;; stur w2, [x28, #4]+;; ldur w16, [x28, #4]+;; sub x28, x28, #4+;; mov sp, x28+;; stur w16, [x28]+;; ldur w0, [x28]+;; add x28, x28, #4+;; mov sp, x28+;; ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; ldur w3, [x9, #0x30]+;; tst w0, w0+;; b.eq #0xbc+;; b #0x7c+;; 7c: mov w16, w0+;; and w16, w16, #1+;; tst w16, w16+;; b.ne #0xbc+;; b #0x90+;; 90: mov x16, x0+;; add x16, x16, #0x10+;; cmp x16, x2, uxtx+;; sub sp, x28, #8+;; b.hi #0x16c+;; a4: mov sp, x28+;; mov x4, x1+;; add x4, x4, x0, uxtx+;; ldur x5, [x4, #8]+;; add x5, x5, #1+;; stur x5, [x4, #8]+;; stur w0, [x9, #0x30]+;; tst w3, w3+;; b.eq #0x150+;; b #0xcc+;; cc: mov w16, w3+;; and w16, w16, #1+;; tst w16, w16+;; b.ne #0x150+;; b #0xe0+;; e0: mov x16, x3+;; add x16, x16, #0x10+;; cmp x16, x2, uxtx+;; sub sp, x28, #8+;; b.hi #0x170+;; f4: mov sp, x28+;; mov x4, x1+;; add x4, x4, x3, uxtx+;; ldur x5, [x4, #8]+;; sub x5, x5, #1+;; cmp x5, #0+;; b.eq #0x11c+;; b #0x114+;; 114: stur x5, [x4, #8]+;; b #0x150+;; 11c: sub x28, x28, #4+;; mov sp, x28+;; stur w3, [x28]+;; sub x28, x28, #4+;; mov sp, x28+;; mov x0, x9+;; ldur w1, [x28, #4]+;; bl #0x1ec+;; 13c: add x28, x28, #4+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]+;; mov sp, x28+;; add x28, x28, #4+;; mov sp, x28+;; ldur x9, [x28, #0x10]+;; add x28, x28, #0x18+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 168: udf #0xc11f+;; 16c: udf #0xc11f+;; 170: udf #0xc11fdiff --git a/tests/disas/winch/x64/global-get-externref-drc.wat b/tests/disas/winch/x64/global-get-externref-drc.watnew file mode 100644index 0000000000..4d04893636--- /dev/null+++ b/tests/disas/winch/x64/global-get-externref-drc.wat@@ -0,0 +1,75 @@+;;! target = "x86_64"+;;! test = "winch"+;;! flags = "-Ccollector=drc"++(module+ (global $g (mut externref) (ref.null extern))+ (func (export "get") (result externref)+ (global.get $g)))+;; wasm[0]::function[0]:+;; pushq %rbp+;; movq %rsp, %rbp+;; movq 8(%rdi), %r11+;; movq 0x18(%r11), %r11+;; addq $0x20, %r11+;; cmpq %rsp, %r11+;; ja 0x112+;; 1c: movq %rdi, %r14+;; subq $0x10, %rsp+;; movq %rdi, 8(%rsp)+;; movq %rsi, (%rsp)+;; movl 0x30(%r14), %eax+;; subq $4, %rsp+;; movl %eax, (%rsp)+;; movl (%rsp), %eax+;; testl %eax, %eax+;; je 0xff+;; 48: movl %eax, %r11d+;; andl $1, %r11d+;; testl %r11d, %r11d+;; jne 0xff+;; 5b: movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movq %rax, %r11+;; addq $0x14, %r11+;; cmpq %rdx, %r11+;; ja 0x114+;; 7a: movq %rcx, %rdx+;; addq %rax, %rdx+;; movl (%rdx), %r11d+;; andl $2, %r11d+;; testl %r11d, %r11d+;; jne 0xff+;; 93: movq 0x20(%r14), %rbx+;; movl (%rbx), %r11d+;; movl %r11d, 0x10(%rdx)+;; movl (%rdx), %r11d+;; orl $2, %r11d+;; movl %r11d, (%rdx)+;; movq 8(%rdx), %rsi+;; addq $1, %rsi+;; movq %rsi, 8(%rdx)+;; movl %eax, (%rbx)+;; movl 4(%rbx), %esi+;; addl $1, %esi+;; movl %esi, 4(%rbx)+;; movl 8(%rbx), %r11d+;; addl %r11d, %r11d+;; cmpl %r11d, %esi+;; jb 0xff+;; d8: cmpl $0x400, %esi+;; jb 0xff+;; e4: subq $0xc, %rsp+;; movq %r14, %rdi+;; callq 0x21a+;; addq $0xc, %rsp+;; ╰─╼ stack_map: frame_size=32, frame_offsets=[12]+;; movq 0xc(%rsp), %r14+;; movl (%rsp), %eax+;; addq $4, %rsp+;; addq $0x10, %rsp+;; popq %rbp+;; retq+;; 112: ud2+;; 114: ud2diff --git a/tests/disas/winch/x64/global-set-externref-drc.wat b/tests/disas/winch/x64/global-set-externref-drc.watnew file mode 100644index 0000000000..d272b62c6c--- /dev/null+++ b/tests/disas/winch/x64/global-set-externref-drc.wat@@ -0,0 +1,80 @@+;;! target = "x86_64"+;;! test = "winch"+;;! flags = "-Ccollector=drc"++(module+ (global $g (mut externref) (ref.null extern))+ (func (param externref)+ (global.set $g (local.get 0))))+;; wasm[0]::function[0]:+;; pushq %rbp+;; movq %rsp, %rbp+;; movq 8(%rdi), %r11+;; movq 0x18(%r11), %r11+;; addq $0x30, %r11+;; cmpq %rsp, %r11+;; ja 0x130+;; 1c: movq %rdi, %r14+;; subq $0x20, %rsp+;; movq %rdi, 0x18(%rsp)+;; movq %rsi, 0x10(%rsp)+;; movl %edx, 0xc(%rsp)+;; movl 0xc(%rsp), %r11d+;; subq $4, %rsp+;; movl %r11d, (%rsp)+;; movl (%rsp), %eax+;; addq $4, %rsp+;; movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movl 0x30(%r14), %ebx+;; testl %eax, %eax+;; je 0xa1+;; 66: movl %eax, %r11d+;; andl $1, %r11d+;; testl %r11d, %r11d+;; jne 0xa1+;; 79: movq %rax, %r11+;; addq $0x10, %r11+;; cmpq %rdx, %r11+;; ja 0x132+;; 8c: movq %rcx, %rsi+;; addq %rax, %rsi+;; movq 8(%rsi), %rdi+;; addq $1, %rdi+;; movq %rdi, 8(%rsi)+;; movl %eax, 0x30(%r14)+;; testl %ebx, %ebx+;; je 0x127+;; ad: movl %ebx, %r11d+;; andl $1, %r11d+;; testl %r11d, %r11d+;; jne 0x127+;; c0: movq %rbx, %r11+;; addq $0x10, %r11+;; cmpq %rdx, %r11+;; ja 0x134+;; d3: movq %rcx, %rsi+;; addq %rbx, %rsi+;; movq 8(%rsi), %rdi+;; subq $1, %rdi+;; cmpq $0, %rdi+;; je 0xf7+;; ee: movq %rdi, 8(%rsi)+;; jmp 0x127+;; f7: subq $4, %rsp+;; movl %ebx, (%rsp)+;; subq $0xc, %rsp+;; movq %r14, %rdi+;; movl 0xc(%rsp), %esi+;; callq 0x193+;; addq $0xc, %rsp+;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]+;; addq $4, %rsp+;; movq 0x18(%rsp), %r14+;; addq $0x20, %rsp+;; popq %rbp+;; retq+;; 130: ud2+;; 132: ud2+;; 134: ud2diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wastindex 4f242b0223..d9ba79a13a 100644--- a/tests/misc_testsuite/winch/gc-refs.wast+++ b/tests/misc_testsuite/winch/gc-refs.wast@@ -2,8 +2,8 @@;;! gc_types = true;; Winch reference-value support: externref flows through params, results,-;; locals, and globals, and survives calls (stack maps). Runs under the-;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers.+;; locals, and globals, and survives calls (stack maps). Global reads and+;; writes use barriers when running under the DRC collector.(module(global $g (mut externref) (ref.null extern))diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rsnew file mode 100644index 0000000000..d3eb52d321--- /dev/null+++ b/winch/codegen/src/codegen/drc.rs@@ -0,0 +1,420 @@+use super::{Callee, CodeGen, CodeGenError, Emission, FnCall};+use crate::{+ Result,+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm},+ reg::{Reg, writable},+ stack::{TypedReg, Val},+};+use cranelift_codegen::MachLabel;+use wasmtime_environ::{+ DRC_HEADER_IN_OVER_APPROX_LIST_BIT, DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD, PtrSize,+ WasmValType,+};++#[derive(Clone, Copy)]+enum RefCountMutation {+ Increment,+ Decrement,+}++impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>+where+ M: MacroAssembler,+{+ /// Emits a DRC read barrier for a value loaded from `addr`.+ ///+ /// The loaded reference is first pushed and spilled so it is represented+ /// in a stack map if this barrier calls `force_gc`. Null and i31 references+ /// do not point into the GC heap, so they skip the heap access and barrier+ /// bookkeeping. For a heap reference, the barrier bounds-checks the header+ /// access and, unless the object is already present, links it into the+ /// over-approximated stack-roots list, marks it as linked, and retains it.+ /// Finally it forces a collection when the roots list reaches both the+ /// proportional and absolute thresholds.+ ///+ /// Leaves the loaded reference on the value stack and marks+ /// `storage_base` available for register reuse before returning.+ pub(crate) fn emit_drc_read_barrier(+ &mut self,+ ty: WasmValType,+ storage_base: Reg,+ addr: M::Address,+ ) -> Result<()> {+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;+ self.context.stack.push(Val::reg(gc_ref, ty));++ // Spill the loaded result into a stack-map-visible slot before the+ // possible `force_gc` call, then reload a temporary copy for the+ // barrier calculations.+ self.context.spill(self.masm)?;+ let slot = self+ .context+ .stack+ .peek()+ .ok_or_else(|| CodeGenError::missing_values_in_stack())?+ .unwrap_mem()+ .slot;+ let ref_reg = self.context.any_gpr(self.masm)?;+ self.masm.load(+ self.masm.address_from_sp(slot.offset)?,+ writable!(ref_reg),+ OperandSize::S32,+ )?;++ let skip_barrier = self.masm.get_label()?;+ self.emit_skip_if_gc_ref_is_null_or_i31(ref_reg, skip_barrier)?;++ let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?;++ // The read barrier accesses through the header's `next` field.+ let header_extent = i64::from(+ self.env+ .vmoffsets+ .vm_drc_header_next_over_approximated_stack_root(),+ ) + 4;+ self.emit_gc_ref_bounds_check(ref_reg, bound_reg, header_extent)?;++ // The bound is dead after the check. Free it before requesting the+ // object address so the allocator can hand the same register back and+ // keep register pressure low.+ self.context.free_reg(bound_reg);+ let object_addr = self.emit_gc_ref_addr(ref_reg, heap_reg)?;++ self.emit_skip_if_in_over_approx_stack_roots(object_addr, skip_barrier)?;++ let (heap_data_reg, roots_len) =+ self.emit_push_over_approx_stack_root(ref_reg, object_addr)?;++ // Only the roots-list length and the heap data pointer are still live;+ // both are consumed and freed by `emit_maybe_force_gc`.+ self.context.free_reg(ref_reg);+ self.context.free_reg(heap_reg);+ self.context.free_reg(object_addr);+ self.emit_maybe_force_gc(roots_len, heap_data_reg)?;++ self.masm.bind(skip_barrier)?;+ self.context.free_reg(storage_base);+ Ok(())+ }++ /// Emits a DRC write barrier for replacing the reference at `addr`.+ ///+ /// The barrier increments the new reference's count before decrementing the+ /// old reference's count. It stores the new reference before a possible+ /// out-of-line `drop_gc_ref` call. Null and i31 references skip their+ /// respective retain and release operations.+ ///+ /// Consumes the new reference from the value stack and marks+ /// `storage_base` available for register reuse before a possible runtime+ /// call.+ pub(crate) fn emit_drc_write_barrier(+ &mut self,+ ty: WasmValType,+ storage_base: Reg,+ addr: M::Address,+ ) -> Result<()> {+ self.context.spill(self.masm)?;+ let new_ref = self.context.pop_to_reg(self.masm, None)?;+ let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?;++ let old_reg = self.context.any_gpr(self.masm)?;+ self.masm.load(addr, writable!(old_reg), OperandSize::S32)?;++ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();+ let header_extent = i64::from(ref_count_offset) + 8;++ // Retain the new heap reference before publishing it. This ordering+ // keeps self-assignment from temporarily dropping the final owner.+ let skip_inc = self.masm.get_label()?;+ self.emit_skip_if_gc_ref_is_null_or_i31(new_ref.reg, skip_inc)?;+ self.emit_gc_ref_bounds_check(new_ref.reg, bound_reg, header_extent)?;+ let new_addr = self.emit_gc_ref_addr(new_ref.reg, heap_reg)?;+ let count = self.emit_mutate_ref_count(new_addr, RefCountMutation::Increment)?;+ self.emit_store_ref_count(new_addr, count)?;+ self.context.free_reg(count);+ self.context.free_reg(new_addr);++ self.masm.bind(skip_inc)?;+ // Publish the new value before releasing the old one because the+ // zero-count path below can make an out-of-line runtime call.+ self.masm.store(new_ref.reg.into(), addr, ty.try_into()?)?;++ // Release the old heap reference, storing a nonzero count inline and+ // delegating the zero-count case to `drop_gc_ref`.+ let skip_dec = self.masm.get_label()?;+ self.emit_skip_if_gc_ref_is_null_or_i31(old_reg, skip_dec)?;+ self.emit_gc_ref_bounds_check(old_reg, bound_reg, header_extent)?;+ let old_addr = self.emit_gc_ref_addr(old_reg, heap_reg)?;+ let count = self.emit_mutate_ref_count(old_addr, RefCountMutation::Decrement)?;++ let drop_old = self.masm.get_label()?;+ self.masm.branch(+ IntCmpKind::Eq,+ count,+ RegImm::i64(0),+ drop_old,+ OperandSize::S64,+ )?;+ self.emit_store_ref_count(old_addr, count)?;+ self.masm.jmp(skip_dec)?;++ self.context.free_reg(count);+ self.context.free_reg(old_addr);+ self.context.free_reg(bound_reg);+ self.context.free_reg(heap_reg);+ self.context.free_reg(new_ref.reg);+ self.context.free_reg(storage_base);++ self.masm.bind(drop_old)?;+ let drop_gc_ref = self.env.builtins.drop_gc_ref::<M::ABI>()?;+ self.context.stack.push(TypedReg::i32(old_reg).into());+ FnCall::emit::<M>(+ &mut self.env,+ self.masm,+ &mut self.context,+ Callee::Builtin(drop_gc_ref),+ )?;++ self.masm.bind(skip_dec)+ }++ /// Branches to `skip` when `object_addr` is already linked into the+ /// over-approximated stack-roots list.+ ///+ /// Uses the dedicated scratch register, so it must not be called from+ /// inside another `with_scratch` scope.+ fn emit_skip_if_in_over_approx_stack_roots(+ &mut self,+ object_addr: Reg,+ skip: MachLabel,+ ) -> Result<()> {+ let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits();+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.load(+ masm.address_at_reg(object_addr, reserved_offset)?,+ scratch.writable(),+ OperandSize::S32,+ )?;+ masm.and(+ scratch.writable(),+ scratch.inner(),+ RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32),+ OperandSize::S32,+ )?;+ masm.branch(+ IntCmpKind::Ne,+ scratch.inner(),+ scratch.inner().into(),+ skip,+ OperandSize::S32,+ )+ })+ }++ /// Adds `gc_ref` to the DRC heap's over-approximated stack-roots list.+ ///+ /// The object is linked at the head, marked as linked, and retained for+ /// the list's ownership.+ ///+ /// Returns the DRC heap data pointer and the updated roots-list length,+ /// both allocated here and owned by the caller, which must eventually free+ /// them. The length feeds the collection-threshold check.+ fn emit_push_over_approx_stack_root(+ &mut self,+ gc_ref: Reg,+ object_addr: Reg,+ ) -> Result<(Reg, Reg)> {+ let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data();+ let roots_head_offset = u32::from(+ self.env+ .vmoffsets+ .ptr+ .vmdrc_heap_data_over_approximated_stack_roots(),+ );+ let roots_len_offset = u32::from(+ self.env+ .vmoffsets+ .ptr+ .vmdrc_heap_data_current_over_approximated_stack_roots_len(),+ );+ let next_offset = self+ .env+ .vmoffsets+ .vm_drc_header_next_over_approximated_stack_root();+ let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits();++ let heap_data_reg = self.context.any_gpr(self.masm)?;+ self.masm.load_ptr(+ self.masm.address_at_vmctx(u32::from(heap_data_offset))?,+ writable!(heap_data_reg),+ )?;++ // Link this object to the old head of the over-approximated list, then+ // mark it as present so subsequent reads can take the fast path. Both+ // steps only need a temporary, so they share the scratch register.+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.load(+ masm.address_at_reg(heap_data_reg, roots_head_offset)?,+ scratch.writable(),+ OperandSize::S32,+ )?;+ masm.store(+ scratch.inner().into(),+ masm.address_at_reg(object_addr, next_offset)?,+ OperandSize::S32,+ )?;++ masm.load(+ masm.address_at_reg(object_addr, reserved_offset)?,+ scratch.writable(),+ OperandSize::S32,+ )?;+ masm.or(+ scratch.writable(),+ scratch.inner(),+ RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32),+ OperandSize::S32,+ )?;+ masm.store(+ scratch.inner().into(),+ masm.address_at_reg(object_addr, reserved_offset)?,+ OperandSize::S32,+ )+ })?;++ // Retain the object for the list's ownership.+ let count = self.emit_mutate_ref_count(object_addr, RefCountMutation::Increment)?;+ self.emit_store_ref_count(object_addr, count)?;+ self.context.free_reg(count);++ // Publish the new head, then the updated length. The length outlives+ // this helper, so it gets its own allocated register.+ self.masm.store(+ gc_ref.into(),+ self.masm.address_at_reg(heap_data_reg, roots_head_offset)?,+ OperandSize::S32,+ )?;+ let roots_len = self.context.any_gpr(self.masm)?;+ self.masm.load(+ self.masm.address_at_reg(heap_data_reg, roots_len_offset)?,+ writable!(roots_len),+ OperandSize::S32,+ )?;+ self.masm.add(+ writable!(roots_len),+ roots_len,+ RegImm::i32(1),+ OperandSize::S32,+ )?;+ self.masm.store(+ roots_len.into(),+ self.masm.address_at_reg(heap_data_reg, roots_len_offset)?,+ OperandSize::S32,+ )?;++ Ok((heap_data_reg, roots_len))+ }++ /// Forces a collection when the over-approximated roots list has reached+ /// both twice its post-GC length and the absolute minimum threshold.+ ///+ /// This method consumes and frees both register arguments before a+ /// possible call to `force_gc`.+ ///+ /// The doubled post-GC length is only needed for the first comparison, so+ /// it lives in the dedicated scratch register. This helper must therefore+ /// not be called from inside another `with_scratch` scope.+ fn emit_maybe_force_gc(&mut self, current_len: Reg, heap_data: Reg) -> Result<()> {+ let last_len_offset = u32::from(+ self.env+ .vmoffsets+ .ptr+ .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(),+ );+ let skip_gc = self.masm.get_label()?;+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.load(+ masm.address_at_reg(heap_data, last_len_offset)?,+ scratch.writable(),+ OperandSize::S32,+ )?;+ masm.add(+ scratch.writable(),+ scratch.inner(),+ scratch.inner().into(),+ OperandSize::S32,+ )?;+ masm.branch(+ IntCmpKind::LtU,+ current_len,+ scratch.inner().into(),+ skip_gc,+ OperandSize::S32,+ )+ })?;+ let min_threshold = i32::try_from(DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD).unwrap();+ self.masm.branch(+ IntCmpKind::LtU,+ current_len,+ RegImm::i32(min_threshold),+ skip_gc,+ OperandSize::S32,+ )?;++ self.context.free_reg(heap_data);+ self.context.free_reg(current_len);++ let force_gc = self.env.builtins.force_gc::<M::ABI>()?;+ FnCall::emit::<M>(+ &mut self.env,+ self.masm,+ &mut self.context,+ Callee::Builtin(force_gc),+ )?;+ self.context.pop_and_free(self.masm)?;++ self.masm.bind(skip_gc)+ }++ /// Loads a reference count and applies `mutation`, returning the register+ /// holding the updated value. The caller decides whether and when to store+ /// it, and must eventually free the returned register.+ ///+ /// The count is allocated rather than taken from the caller because the+ /// decrement path inspects it after this helper returns, to decide between+ /// storing it and calling `drop_gc_ref`.+ fn emit_mutate_ref_count(+ &mut self,+ object_addr: Reg,+ mutation: RefCountMutation,+ ) -> Result<Reg> {+ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();+ let count = self.context.any_gpr(self.masm)?;+ self.masm.load(+ self.masm.address_at_reg(object_addr, ref_count_offset)?,+ writable!(count),+ OperandSize::S64,+ )?;+ match mutation {+ RefCountMutation::Increment => {+ self.masm+ .add(writable!(count), count, RegImm::i64(1), OperandSize::S64)?+ }+ RefCountMutation::Decrement => {+ self.masm+ .sub(writable!(count), count, RegImm::i64(1), OperandSize::S64)?+ }+ }+ Ok(count)+ }++ fn emit_store_ref_count(&mut self, object_addr: Reg, count: Reg) -> Result<()> {+ let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count();+ self.masm.store(+ count.into(),+ self.masm.address_at_reg(object_addr, ref_count_offset)?,+ OperandSize::S64,+ )+ }+}diff --git a/winch/codegen/src/codegen/gc.rs b/winch/codegen/src/codegen/gc.rsnew file mode 100644index 0000000000..a7f7ad86d5--- /dev/null+++ b/winch/codegen/src/codegen/gc.rs@@ -0,0 +1,140 @@+use super::{CodeGen, Emission};+use crate::{+ Result,+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm},+ reg::{Reg, writable},+};+use cranelift_codegen::MachLabel;+use wasmtime_cranelift::TRAP_GC_HEAP_CORRUPT;+use wasmtime_environ::{I31_DISCRIMINANT, PtrSize};++impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>+where+ M: MacroAssembler,+{+ /// Branches to `skip` when `gc_ref` is null or an unboxed i31 reference.+ ///+ /// A `VMGcRef` is null when all its bits are zero. The first branch tests+ /// this by comparing the register with itself. The second test masks off+ /// every bit except the i31 discriminant and skips the heap access when that bit is set.+ ///+ /// The discriminant test uses the dedicated scratch register rather than an+ /// allocated temporary: the scratch register is non-allocatable, so it can+ /// never alias `gc_ref`, which masking in place would otherwise destroy on+ /// the fall-through path. Note that this means the helper must not be+ /// called from inside another `with_scratch` scope.+ pub(super) fn emit_skip_if_gc_ref_is_null_or_i31(+ &mut self,+ gc_ref: Reg,+ skip: MachLabel,+ ) -> Result<()> {+ self.masm.branch(+ IntCmpKind::Eq,+ gc_ref,+ gc_ref.into(),+ skip,+ OperandSize::S32,+ )?;+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S32)?;+ masm.and(+ scratch.writable(),+ scratch.inner(),+ RegImm::i32(I31_DISCRIMINANT as i32),+ OperandSize::S32,+ )?;+ masm.branch(+ IntCmpKind::Ne,+ scratch.inner(),+ scratch.inner().into(),+ skip,+ OperandSize::S32,+ )+ })+ }++ /// Loads the GC heap's base address and current length.+ ///+ /// Both returned registers are allocated from the code-generation context+ /// and are owned by the caller. The caller must eventually free them.+ pub(super) fn emit_load_gc_heap_base_and_bound(&mut self) -> Result<(Reg, Reg)> {+ let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();+ let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base();+ let gc_heap_len_offset = self+ .env+ .vmoffsets+ .ptr+ .vm_store_context()+ .gc_heap_current_length();+ let heap_base = self.context.any_gpr(self.masm)?;+ let heap_bound = self.context.any_gpr(self.masm)?;++ self.masm.load_ptr(+ self.masm+ .address_at_vmctx(u32::from(store_context_offset))?,+ writable!(heap_base),+ )?;+ self.masm.load(+ self.masm+ .address_at_reg(heap_base, u32::from(gc_heap_len_offset))?,+ writable!(heap_bound),+ OperandSize::S64,+ )?;+ self.masm.load_ptr(+ self.masm+ .address_at_reg(heap_base, u32::from(gc_heap_base_offset))?,+ writable!(heap_base),+ )?;++ Ok((heap_base, heap_bound))+ }++ /// Checks that accessing `access_extent` bytes starting at `gc_ref` stays+ /// within the GC heap.+ ///+ /// Callers must perform this check before converting a non-null, non-i31+ /// reference into a native address and accessing its object header.+ ///+ /// The end of the access is computed in the dedicated scratch register. It+ /// is non-allocatable, so it can alias neither `gc_ref` — which would leave+ /// the reference offset by `access_extent` — nor `heap_bound`, which would+ /// turn the comparison into `cmp scratch, scratch` and silently disable the+ /// check. Using it also guarantees nothing is emitted between `cmp` and+ /// `trapif`. Note that this means the helper must not be called from inside+ /// another `with_scratch` scope.+ pub(super) fn emit_gc_ref_bounds_check(+ &mut self,+ gc_ref: Reg,+ heap_bound: Reg,+ access_extent: i64,+ ) -> Result<()> {+ self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S64)?;+ masm.add(+ scratch.writable(),+ scratch.inner(),+ RegImm::i64(access_extent),+ OperandSize::S64,+ )?;+ masm.cmp(scratch.inner(), heap_bound.into(), OperandSize::S64)?;+ masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT)+ })+ }++ /// Converts a bounds-checked `VMGcRef` heap offset into a native address.+ ///+ /// The returned register is allocated from the code-generation context and+ /// is owned by the caller, which must eventually free it. Allocating here+ /// keeps the result from aliasing `gc_ref`, which the addition would+ /// otherwise clobber. Callers that want the address to reuse a register+ /// that is dead by this point should free that register first and let the+ /// allocator hand it back.+ pub(super) fn emit_gc_ref_addr(&mut self, gc_ref: Reg, heap_base: Reg) -> Result<Reg> {+ let dst = self.context.any_gpr(self.masm)?;+ self.masm+ .mov(writable!(dst), heap_base.into(), OperandSize::S64)?;+ self.masm+ .add(writable!(dst), dst, gc_ref.into(), OperandSize::S64)?;+ Ok(dst)+ }+}diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex 0df13d76d9..1577c091ae 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -40,6 +40,8 @@ pub(crate) use control::*;mod builtin;pub use builtin::*;pub(crate) mod bounds;+mod drc;+mod gc;use bounds::{Bounds, ImmOffset, Index};@@ -490,6 +492,14 @@ where}}+ /// Whether a GC barrier must be emitted when writing or reading a+ /// reference of the given type through a collector-visible location.+ pub fn gc_barrier_needed(&self, ty: &WasmValType) -> bool {+ ty.is_vmgcref_type_and_not_i31()+ && self.tunables.collector+ == Some(wasmtime_environ::Collector::DeferredReferenceCounting)+ }+/// Emits a a series of instructions that will type check a function reference call.pub fn emit_typecheck_funcref(&mut self,diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rsindex 791f7354ee..90deb3d394 100644--- a/winch/codegen/src/visitor.rs+++ b/winch/codegen/src/visitor.rs@@ -2066,11 +2066,15 @@ wherelet index = GlobalIndex::from_u32(global_index);let (ty, base, offset) = self.emit_get_global_addr(index)?;let addr = self.masm.address_at_reg(base, offset)?;- let dst = self.context.reg_for_type(ty, self.masm)?;- self.masm.load(addr, writable!(dst), ty.try_into()?)?;- self.context.stack.push(Val::reg(dst, ty));+ if self.gc_barrier_needed(&ty) {+ self.emit_drc_read_barrier(ty, base, addr)?;+ } else {+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;+ self.context.stack.push(Val::reg(gc_ref, ty));- self.context.free_reg(base);+ self.context.free_reg(base);+ }Ok(())}@@ -2080,11 +2084,15 @@ wherelet (ty, base, offset) = self.emit_get_global_addr(index)?;let addr = self.masm.address_at_reg(base, offset)?;- let typed_reg = self.context.pop_to_reg(self.masm, None)?;- self.masm- .store(typed_reg.reg.into(), addr, ty.try_into()?)?;- self.context.free_reg(typed_reg.reg);- self.context.free_reg(base);+ if self.gc_barrier_needed(&ty) {+ self.emit_drc_write_barrier(ty, base, addr)?;+ } else {+ let typed_reg = self.context.pop_to_reg(self.masm, None)?;+ self.masm+ .store(typed_reg.reg.into(), addr, ty.try_into()?)?;+ self.context.free_reg(typed_reg.reg);+ self.context.free_reg(base);+ }Ok(())}