Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Remove Cargo features from `wasmtime-internal-cranelift` (#13560)

Alex Crichton07db6feef273
diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml
index 0aadcedf23..381e304a2f 100644
--- a/crates/cranelift/Cargo.toml
+++ b/crates/cranelift/Cargo.toml
@@ -16,8 +16,7 @@ workspace = true
[dependencies]
log = { workspace = true }
-wasmtime-environ = { workspace = true, features = ['compile'] }
-cranelift-codegen = { workspace = true, features = ["host-arch", "timing"] }
+cranelift-codegen = { workspace = true, features = ["host-arch", "timing", "incremental-cache"] }
cranelift-frontend = { workspace = true }
cranelift-entity = { workspace = true }
cranelift-native = { workspace = true }
@@ -35,17 +34,20 @@ pulley-interpreter = { workspace = true, optional = true }
wasmtime-core = { workspace = true }
wasmtime-unwinder = { workspace = true, features = ["cranelift"] }
+[dependencies.wasmtime-environ]
+workspace = true
+features = [
+ "compile",
+ "component-model",
+ "gc-drc",
+ "gc-null",
+ "gc-copying",
+ "stack-switching",
+ "threads",
+]
+
[features]
all-arch = ["cranelift-codegen/all-arch"]
host-arch = ["cranelift-codegen/host-arch"]
pulley = ["cranelift-codegen/pulley", "dep:pulley-interpreter"]
trace-log = ["cranelift-codegen/trace-log"]
-component-model = ["wasmtime-environ/component-model"]
-incremental-cache = ["cranelift-codegen/incremental-cache"]
-wmemcheck = ["wasmtime-environ/wmemcheck"]
-gc = ["wasmtime-environ/gc"]
-gc-drc = ["gc", "wasmtime-environ/gc-drc"]
-gc-null = ["gc", "wasmtime-environ/gc-null"]
-gc-copying = ["gc", "wasmtime-environ/gc-copying"]
-stack-switching = []
-threads = ["wasmtime-environ/threads"]
diff --git a/crates/cranelift/build.rs b/crates/cranelift/build.rs
index d70c2572c8..dfb67d82f7 100644
--- a/crates/cranelift/build.rs
+++ b/crates/cranelift/build.rs
@@ -2,4 +2,23 @@ fn main() {
if std::env::var("CARGO_CFG_FUZZING").is_ok() {
println!("cargo:rustc-cfg=gc_zeal");
}
+
+ // Pretend that Cranelift has these features. These are injected via the
+ // `wasmtime_environ::foreach_builtin_function!` macro so to ensure that we
+ // don't get spurious warnings about unused cfg's these are printed out
+ // here.
+ faux_feature("stack-switching");
+ faux_feature("gc");
+ faux_feature("gc-copying");
+ faux_feature("gc-null");
+ faux_feature("gc-drc");
+ faux_feature("threads");
+ faux_feature("wmemcheck");
+ faux_feature("component-model");
+ faux_feature("incremental-cache");
+}
+
+fn faux_feature(feature: &str) {
+ println!("cargo:rustc-check-cfg=cfg(feature, values(\"{feature}\"))");
+ println!("cargo:rustc-cfg=feature=\"{feature}\"");
}
diff --git a/crates/cranelift/src/alias_region_key.rs b/crates/cranelift/src/alias_region_key.rs
index 00fd51ebc1..115ccae0b9 100644
--- a/crates/cranelift/src/alias_region_key.rs
+++ b/crates/cranelift/src/alias_region_key.rs
@@ -21,10 +21,6 @@ pub(crate) enum AliasRegionKey {
},
/// A `VMStoreContext` field access.
- #[cfg_attr(
- not(feature = "component-model"),
- expect(dead_code, reason = "easier not to cfg off")
- )]
VMStoreContext {
/// The offset of the `VMStoreContext` field being accessed.
offset: u32,
diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rs
index ab9bfc3dc9..9c6c1dd75d 100644
--- a/crates/cranelift/src/bounds_checks.rs
+++ b/crates/cranelift/src/bounds_checks.rs
@@ -53,7 +53,6 @@ pub enum BoundsCheck {
/// ```ignore
/// index + object_size <= bound
/// ```
- #[cfg(feature = "gc")]
StaticObjectField {
offset: u32,
access_size: u8,
@@ -64,7 +63,6 @@ pub enum BoundsCheck {
///
/// It is *your* responsibility to ensure that the `offset + access_size <=
/// object_size` precondition holds.
- #[cfg(feature = "gc")]
DynamicObjectField {
offset: ir::Value,
object_size: ir::Value,
@@ -91,7 +89,6 @@ pub fn bounds_check_and_compute_addr(
access_size,
} => bounds_check_field_access(builder, env, heap, index, offset, access_size, trap),
- #[cfg(feature = "gc")]
BoundsCheck::StaticObjectField {
offset,
access_size,
@@ -129,7 +126,6 @@ pub fn bounds_check_and_compute_addr(
// Compute the index of the end of the object, bounds check that and get
// a pointer to just after the object, and then reverse offset from that
// to get the pointer to the field being accessed.
- #[cfg(feature = "gc")]
BoundsCheck::DynamicObjectField {
offset,
object_size,
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 5cb1f76eca..c71e50eaae 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -45,11 +45,9 @@ use wasmtime_environ::{
};
use wasmtime_unwinder::ExceptionTableBuilder;
-#[cfg(feature = "component-model")]
pub(crate) mod component;
struct IncrementalCacheContext {
- #[cfg(feature = "incremental-cache")]
cache_store: Arc<dyn CacheStore>,
num_hits: usize,
num_cached: usize,
@@ -87,7 +85,6 @@ pub struct Compiler {
linkopts: LinkOptions,
cache_store: Option<Arc<dyn CacheStore>>,
clif_dir: Option<path::PathBuf>,
- #[cfg(feature = "wmemcheck")]
pub(crate) wmemcheck: bool,
}
@@ -138,7 +135,6 @@ impl Compiler {
linkopts,
cache_store,
clif_dir,
- #[cfg(feature = "wmemcheck")]
wmemcheck,
}
}
@@ -628,7 +624,6 @@ impl wasmtime_environ::Compiler for Compiler {
let ty = types[ty].unwrap_func();
self.compile_wasm_to_array_trampoline(ty, key, symbol)
}
- #[cfg(feature = "component-model")]
FuncKey::ResourceDropTrampoline => {
let ty = types.find_resource_drop_signature().unwrap();
let ty = types[ty].unwrap_func();
@@ -645,7 +640,6 @@ impl wasmtime_environ::Compiler for Compiler {
FuncKey::PulleyHostCall(_) => unreachable!(),
- #[cfg(feature = "component-model")]
FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => {
unreachable!()
}
@@ -856,7 +850,6 @@ impl wasmtime_environ::Compiler for Compiler {
self.isa.is_branch_protection_enabled()
}
- #[cfg(feature = "component-model")]
fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {
self
}
@@ -969,7 +962,6 @@ impl InliningCompiler for Compiler {
.map(|name| FuncKey::from_raw_parts(name.namespace, name.index))
.filter(|key| match key {
FuncKey::DefinedWasmFunction(..) => true,
- #[cfg(feature = "component-model")]
FuncKey::UnsafeIntrinsic(..) => true,
_ => false,
}),
@@ -1029,7 +1021,6 @@ impl InliningCompiler for Compiler {
let callee = FuncKey::from_raw_parts(callee.namespace, callee.index);
match callee {
FuncKey::DefinedWasmFunction(..) => {}
- #[cfg(feature = "component-model")]
FuncKey::UnsafeIntrinsic(..) => {}
_ => return InlineCommand::KeepCall,
}
@@ -1094,56 +1085,39 @@ impl InliningCompiler for Compiler {
}
}
-#[cfg(feature = "incremental-cache")]
-mod incremental_cache {
- use super::*;
+struct CraneliftCacheStore(Arc<dyn CacheStore>);
- struct CraneliftCacheStore(Arc<dyn CacheStore>);
-
- impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore {
- fn get(&self, key: &[u8]) -> Option<std::borrow::Cow<'_, [u8]>> {
- self.0.get(key)
- }
- fn insert(&mut self, key: &[u8], val: Vec<u8>) {
- self.0.insert(key, val);
- }
+impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore {
+ fn get(&self, key: &[u8]) -> Option<std::borrow::Cow<'_, [u8]>> {
+ self.0.get(key)
}
-
- pub(super) fn compile_maybe_cached<'a>(
- context: &'a mut Context,
- isa: &dyn TargetIsa,
- cache_ctx: Option<&mut IncrementalCacheContext>,
- ) -> Result<CompiledCode, CompileError> {
- let cache_ctx = match cache_ctx {
- Some(ctx) => ctx,
- None => return compile_uncached(context, isa),
- };
-
- let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone());
- let (_compiled_code, from_cache) = context
- .compile_with_cache(isa, &mut cache_store, &mut Default::default())
- .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?;
-
- if from_cache {
- cache_ctx.num_hits += 1;
- } else {
- cache_ctx.num_cached += 1;
- }
-
- Ok(context.take_compiled_code().unwrap())
+ fn insert(&mut self, key: &[u8], val: Vec<u8>) {
+ self.0.insert(key, val);
}
}
-#[cfg(feature = "incremental-cache")]
-use incremental_cache::*;
-
-#[cfg(not(feature = "incremental-cache"))]
fn compile_maybe_cached<'a>(
context: &'a mut Context,
isa: &dyn TargetIsa,
- _cache_ctx: Option<&mut IncrementalCacheContext>,
+ cache_ctx: Option<&mut IncrementalCacheContext>,
) -> Result<CompiledCode, CompileError> {
- compile_uncached(context, isa)
+ let cache_ctx = match cache_ctx {
+ Some(ctx) => ctx,
+ None => return compile_uncached(context, isa),
+ };
+
+ let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone());
+ let (_compiled_code, from_cache) = context
+ .compile_with_cache(isa, &mut cache_store, &mut Default::default())
+ .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?;
+
+ if from_cache {
+ cache_ctx.num_hits += 1;
+ } else {
+ cache_ctx.num_cached += 1;
+ }
+
+ Ok(context.take_compiled_code().unwrap())
}
fn compile_uncached<'a>(
@@ -1285,7 +1259,6 @@ impl Compiler {
ctx
})
.unwrap_or_else(|| CompilerContext {
- #[cfg(feature = "incremental-cache")]
incremental_cache_ctx: self.cache_store.as_ref().map(|cache_store| {
IncrementalCacheContext {
cache_store: cache_store.clone(),
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index c0f2759c77..1b32303155 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -153,21 +153,17 @@ pub struct FuncEnvironment<'module_environment> {
/// Translation state at the given point.
pub(crate) stacks: FuncTranslationStacks,
- #[cfg(feature = "gc")]
ty_to_gc_layout: std::collections::HashMap<
wasmtime_environ::ModuleInternedTypeIndex,
wasmtime_environ::GcLayout,
>,
- #[cfg(feature = "gc")]
gc_heap: Option<Heap>,
/// The Cranelift global holding the GC heap's base address.
- #[cfg(feature = "gc")]
gc_heap_base: Option<ir::GlobalValue>,
/// The Cranelift global holding the GC heap's base address.
- #[cfg(feature = "gc")]
gc_heap_bound: Option<ir::GlobalValue>,
translation: &'module_environment ModuleTranslation<'module_environment>,
@@ -285,13 +281,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
entities: WasmEntities::default(),
stacks: FuncTranslationStacks::new(),
- #[cfg(feature = "gc")]
ty_to_gc_layout: std::collections::HashMap::new(),
- #[cfg(feature = "gc")]
gc_heap: None,
- #[cfg(feature = "gc")]
gc_heap_base: None,
- #[cfg(feature = "gc")]
gc_heap_bound: None,
heaps: PrimaryMap::default(),
@@ -441,10 +433,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
self.alias_region(func, key)
}
- #[cfg_attr(
- not(any(feature = "gc-copying", feature = "gc-drc", feature = "gc-null")),
- expect(dead_code, reason = "easier not to cfg off; used in upcoming commits")
- )]
pub(crate) fn vmctx_alias_region(
&mut self,
func: &mut Function,
@@ -453,7 +441,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
self.alias_region(func, AliasRegionKey::VMContext { offset })
}
- #[cfg(feature = "threads")]
fn get_memory_atomic_wait(&mut self, func: &mut Function, ty: ir::Type) -> ir::FuncRef {
match ty {
I32 => self.builtin_functions.memory_atomic_wait32(func),
@@ -773,7 +760,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
self.epoch_check_full(builder, cur_epoch_value, continuation_block);
}
- #[cfg(feature = "wmemcheck")]
fn hook_malloc_exit(&mut self, builder: &mut FunctionBuilder, retvals: &[ir::Value]) {
let check_malloc = self.builtin_functions.check_malloc(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
@@ -796,7 +782,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
builder.ins().call(check_malloc, &[vmctx, retval, len]);
}
- #[cfg(feature = "wmemcheck")]
fn hook_free_exit(&mut self, builder: &mut FunctionBuilder) {
let check_free = self.builtin_functions.check_free(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
@@ -1078,21 +1063,18 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
result_param
}
- #[cfg(feature = "wmemcheck")]
fn check_malloc_start(&mut self, builder: &mut FunctionBuilder) {
let malloc_start = self.builtin_functions.malloc_start(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
builder.ins().call(malloc_start, &[vmctx]);
}
- #[cfg(feature = "wmemcheck")]
fn check_free_start(&mut self, builder: &mut FunctionBuilder) {
let free_start = self.builtin_functions.free_start(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
builder.ins().call(free_start, &[vmctx]);
}
- #[cfg(feature = "wmemcheck")]
fn current_func_name(&self, builder: &mut FunctionBuilder) -> Option<&str> {
let func_index = match &builder.func.name {
ir::UserFuncName::User(user) => FuncIndex::from_u32(user.index),
@@ -1665,7 +1647,6 @@ impl FuncEnvironment<'_> {
let key = match self.translation.known_imported_functions[func_index] {
Some(key @ FuncKey::DefinedWasmFunction(..)) => key,
- #[cfg(feature = "component-model")]
Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,
Some(key) => {
@@ -1866,7 +1847,6 @@ impl FuncEnvironment<'_> {
}
/// Get the type index associated with an exception object.
- #[cfg(feature = "gc")]
pub(crate) fn exception_type_from_tag(&self, tag: TagIndex) -> EngineOrModuleTypeIndex {
self.module.tags[tag].exception
}
@@ -1883,7 +1863,6 @@ impl FuncEnvironment<'_> {
/// Get the runtime instance ID and defined-tag ID in that
/// instance for a particular static tag ID.
- #[cfg(feature = "gc")]
pub(crate) fn get_instance_and_tag(
&mut self,
builder: &mut FunctionBuilder<'_>,
@@ -2042,7 +2021,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// The import is always a compile-time builtin intrinsic. Make a
// direct call to that function (presumably it will eventually be
// inlined).
- #[cfg(feature = "component-model")]
Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {
let callee = self
.env
@@ -2097,7 +2075,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
/// intrinsics. The fallback of issuing a `call` to the intrinsic is always
/// suitable to do and is used in situations where the call instruction may
/// have extra context.
- #[cfg(feature = "component-model")]
fn can_directly_inline_unsafe_intrinsic(&self, abi: wasmtime_environ::Abi) -> bool {
abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest
}
@@ -2303,15 +2280,8 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// Check that they match: in the case of Wasm GC, this means doing a
// full subtype check. Otherwise, we do a simple equality check.
let matches = if features.gc() {
- #[cfg(feature = "gc")]
- {
- self.env
- .is_subtype(self.builder, callee_sig_id, caller_sig_id)
- }
- #[cfg(not(feature = "gc"))]
- {
- unreachable!()
- }
+ self.env
+ .is_subtype(self.builder, callee_sig_id, caller_sig_id)
} else {
self.builder
.ins()
@@ -5085,30 +5055,20 @@ impl FuncEnvironment<'_> {
expected: ir::Value,
timeout: ir::Value,
) -> WasmResult<ir::Value> {
- #[cfg(feature = "threads")]
- {
- let mut pos = builder.cursor();
- let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);
- let implied_ty = pos.func.dfg.value_type(expected);
- let wait_func = self.get_memory_atomic_wait(&mut pos.func, implied_ty);
+ let mut pos = builder.cursor();
+ let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);
+ let implied_ty = pos.func.dfg.value_type(expected);
+ let wait_func = self.get_memory_atomic_wait(&mut pos.func, implied_ty);
- let (memory_vmctx, defined_memory_index) =
- self.memory_vmctx_and_defined_index(&mut pos, memory_index);
+ let (memory_vmctx, defined_memory_index) =
+ self.memory_vmctx_and_defined_index(&mut pos, memory_index);
- let call_inst = pos.ins().call(
- wait_func,
- &[memory_vmctx, defined_memory_index, addr, expected, timeout],
- );
- let ret = pos.func.dfg.inst_results(call_inst)[0];
- Ok(builder.ins().ireduce(ir::types::I32, ret))
- }
- #[cfg(not(feature = "threads"))]
- {
- let _ = (builder, memory_index, addr, expected, timeout);
- Err(wasmtime_environ::WasmError::Unsupported(
- "threads support disabled at compile time".to_string(),
- ))
- }
+ let call_inst = pos.ins().call(
+ wait_func,
+ &[memory_vmctx, defined_memory_index, addr, expected, timeout],
+ );
+ let ret = pos.func.dfg.inst_results(call_inst)[0];
+ Ok(builder.ins().ireduce(ir::types::I32, ret))
}
pub fn translate_atomic_notify(
@@ -5119,28 +5079,18 @@ impl FuncEnvironment<'_> {
addr: ir::Value,
count: ir::Value,
) -> WasmResult<ir::Value> {
- #[cfg(feature = "threads")]
- {
- let mut pos = builder.cursor();
- let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);
- let atomic_notify = self.builtin_functions.memory_atomic_notify(&mut pos.func);
-
- let (memory_vmctx, defined_memory_index) =
- self.memory_vmctx_and_defined_index(&mut pos, memory_index);
- let call_inst = pos.ins().call(
- atomic_notify,
- &[memory_vmctx, defined_memory_index, addr, count],
- );
- let ret = pos.func.dfg.inst_results(call_inst)[0];
- Ok(builder.ins().ireduce(ir::types::I32, ret))
- }
- #[cfg(not(feature = "threads"))]
- {
- let _ = (builder, memory_index, addr, count);
- Err(wasmtime_environ::WasmError::Unsupported(
- "threads support disabled at compile time".to_string(),
- ))
- }
+ let mut pos = builder.cursor();
+ let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);
+ let atomic_notify = self.builtin_functions.memory_atomic_notify(&mut pos.func);
+
+ let (memory_vmctx, defined_memory_index) =
+ self.memory_vmctx_and_defined_index(&mut pos, memory_index);
+ let call_inst = pos.ins().call(
+ atomic_notify,
+ &[memory_vmctx, defined_memory_index, addr, count],
+ );
+ let ret = pos.func.dfg.inst_results(call_inst)[0];
+ Ok(builder.ins().ireduce(ir::types::I32, ret))
}
pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
@@ -5223,7 +5173,6 @@ impl FuncEnvironment<'_> {
self.epoch_function_entry(builder);
}
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck {
let func_name = self.current_func_name(builder);
if func_name == Some("malloc") {
@@ -5375,7 +5324,6 @@ impl FuncEnvironment<'_> {
}
pub fn handle_before_return(&mut self, retvals: &[ir::Value], builder: &mut FunctionBuilder) {
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck {
let func_name = self.current_func_name(builder);
if func_name == Some("malloc") {
@@ -5384,8 +5332,6 @@ impl FuncEnvironment<'_> {
self.hook_free_exit(builder);
}
}
- #[cfg(not(feature = "wmemcheck"))]
- let _ = (retvals, builder);
}
pub fn before_load(
@@ -5395,7 +5341,6 @@ impl FuncEnvironment<'_> {
addr: ir::Value,
offset: u64,
) {
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck {
let check_load = self.builtin_functions.check_load(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
@@ -5405,8 +5350,6 @@ impl FuncEnvironment<'_> {
.ins()
.call(check_load, &[vmctx, num_bytes, addr, offset_val]);
}
- #[cfg(not(feature = "wmemcheck"))]
- let _ = (builder, val_size, addr, offset);
}
pub fn before_store(
@@ -5416,7 +5359,6 @@ impl FuncEnvironment<'_> {
addr: ir::Value,
offset: u64,
) {
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck {
let check_store = self.builtin_functions.check_store(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
@@ -5426,8 +5368,6 @@ impl FuncEnvironment<'_> {
.ins()
.call(check_store, &[vmctx, num_bytes, addr, offset_val]);
}
- #[cfg(not(feature = "wmemcheck"))]
- let _ = (builder, val_size, addr, offset);
}
pub fn update_global(
@@ -5436,7 +5376,6 @@ impl FuncEnvironment<'_> {
global_index: GlobalIndex,
value: ir::Value,
) {
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck {
if global_index.index() == 0 {
// We are making the assumption that global 0 is the auxiliary stack pointer.
@@ -5446,8 +5385,6 @@ impl FuncEnvironment<'_> {
builder.ins().call(update_stack_pointer, &[vmctx, value]);
}
}
- #[cfg(not(feature = "wmemcheck"))]
- let _ = (builder, global_index, value);
}
pub fn before_memory_grow(
@@ -5456,14 +5393,11 @@ impl FuncEnvironment<'_> {
num_pages: ir::Value,
mem_index: MemoryIndex,
) {
- #[cfg(feature = "wmemcheck")]
if self.compiler.wmemcheck && mem_index.as_u32() == 0 {
let update_mem_size = self.builtin_functions.update_mem_size(builder.func);
let vmctx = self.vmctx_val(&mut builder.cursor());
builder.ins().call(update_mem_size, &[vmctx, num_pages]);
}
- #[cfg(not(feature = "wmemcheck"))]
- let _ = (builder, num_pages, mem_index);
}
/// If the ISA has rounding instructions, let Cranelift use them. But if
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index 7dd7b5905f..cb5ec2bb4f 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -1,35 +1,31 @@
//! Interface to compiling GC-related things.
-//!
-//! This module and its interface are implemented twice: once when the `gc`
-//! cargo feature is enabled and once when the feature is disabled. The goal is
-//! to have just a single `cfg(feature = "gc")` for the whole crate, which
-//! selects between these two implementations.
-
-use crate::func_environ::FuncEnvironment;
-use cranelift_codegen::ir;
+
+use crate::TRAP_ARRAY_OUT_OF_BOUNDS;
+use crate::alias_region_key::AliasRegionKey;
+use crate::bounds_checks::BoundsCheck;
+use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};
+use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};
+use crate::trap::TranslateTrap;
+use crate::{Reachability, TRAP_GC_HEAP_CORRUPT, TRAP_INTERNAL_ASSERT};
+use cranelift_codegen::ir::immediates::Offset32;
+use cranelift_codegen::ir::{BlockArg, ExceptionTableData, ExceptionTableItem};
+use cranelift_codegen::{
+ cursor::FuncCursor,
+ ir::{self, InstBuilder, condcodes::IntCC},
+};
+use cranelift_entity::packed_option::ReservedValue;
use cranelift_frontend::FunctionBuilder;
+use smallvec::{SmallVec, smallvec};
use wasmtime_environ::{
- GcTypeLayouts, TagIndex, TypeIndex, WasmRefType, WasmResult, WasmStorageType,
+ Collector, GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, I31_DISCRIMINANT,
+ ModuleInternedTypeIndex, PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,
+ WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
+ wasm_unsupported,
};
-#[cfg(feature = "gc")]
-mod enabled;
-#[cfg(feature = "gc")]
-use enabled as imp;
-
-#[cfg(not(feature = "gc"))]
-mod disabled;
-#[cfg(not(feature = "gc"))]
-use disabled as imp;
-
-// Re-export the GC compilation interface from the implementation that we chose
-// based on the compile-time features enabled.
-pub use imp::*;
-
/// A trait for different collectors to emit any GC barriers they might require.
pub trait GcCompiler {
/// Get the GC type layouts for this GC compiler.
- #[cfg_attr(not(feature = "gc"), allow(dead_code))]
fn layouts(&self) -> &dyn GcTypeLayouts;
/// Whether or not this `GcCompiler`'s collector can move objects.
@@ -39,7 +35,6 @@ pub trait GcCompiler {
///
/// The array should be of the given type and its elements initialized as
/// described by the given `ArrayInit`.
- #[cfg_attr(not(feature = "gc"), allow(dead_code))]
fn alloc_uninit_array(
&mut self,
func_env: &mut FuncEnvironment<'_>,
@@ -52,7 +47,6 @@ pub trait GcCompiler {
///
/// The struct should be of the given type and its fields initialized to the
/// given values.
- #[cfg_attr(not(feature = "gc"), allow(dead_code))]
fn alloc_struct(
&mut self,
func_env: &mut FuncEnvironment<'_>,
@@ -68,7 +62,6 @@ pub trait GcCompiler {
/// uninitialized; that is the responsibility of generated code to
/// fill in. `tag_index` is used only to look up the appropriate
/// exception object type.
- #[cfg_attr(not(feature = "gc"), allow(dead_code))]
fn alloc_exn(
&mut self,
func_env: &mut FuncEnvironment<'_>,
@@ -180,7 +173,6 @@ pub trait GcCompiler {
///
/// This initializes a previously-uninitialized field, so barriers on the
/// value previously stored at `addr`, if necessary, should not be executed.
- #[cfg_attr(not(feature = "gc"), allow(dead_code))]
fn init_field(
&mut self,
func_env: &mut FuncEnvironment<'_>,
@@ -190,3 +182,1550 @@ pub trait GcCompiler {
val: ir::Value,
) -> WasmResult<()>;
}
+
+mod copying;
+mod drc;
+mod null;
+
+/// Get the default GC compiler.
+pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {
+ // If this function requires a GC compiler, that is not too bad of an
+ // over-approximation for it requiring a GC heap.
+ func_env.needs_gc_heap = true;
+
+ match func_env.tunables.collector {
+ Some(Collector::DeferredReferenceCounting) => Ok(Box::new(drc::DrcCompiler::default())),
+
+ Some(Collector::Null) => Ok(Box::new(null::NullCompiler::default())),
+
+ Some(Collector::Copying) => Ok(Box::new(copying::CopyingCompiler::default())),
+
+ None => Err(wasm_unsupported!(
+ "support for GC types disabled at configuration time"
+ )),
+ }
+}
+
+fn unbarriered_load_gc_ref(
+ builder: &mut FunctionBuilder,
+ ty: WasmHeapType,
+ ptr_to_gc_ref: ir::Value,
+ flags: ir::MemFlagsData,
+) -> WasmResult<ir::Value> {
+ debug_assert!(ty.is_vmgcref_type());
+ let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);
+ if ty != WasmHeapType::I31 {
+ builder.declare_value_needs_stack_map(gc_ref);
+ }
+ Ok(gc_ref)
+}
+
+fn unbarriered_store_gc_ref(
+ builder: &mut FunctionBuilder,
+ ty: WasmHeapType,
+ dst: ir::Value,
+ gc_ref: ir::Value,
+ flags: ir::MemFlagsData,
+) -> WasmResult<()> {
+ debug_assert!(ty.is_vmgcref_type());
+ builder.ins().store(flags, gc_ref, dst, 0);
+ Ok(())
+}
+
+/// Emit CLIF to call the `gc_alloc_raw` libcall.
+fn emit_gc_raw_alloc(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ kind: VMGcKind,
+ ty: ModuleInternedTypeIndex,
+ size: ir::Value,
+ align: u32,
+ reserved_bits: u32,
+) -> ir::Value {
+ let gc_alloc_raw_builtin = func_env.builtin_functions.gc_alloc_raw(builder.func);
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+ let kind = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(kind.as_u32() | reserved_bits));
+
+ let ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);
+
+ assert!(align.is_power_of_two());
+ let align = builder.ins().iconst(ir::types::I32, i64::from(align));
+
+ let call_inst = builder
+ .ins()
+ .call(gc_alloc_raw_builtin, &[vmctx, kind, ty, size, align]);
+
+ let gc_ref = builder.func.dfg.first_result(call_inst);
+ builder.declare_value_needs_stack_map(gc_ref);
+ gc_ref
+}
+
+/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the
+/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.
+///
+/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).
+fn emit_gc_kind_assert(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ gc_ref: ir::Value,
+ expected_kind: VMGcKind,
+) {
+ if !cfg!(gc_zeal) {
+ return;
+ }
+
+ func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);
+
+ let kind_addr = func_env.prepare_gc_ref_access(
+ builder,
+ gc_ref,
+ BoundsCheck::StaticObjectField {
+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,
+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,
+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,
+ },
+ );
+ let flags = func_env.gc_memflags(&mut builder.func).with_readonly();
+ let kind_and_reserved_bits = builder.ins().load(ir::types::I32, flags, kind_addr, 0);
+ let kind_mask = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(VMGcKind::MASK));
+ let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);
+
+ let expected_kind = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));
+
+ // NB: Do a subtype check rather than a strict equality check. See
+ // `VMGcKind::matches` for details.
+ let and = builder.ins().band(actual_kind, expected_kind);
+ let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);
+
+ builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);
+}
+
+/// Read a struct field or array element from its raw address in the GC heap.
+///
+/// The given address MUST have already been bounds-checked via
+/// `prepare_gc_ref_access`.
+pub fn read_field_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ty: WasmStorageType,
+ addr: ir::Value,
+ extension: Option<Extension>,
+) -> WasmResult<ir::Value> {
+ assert_eq!(extension.is_none(), matches!(ty, WasmStorageType::Val(_)));
+ assert_eq!(
+ extension.is_some(),
+ matches!(ty, WasmStorageType::I8 | WasmStorageType::I16)
+ );
+
+ // Data inside GC objects is always little endian.
+ let flags = func_env
+ .gc_memflags(&mut builder.func)
+ .with_endianness(ir::Endianness::Little);
+
+ let value = match ty {
+ WasmStorageType::I8 => builder.ins().load(ir::types::I8, flags, addr, 0),
+ WasmStorageType::I16 => builder.ins().load(ir::types::I16, flags, addr, 0),
+ WasmStorageType::Val(v) => match v {
+ WasmValType::I32 => builder.ins().load(ir::types::I32, flags, addr, 0),
+ WasmValType::I64 => builder.ins().load(ir::types::I64, flags, addr, 0),
+ WasmValType::F32 => builder.ins().load(ir::types::F32, flags, addr, 0),
+ WasmValType::F64 => builder.ins().load(ir::types::F64, flags, addr, 0),
+ WasmValType::V128 => builder.ins().load(ir::types::I8X16, flags, addr, 0),
+ WasmValType::Ref(r) => match r.heap_type.top() {
+ WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
+ gc_compiler(func_env)?
+ .translate_read_gc_reference(func_env, builder, r, addr, flags)?
+ }
+ WasmHeapTopType::Func => {
+ let expected_ty = match r.heap_type {
+ WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),
+ WasmHeapType::ConcreteFunc(ty) => ty.unwrap_module_type_index(),
+ WasmHeapType::NoFunc => {
+ let null = builder.ins().iconst(func_env.pointer_type(), 0);
+ if !r.nullable {
+ // Because `nofunc` is uninhabited, and this
+ // reference is non-null, this is unreachable
+ // code. Unconditionally trap via conditional
+ // trap instructions to avoid inserting block
+ // terminators in the middle of this block.
+ builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);
+ }
+ return Ok(null);
+ }
+ _ => unreachable!("not a function heap type"),
+ };
+ let expected_ty = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(expected_ty.as_bits()));
+
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+ let func_ref_id = builder.ins().load(ir::types::I32, flags, addr, 0);
+ let get_interned_func_ref = func_env
+ .builtin_functions
+ .get_interned_func_ref(builder.func);
+
+ let call_inst = builder
+ .ins()
+ .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);
+ builder.func.dfg.first_result(call_inst)
+ }
+ WasmHeapTopType::Cont => {
+ // TODO(#10248) GC integration for stack switching
+ return stack_switching_unsupported();
+ }
+ },
+ },
+ };
+
+ let value = match extension {
+ Some(Extension::Sign) => builder.ins().sextend(ir::types::I32, value),
+ Some(Extension::Zero) => builder.ins().uextend(ir::types::I32, value),
+ None => value,
+ };
+
+ Ok(value)
+}
+
+pub fn intern_func_ref(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ref_type: WasmRefType,
+ func_ref: ir::Value,
+) -> WasmResult<ir::Value> {
+ assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);
+
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+ let intern_func_ref_for_gc_heap = func_env
+ .builtin_functions
+ .intern_func_ref_for_gc_heap(builder.func);
+
+ let func_ref = if ref_type.heap_type == WasmHeapType::NoFunc {
+ let null = builder.ins().iconst(func_env.pointer_type(), 0);
+ if !ref_type.nullable {
+ // Because `nofunc` is uninhabited, and this reference is
+ // non-null, this is unreachable code. Unconditionally trap
+ // via conditional trap instructions to avoid inserting
+ // block terminators in the middle of this block.
+ builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);
+ }
+ null
+ } else {
+ func_ref
+ };
+
+ // Convert the raw `funcref` into a `FuncRefTableId` for use in the
+ // GC heap.
+ let call_inst = builder
+ .ins()
+ .call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);
+ let func_ref_id = builder.func.dfg.first_result(call_inst);
+ Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))
+}
+
+fn write_func_ref_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ref_type: WasmRefType,
+ flags: ir::MemFlagsData,
+ field_addr: ir::Value,
+ func_ref: ir::Value,
+) -> WasmResult<()> {
+ // Convert the raw `funcref` into a `FuncRefTableId` for use in the
+ // GC heap.
+ let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;
+
+ // Store the id in the field.
+ builder.ins().store(flags, func_ref_id, field_addr, 0);
+
+ Ok(())
+}
+
+pub fn init_field_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ field_ty: WasmStorageType,
+ field_addr: ir::Value,
+ new_val: ir::Value,
+) -> WasmResult<()> {
+ gc_compiler(func_env)?.init_field(func_env, builder, field_ty, field_addr, new_val)
+}
+
+pub fn write_field_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ field_ty: WasmStorageType,
+ field_addr: ir::Value,
+ new_val: ir::Value,
+) -> WasmResult<()> {
+ // Data inside GC objects is always little endian.
+ let flags = func_env
+ .gc_memflags(&mut builder.func)
+ .with_endianness(ir::Endianness::Little);
+
+ match field_ty {
+ WasmStorageType::I8 => {
+ builder.ins().istore8(flags, new_val, field_addr, 0);
+ }
+ WasmStorageType::I16 => {
+ builder.ins().istore16(flags, new_val, field_addr, 0);
+ }
+ WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {
+ WasmHeapTopType::Func => {
+ write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?
+ }
+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {
+ gc_compiler(func_env)?.translate_write_gc_reference(
+ func_env, builder, r, field_addr, new_val, flags,
+ )?;
+ }
+ WasmHeapTopType::Cont => return stack_switching_unsupported(),
+ },
+ WasmStorageType::Val(_) => {
+ assert_eq!(
+ builder.func.dfg.value_type(new_val).bytes(),
+ wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty)
+ );
+ builder.ins().store(flags, new_val, field_addr, 0);
+ }
+ }
+ Ok(())
+}
+
+pub fn translate_struct_new(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ struct_type_index: TypeIndex,
+ fields: &[ir::Value],
+) -> WasmResult<ir::Value> {
+ gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)
+}
+
+fn default_value(
+ cursor: &mut FuncCursor,
+ func_env: &FuncEnvironment<'_>,
+ ty: &WasmStorageType,
+) -> ir::Value {
+ match ty {
+ WasmStorageType::I8 | WasmStorageType::I16 => cursor.ins().iconst(ir::types::I32, 0),
+ WasmStorageType::Val(v) => match v {
+ WasmValType::I32 => cursor.ins().iconst(ir::types::I32, 0),
+ WasmValType::I64 => cursor.ins().iconst(ir::types::I64, 0),
+ WasmValType::F32 => cursor.ins().f32const(0.0),
+ WasmValType::F64 => cursor.ins().f64const(0.0),
+ WasmValType::V128 => {
+ let c = cursor.func.dfg.constants.insert(vec![0; 16].into());
+ cursor.ins().vconst(ir::types::I8X16, c)
+ }
+ WasmValType::Ref(r) => {
+ assert!(r.nullable);
+ let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);
+
+ // NB: The collector doesn't need to know about null references.
+ let _ = needs_stack_map;
+
+ cursor.ins().iconst(ty, 0)
+ }
+ },
+ }
+}
+
+pub fn translate_struct_new_default(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ struct_type_index: TypeIndex,
+) -> WasmResult<ir::Value> {
+ let interned_ty = func_env.module.types[struct_type_index].unwrap_module_type_index();
+ let struct_ty = func_env.types.unwrap_struct(interned_ty)?;
+ let fields = struct_ty
+ .fields
+ .iter()
+ .map(|f| default_value(&mut builder.cursor(), func_env, &f.element_type))
+ .collect::<StructFieldsVec>();
+ gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)
+}
+
+pub fn translate_struct_get(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ struct_type_index: TypeIndex,
+ field_index: u32,
+ struct_ref: ir::Value,
+ extension: Option<Extension>,
+) -> WasmResult<ir::Value> {
+ log::trace!(
+ "translate_struct_get({struct_type_index:?}, {field_index:?}, {struct_ref:?}, {extension:?})"
+ );
+
+ // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a
+ // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that
+ // type info from `wasmparser` and through to here is a bit funky.
+ func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
+
+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
+
+ let field_index = usize::try_from(field_index).unwrap();
+ let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
+
+ let struct_layout = func_env.struct_or_exn_layout(interned_type_index);
+ let struct_size = struct_layout.size;
+
+ let field_offset = struct_layout.fields[field_index].offset;
+ let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];
+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);
+ assert!(field_offset + field_size <= struct_size);
+
+ let field_addr = func_env.prepare_gc_ref_access(
+ builder,
+ struct_ref,
+ BoundsCheck::StaticObjectField {
+ offset: field_offset,
+ access_size: u8::try_from(field_size).unwrap(),
+ object_size: struct_size,
+ },
+ );
+
+ let result = read_field_at_addr(
+ func_env,
+ builder,
+ field_ty.element_type,
+ field_addr,
+ extension,
+ );
+ log::trace!("translate_struct_get(..) -> {result:?}");
+ result
+}
+
+pub fn translate_struct_set(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ struct_type_index: TypeIndex,
+ field_index: u32,
+ struct_ref: ir::Value,
+ new_val: ir::Value,
+) -> WasmResult<()> {
+ log::trace!(
+ "translate_struct_set({struct_type_index:?}, {field_index:?}, struct_ref: {struct_ref:?}, new_val: {new_val:?})"
+ );
+
+ // TODO: See comment in `translate_struct_get` about the `trapz`.
+ func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
+
+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
+
+ let field_index = usize::try_from(field_index).unwrap();
+ let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
+
+ let struct_layout = func_env.struct_or_exn_layout(interned_type_index);
+ let struct_size = struct_layout.size;
+
+ let field_offset = struct_layout.fields[field_index].offset;
+ let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];
+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);
+ assert!(field_offset + field_size <= struct_size);
+
+ let field_addr = func_env.prepare_gc_ref_access(
+ builder,
+ struct_ref,
+ BoundsCheck::StaticObjectField {
+ offset: field_offset,
+ access_size: u8::try_from(field_size).unwrap(),
+ object_size: struct_size,
+ },
+ );
+
+ write_field_at_addr(
+ func_env,
+ builder,
+ field_ty.element_type,
+ field_addr,
+ new_val,
+ )?;
+
+ log::trace!("translate_struct_set: finished");
+ Ok(())
+}
+
+pub fn translate_exn_unbox(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ tag_index: TagIndex,
+ exn_ref: ir::Value,
+) -> WasmResult<SmallVec<[ir::Value; 4]>> {
+ log::trace!("translate_exn_unbox({tag_index:?}, {exn_ref:?})");
+
+ // We know that the `exn_ref` is not null because we reach this
+ // operation only in catch blocks, and throws are initiated from
+ // runtime code that checks for nulls first.
+
+ // Get the GcExceptionLayout associated with this tag's
+ // function type, and generate loads for each field.
+ let exception_ty_idx = func_env
+ .exception_type_from_tag(tag_index)
+ .unwrap_module_type_index();
+ let exception_ty = func_env.types.unwrap_exn(exception_ty_idx)?;
+ let exn_layout = func_env.struct_or_exn_layout(exception_ty_idx);
+ let exn_size = exn_layout.size;
+
+ // Gather accesses first because these require a borrow on
+ // `func_env`, which we later mutate below via
+ // `prepare_gc_ref_access()`.
+ let mut accesses: SmallVec<[_; 4]> = smallvec![];
+ for (field_ty, field_layout) in exception_ty.fields.iter().zip(exn_layout.fields.iter()) {
+ accesses.push((field_layout.offset, field_ty.element_type));
+ }
+
+ let mut result = smallvec![];
+ for (field_offset, field_ty) in accesses {
+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty);
+ assert!(field_offset + field_size <= exn_size);
+ let field_addr = func_env.prepare_gc_ref_access(
+ builder,
+ exn_ref,
+ BoundsCheck::StaticObjectField {
+ offset: field_offset,
+ access_size: u8::try_from(field_size).unwrap(),
+ object_size: exn_size,
+ },
+ );
+
+ let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None)?;
+ result.push(value);
+ }
+
+ log::trace!("translate_exn_unbox(..) -> {result:?}");
+ Ok(result)
+}
+
+pub fn translate_exn_throw(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ tag_index: TagIndex,
+ args: &[ir::Value],
+) -> WasmResult<()> {
+ let (instance_id, defined_tag_id) = func_env.get_instance_and_tag(builder, tag_index);
+ let exnref = gc_compiler(func_env)?.alloc_exn(
+ func_env,
+ builder,
+ tag_index,
+ args,
+ instance_id,
+ defined_tag_id,
+ )?;
+ translate_exn_throw_ref(func_env, builder, exnref)
+}
+
+pub fn translate_exn_throw_ref(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ exnref: ir::Value,
+) -> WasmResult<()> {
+ let builtin = func_env.builtin_functions.throw_ref(builder.func);
+ let sig = builder.func.dfg.ext_funcs[builtin].signature;
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+ // Generate a `try_call` with handlers from the current
+ // stack. This libcall is unique among libcall implementations of
+ // opcodes: we know the others will not throw, but `throw_ref`'s
+ // entire purpose is to throw. So if there are any handlers in the
+ // local function body, we need to attach them to this callsite
+ // like any other.
+ let continuation = builder.create_block();
+ let current_block = builder.current_block().unwrap();
+ builder.insert_block_after(continuation, current_block);
+ let continuation_call = builder.func.dfg.block_call(continuation, &[]);
+ let mut table_items = vec![ExceptionTableItem::Context(vmctx)];
+ for (tag, block) in func_env.stacks.handlers.handlers() {
+ let block_call = builder
+ .func
+ .dfg
+ .block_call(block, &[BlockArg::TryCallExn(0)]);
+ table_items.push(match tag {
+ Some(tag) => ExceptionTableItem::Tag(tag, block_call),
+ None => ExceptionTableItem::Default(block_call),
+ });
+ }
+ let etd = ExceptionTableData::new(sig, continuation_call, table_items);
+ let et = builder.func.dfg.exception_tables.push(etd);
+
+ builder.ins().try_call(builtin, &[vmctx, exnref], et);
+
+ builder.switch_to_block(continuation);
+ builder.seal_block(continuation);
+ func_env.trap(builder, crate::TRAP_UNREACHABLE);
+
+ Ok(())
+}
+
+pub fn translate_array_new(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ elem: ir::Value,
+ len: ir::Value,
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");
+ let result =
+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+ let ty = func_env.module.types[array_type_index].unwrap_module_type_index();
+ func_env.translate_entity_fill(
+ builder,
+ CheckedEntity::Array {
+ array: result,
+ ty,
+ initialized: false,
+ },
+ zero,
+ elem,
+ len,
+ )?;
+ log::trace!("translate_array_new(..) -> {result:?}");
+ Ok(result)
+}
+
+pub fn translate_array_new_default(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ len: ir::Value,
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");
+
+ let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();
+ let array_ty = func_env.types.unwrap_array(interned_ty)?;
+ let result =
+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
+ log::trace!("translate_array_new_default(..) -> {result:?}");
+ let elem = default_value(&mut builder.cursor(), func_env, &array_ty.0.element_type);
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+ func_env.translate_entity_fill(
+ builder,
+ CheckedEntity::Array {
+ array: result,
+ ty: interned_ty,
+ initialized: false,
+ },
+ zero,
+ elem,
+ len,
+ )?;
+ Ok(result)
+}
+
+pub fn translate_array_new_fixed(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ elems: &[ir::Value],
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_array_new_fixed({array_type_index:?}, {elems:?})");
+ let len = builder
+ .ins()
+ .iconst(ir::types::I32, i64::try_from(elems.len()).unwrap());
+ let result =
+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
+ log::trace!("translate_array_new_fixed(..) -> {result:?}");
+ let ty = func_env.module.types[array_type_index].unwrap_module_type_index();
+ let array_ty = func_env.types.unwrap_array(ty)?;
+
+ for (i, elem) in elems.iter().enumerate() {
+ let index = builder
+ .ins()
+ .iconst(ir::types::I32, i64::try_from(i).unwrap());
+ let addr = array_elem_addr(func_env, builder, ty, result, index)?;
+ gc_compiler(func_env)?.init_field(
+ func_env,
+ builder,
+ array_ty.0.element_type,
+ addr,
+ *elem,
+ )?;
+ }
+ Ok(result)
+}
+
+pub fn translate_array_len(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_ref: ir::Value,
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_array_len({array_ref:?})");
+
+ func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE);
+
+ let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();
+ let len_field = func_env.prepare_gc_ref_access(
+ builder,
+ array_ref,
+ // Note: We can't bounds check the whole array object's size because we
+ // don't know its length yet. Chicken and egg problem.
+ BoundsCheck::StaticOffset {
+ offset: len_offset,
+ access_size: u8::try_from(ir::types::I32.bytes()).unwrap(),
+ },
+ );
+ let flags = func_env.gc_memflags(&mut builder.func).with_readonly();
+ let result = builder.ins().load(ir::types::I32, flags, len_field, 0);
+ log::trace!("translate_array_len(..) -> {result:?}");
+ Ok(result)
+}
+
+struct ArraySizeInfo {
+ /// The `i32` size of the whole array object, in bytes.
+ obj_size: ir::Value,
+
+ /// The `i32` size of each one of the array's elements, in bytes.
+ one_elem_size: ir::Value,
+
+ /// The `i32` size of the array's base object, in bytes. This is also the
+ /// offset from the start of the array object to its elements.
+ base_size: ir::Value,
+}
+
+/// Emit code to get the dynamic size (in bytes) of a whole array object, along
+/// with some other related bits.
+fn emit_array_size_info(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ array_type_index: ModuleInternedTypeIndex,
+ // `i32` value containing the array's length.
+ array_len: ir::Value,
+) -> WasmResult<ArraySizeInfo> {
+ let array_layout = func_env.array_layout(array_type_index)?;
+
+ // Note that we check for overflow below because we can't trust the array's
+ // length: it came from inside the GC heap.
+ //
+ // We check for 32-bit multiplication overflow by performing a 64-bit
+ // multiplication and testing the high bits.
+ let one_elem_size = builder
+ .ins()
+ .iconst(ir::types::I64, i64::from(array_layout.elem_size));
+ let array_len = builder.ins().uextend(ir::types::I64, array_len);
+ let all_elems_size = builder.ins().imul(one_elem_size, array_len);
+
+ let high_bits = builder.ins().ushr_imm(all_elems_size, 32);
+ builder.ins().trapnz(high_bits, TRAP_GC_HEAP_CORRUPT);
+
+ let all_elems_size = builder.ins().ireduce(ir::types::I32, all_elems_size);
+ let base_size = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(array_layout.base_size));
+ let obj_size =
+ builder
+ .ins()
+ .uadd_overflow_trap(all_elems_size, base_size, TRAP_GC_HEAP_CORRUPT);
+
+ let one_elem_size = builder.ins().ireduce(ir::types::I32, one_elem_size);
+
+ Ok(ArraySizeInfo {
+ obj_size,
+ one_elem_size,
+ base_size,
+ })
+}
+
+/// Get the bounds-checked address of an element in an array.
+///
+/// The emitted code will trap if `index >= array.length`.
+///
+/// Returns the `ir::Value` containing the address of the `index`th element in
+/// the array. You may read or write a value of the array's element type at this
+/// address. You may not use it for any other kind of access, nor reuse this
+/// value across GC safepoints.
+fn array_elem_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ array_type_index: ModuleInternedTypeIndex,
+ array_ref: ir::Value,
+ index: ir::Value,
+) -> WasmResult<ir::Value> {
+ // First, assert that `index < array.length`.
+ //
+ // This check is visible at the Wasm-semantics level.
+ //
+ // TODO: We should emit spectre-safe bounds checks for array accesses (if
+ // configured) but we don't currently have a great way to do that here. The
+ // proper solution is to use linear memories to back GC heaps and reuse the
+ // code in `bounds_check.rs` to implement these bounds checks. That is all
+ // planned, but not yet implemented.
+
+ let len = translate_array_len(func_env, builder, array_ref).unwrap();
+
+ let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len);
+ func_env.trapz(builder, in_bounds, TRAP_ARRAY_OUT_OF_BOUNDS);
+
+ // Compute the size (in bytes) of the whole array object.
+ let ArraySizeInfo {
+ obj_size,
+ one_elem_size,
+ base_size,
+ } = emit_array_size_info(func_env, builder, array_type_index, len)?;
+
+ // Compute the offset of the `index`th element within the array object.
+ //
+ // NB: no need to check for overflow here, since at this point we know that
+ // `len * elem_size + base_size` did not overflow and `i < len`.
+ let offset_in_elems = builder.ins().imul(index, one_elem_size);
+ let offset_in_array = builder.ins().iadd(offset_in_elems, base_size);
+
+ // Finally, use the object size and element offset we just computed to
+ // perform our implementation-internal bounds checks.
+ //
+ // Checking the whole object's size, rather than the `index`th element's
+ // size allows these bounds checks to be deduplicated across repeated
+ // accesses to the same array at different indices.
+ //
+ // This check should not be visible to Wasm, and serve to protect us from
+ // our own implementation bugs. The goal is to keep any potential widgets
+ // confined within the GC heap, and turn what would otherwise be a security
+ // vulnerability into a simple bug.
+ //
+ // TODO: Ideally we should fold the first Wasm-visible bounds check into
+ // this internal bounds check, so that we aren't performing multiple,
+ // redundant bounds checks. But we should figure out how to do this in a way
+ // that doesn't defeat the object-size bounds checking's deduplication
+ // mentioned above.
+ Ok(func_env.prepare_gc_ref_access(
+ builder,
+ array_ref,
+ BoundsCheck::DynamicObjectField {
+ offset: offset_in_array,
+ object_size: obj_size,
+ },
+ ))
+}
+
+pub fn translate_array_get(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ array_ref: ir::Value,
+ index: ir::Value,
+ extension: Option<Extension>,
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");
+
+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
+
+ let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
+ let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;
+
+ let array_ty = func_env.types.unwrap_array(array_type_index)?;
+ let elem_ty = array_ty.0.element_type;
+
+ let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension)?;
+ log::trace!("translate_array_get(..) -> {result:?}");
+ Ok(result)
+}
+
+pub fn translate_array_set(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ array_ref: ir::Value,
+ index: ir::Value,
+ value: ir::Value,
+) -> WasmResult<()> {
+ log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");
+
+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
+
+ let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
+ let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;
+
+ let array_ty = func_env.types.unwrap_array(array_type_index)?;
+ let elem_ty = array_ty.0.element_type;
+
+ write_field_at_addr(func_env, builder, elem_ty, elem_addr, value)?;
+
+ log::trace!("translate_array_set: finished");
+ Ok(())
+}
+
+pub fn translate_ref_test(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ test_ty: WasmRefType,
+ val: ir::Value,
+ val_ty: WasmRefType,
+) -> WasmResult<ir::Value> {
+ log::trace!("translate_ref_test({test_ty:?}, {val:?})");
+
+ // First special case: testing for references to bottom types.
+ if test_ty.heap_type.is_bottom() {
+ let result = if test_ty.nullable {
+ // All null references (within the same type hierarchy) match null
+ // references to the bottom type.
+ func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?
+ } else {
+ // `ref.test` is always false for non-nullable bottom types, as the
+ // bottom types are uninhabited.
+ builder.ins().iconst(ir::types::I32, 0)
+ };
+ log::trace!("translate_ref_test(..) -> {result:?}");
+ return Ok(result);
+ }
+
+ // And because `ref.test heap_ty` is only valid on operands whose type is in
+ // the same type hierarchy as `heap_ty`, if `heap_ty` is its hierarchy's top
+ // type, we only need to worry about whether we are testing for nullability
+ // or not.
+ if test_ty.heap_type.is_top() {
+ let result = if test_ty.nullable {
+ builder.ins().iconst(ir::types::I32, 1)
+ } else {
+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+ let one = builder.ins().iconst(ir::types::I32, 1);
+ builder.ins().select(is_null, zero, one)
+ };
+ log::trace!("translate_ref_test(..) -> {result:?}");
+ return Ok(result);
+ }
+
+ // `i31ref`s are a little interesting because they don't point to GC
+ // objects; we test the bit pattern of the reference itself.
+ if test_ty.heap_type == WasmHeapType::I31 {
+ let i31_mask = builder.ins().iconst(
+ ir::types::I32,
+ i64::from(wasmtime_environ::I31_DISCRIMINANT),
+ );
+ let is_i31 = builder.ins().band(val, i31_mask);
+ let result = if test_ty.nullable {
+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
+ builder.ins().bor(is_null, is_i31)
+ } else {
+ is_i31
+ };
+ log::trace!("translate_ref_test(..) -> {result:?}");
+ return Ok(result);
+ }
+
+ // Otherwise, in the general case, we need to inspect our given object's
+ // actual type, which also requires null-checking and i31-checking it.
+
+ let is_any_hierarchy = test_ty.heap_type.top() == WasmHeapTopType::Any;
+
+ let non_null_block = builder.create_block();
+ let non_null_non_i31_block = builder.create_block();
+ let continue_block = builder.create_block();
+
+ // Current block: check if the reference is null and branch appropriately.
+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
+ let result_when_is_null = builder
+ .ins()
+ .iconst(ir::types::I32, test_ty.nullable as i64);
+ builder.ins().brif(
+ is_null,
+ continue_block,
+ &[result_when_is_null.into()],
+ non_null_block,
+ &[],
+ );
+
+ // Non-null block: We know the GC ref is non-null, but we need to also check
+ // for `i31` references that don't point to GC objects.
+ builder.switch_to_block(non_null_block);
+ log::trace!("translate_ref_test: non-null ref block");
+ if is_any_hierarchy {
+ let i31_mask = builder.ins().iconst(
+ ir::types::I32,
+ i64::from(wasmtime_environ::I31_DISCRIMINANT),
+ );
+ let is_i31 = builder.ins().band(val, i31_mask);
+ // If it is an `i31`, then create the result value based on whether we
+ // want `i31`s to pass the test or not.
+ let result_when_is_i31 = builder.ins().iconst(
+ ir::types::I32,
+ matches!(
+ test_ty.heap_type,
+ WasmHeapType::Any | WasmHeapType::Eq | WasmHeapType::I31
+ ) as i64,
+ );
+ builder.ins().brif(
+ is_i31,
+ continue_block,
+ &[result_when_is_i31.into()],
+ non_null_non_i31_block,
+ &[],
+ );
+ } else {
+ // If we aren't testing the `any` hierarchy, the reference cannot be an
+ // `i31ref`. Jump directly to the non-null and non-i31 block; rely on
+ // branch folding during lowering to clean this up.
+ builder.ins().jump(non_null_non_i31_block, &[]);
+ }
+
+ // Non-null and non-i31 block: Read the actual `VMGcKind` or
+ // `VMSharedTypeIndex` out of the object's header and check whether it
+ // matches the expected type.
+ builder.switch_to_block(non_null_non_i31_block);
+ log::trace!("translate_ref_test: non-null and non-i31 ref block");
+ let check_header_kind = |func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ val: ir::Value,
+ expected_kind: VMGcKind|
+ -> ir::Value {
+ let kind_addr = func_env.prepare_gc_ref_access(
+ builder,
+ val,
+ BoundsCheck::StaticObjectField {
+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,
+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,
+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,
+ },
+ );
+ let gc_memflags = func_env.gc_memflags(&mut builder.func);
+ let actual_kind =
+ builder
+ .ins()
+ .load(ir::types::I32, gc_memflags.with_readonly(), kind_addr, 0);
+ let expected_kind = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));
+ // Inline version of `VMGcKind::matches`.
+ let and = builder.ins().band(actual_kind, expected_kind);
+ let kind_matches = builder
+ .ins()
+ .icmp(ir::condcodes::IntCC::Equal, and, expected_kind);
+ builder.ins().uextend(ir::types::I32, kind_matches)
+ };
+ let result = match test_ty.heap_type {
+ WasmHeapType::Any
+ | WasmHeapType::None
+ | WasmHeapType::Extern
+ | WasmHeapType::NoExtern
+ | WasmHeapType::Func
+ | WasmHeapType::NoFunc
+ | WasmHeapType::Cont
+ | WasmHeapType::NoCont
+ | WasmHeapType::Exn
+ | WasmHeapType::NoExn
+ | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"),
+
+ // For these abstract but non-top and non-bottom types, we check the
+ // `VMGcKind` that is in the object's header.
+ WasmHeapType::Eq => check_header_kind(func_env, builder, val, VMGcKind::EqRef),
+ WasmHeapType::Struct => check_header_kind(func_env, builder, val, VMGcKind::StructRef),
+ WasmHeapType::Array => check_header_kind(func_env, builder, val, VMGcKind::ArrayRef),
+
+ // For concrete types, we need to do a full subtype check between the
+ // `VMSharedTypeIndex` in the object's header and the
+ // `ModuleInternedTypeIndex` we have here.
+ //
+ // TODO: This check should ideally be done inline, but we don't have a
+ // good way to access the `TypeRegistry`'s supertypes arrays from Wasm
+ // code at the moment.
+ WasmHeapType::ConcreteArray(ty)
+ | WasmHeapType::ConcreteStruct(ty)
+ | WasmHeapType::ConcreteExn(ty) => {
+ let expected_interned_ty = ty.unwrap_module_type_index();
+ let expected_shared_ty =
+ func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
+
+ let ty_addr = func_env.prepare_gc_ref_access(
+ builder,
+ val,
+ BoundsCheck::StaticOffset {
+ offset: wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET,
+ access_size: func_env.offsets.size_of_vmshared_type_index(),
+ },
+ );
+ let gc_memflags = func_env.gc_memflags(&mut builder.func);
+ let actual_shared_ty =
+ builder
+ .ins()
+ .load(ir::types::I32, gc_memflags.with_readonly(), ty_addr, 0);
+
+ func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)
+ }
+
+ // Same as for concrete arrays and structs except that a `VMFuncRef`
+ // doesn't begin with a `VMGcHeader` and is a raw pointer rather than GC
+ // heap index.
+ WasmHeapType::ConcreteFunc(ty) => {
+ let expected_interned_ty = ty.unwrap_module_type_index();
+ let expected_shared_ty =
+ func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
+
+ let gc_memflags = func_env.gc_memflags(&mut builder.func);
+ let actual_shared_ty = func_env.load_funcref_type_index(
+ &mut builder.cursor(),
+ gc_memflags.with_readonly(),
+ val,
+ );
+
+ func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)
+ }
+ WasmHeapType::ConcreteCont(_) => {
+ // TODO(#10248) GC integration for stack switching
+ return stack_switching_unsupported();
+ }
+ };
+ builder.ins().jump(continue_block, &[result.into()]);
+
+ // Control flow join point with the result.
+ builder.switch_to_block(continue_block);
+ let result = builder.append_block_param(continue_block, ir::types::I32);
+ log::trace!("translate_ref_test(..) -> {result:?}");
+
+ builder.seal_block(non_null_block);
+ builder.seal_block(non_null_non_i31_block);
+ builder.seal_block(continue_block);
+
+ Ok(result)
+}
+
+fn uextend_i32_to_pointer_type(
+ builder: &mut FunctionBuilder,
+ pointer_type: ir::Type,
+ value: ir::Value,
+) -> ir::Value {
+ assert_eq!(builder.func.dfg.value_type(value), ir::types::I32);
+ match pointer_type {
+ ir::types::I32 => value,
+ ir::types::I64 => builder.ins().uextend(ir::types::I64, value),
+ _ => unreachable!(),
+ }
+}
+
+/// Emit CLIF to compute an array object's total size, given the dynamic length
+/// in its initialization.
+///
+/// Traps if the size overflows.
+fn emit_array_size(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ array_layout: &GcArrayLayout,
+ len: ir::Value,
+) -> ir::Value {
+ let base_size = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(array_layout.base_size));
+
+ // `elems_size = len * elem_size`
+ //
+ // Check for multiplication overflow and trap if it occurs, since that
+ // means Wasm is attempting to allocate an array that is larger than our
+ // implementation limits. (Note: there is no standard implementation
+ // limit for array length beyond `u32::MAX`.)
+ //
+ // We implement this check by encoding our logically-32-bit operands as
+ // i64 values, doing a 64-bit multiplication, and then checking the high
+ // 32 bits of the multiplication's result. If the high 32 bits are not
+ // all zeros, then the multiplication overflowed.
+ debug_assert_eq!(builder.func.dfg.value_type(len), ir::types::I32);
+ let len = builder.ins().uextend(ir::types::I64, len);
+ let elems_size_64 = builder
+ .ins()
+ .imul_imm(len, i64::from(array_layout.elem_size));
+ let high_bits = builder.ins().ushr_imm(elems_size_64, 32);
+ func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE);
+ let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64);
+
+ // And if adding the base size and elements size overflows, then the
+ // allocation is too large.
+ let size = func_env.uadd_overflow_trap(
+ builder,
+ base_size,
+ elems_size,
+ crate::TRAP_ALLOCATION_TOO_LARGE,
+ );
+
+ size
+}
+
+/// Common helper for struct-field initialization that can be reused across
+/// collectors.
+fn initialize_struct_fields(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ struct_ty: ModuleInternedTypeIndex,
+ raw_ptr_to_struct: ir::Value,
+ field_values: &[ir::Value],
+) -> WasmResult<()> {
+ let struct_layout = func_env.struct_or_exn_layout(struct_ty);
+ let struct_size = struct_layout.size;
+ let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().map(|f| f.offset).collect();
+ assert_eq!(field_offsets.len(), field_values.len());
+
+ assert!(!func_env.types[struct_ty].composite_type.shared);
+ let fields = match &func_env.types[struct_ty].composite_type.inner {
+ WasmCompositeInnerType::Struct(s) => &s.fields,
+ WasmCompositeInnerType::Exn(e) => &e.fields,
+ _ => panic!("Not a struct or exception type"),
+ };
+
+ let field_types: SmallVec<[_; 8]> = fields.iter().cloned().collect();
+ assert_eq!(field_types.len(), field_values.len());
+
+ for ((ty, val), offset) in field_types.into_iter().zip(field_values).zip(field_offsets) {
+ let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty.element_type);
+ assert!(offset + size_of_access <= struct_size);
+ let field_addr = builder.ins().iadd_imm(raw_ptr_to_struct, i64::from(offset));
+ gc_compiler(func_env)?.init_field(func_env, builder, ty.element_type, field_addr, *val)?;
+ }
+
+ Ok(())
+}
+
+impl FuncEnvironment<'_> {
+ pub(crate) fn gc_heap_alias_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.alias_region(func, AliasRegionKey::GcHeap)
+ }
+
+ /// Flags to use for general-purpose GC loads/stores.
+ ///
+ /// This is used for accesses to the GC heap which aren't expected to trap, but
+ /// retain internal assertion metadata to report if such a trap happens. This
+ /// is here to ensure that in the face of heap corruption that there's no
+ /// possible UB within Cranelift and/or the runtime.
+ fn gc_memflags(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {
+ let region = self.gc_heap_alias_region(func);
+ ir::MemFlagsData::new()
+ .with_trap_code(Some(crate::TRAP_GC_HEAP_CORRUPT))
+ .with_alias_region(Some(region))
+ }
+
+ fn gc_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcLayout {
+ // Lazily compute and cache the layout.
+ if !self.ty_to_gc_layout.contains_key(&type_index) {
+ let ty = &self.types[type_index].composite_type;
+ let layout = gc_compiler(self)
+ .unwrap()
+ .layouts()
+ .gc_layout(ty)
+ .expect("gc_layout should not OOM at compile time")
+ .expect("should only call `FuncEnvironment::gc_layout` for GC types");
+ self.ty_to_gc_layout.insert(type_index, layout);
+ }
+
+ self.ty_to_gc_layout.get(&type_index).unwrap()
+ }
+
+ /// Get the `GcArrayLayout` for the array type at the given `type_index`.
+ pub(crate) fn array_layout(
+ &mut self,
+ type_index: ModuleInternedTypeIndex,
+ ) -> WasmResult<&GcArrayLayout> {
+ Ok(self.gc_layout(type_index).unwrap_array())
+ }
+
+ /// Get the `GcStructLayout` for the struct or exception type at the given `type_index`.
+ fn struct_or_exn_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcStructLayout {
+ let result = self.gc_layout(type_index).unwrap_struct();
+ result
+ }
+
+ /// Get or create the global for our GC heap's base pointer.
+ fn get_gc_heap_base_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {
+ if let Some(base) = self.gc_heap_base {
+ return base;
+ }
+
+ let store_context_ptr = self.get_vmstore_context_ptr_global(func);
+ let offset = self.offsets.ptr.vmstore_context_gc_heap_base();
+
+ let mut flags = ir::MemFlagsData::trusted();
+ let memory_tunables =
+ wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);
+ if !self
+ .tunables
+ .gc_heap_memory_type()
+ .memory_may_move(&memory_tunables)
+ {
+ flags.set_readonly();
+ flags.set_can_move();
+ }
+
+ let base_flags = func.dfg.mem_flags.insert(flags).unwrap();
+ let base = func.create_global_value(ir::GlobalValueData::Load {
+ base: store_context_ptr,
+ offset: Offset32::new(offset.into()),
+ global_type: self.pointer_type(),
+ flags: base_flags,
+ });
+
+ self.gc_heap_base = Some(base);
+ base
+ }
+
+ /// Get the GC heap's base.
+ pub(crate) fn get_gc_heap_base(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ ) -> WasmResult<ir::Value> {
+ let global = self.get_gc_heap_base_global(&mut builder.func);
+ Ok(builder.ins().global_value(self.pointer_type(), global))
+ }
+
+ fn get_gc_heap_bound_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {
+ if let Some(bound) = self.gc_heap_bound {
+ return bound;
+ }
+ let store_context_ptr = self.get_vmstore_context_ptr_global(func);
+ let offset = self.offsets.ptr.vmstore_context_gc_heap_current_length();
+ let bound_flags = func
+ .dfg
+ .mem_flags
+ .insert(ir::MemFlagsData::trusted())
+ .unwrap();
+ let bound = func.create_global_value(ir::GlobalValueData::Load {
+ base: store_context_ptr,
+ offset: Offset32::new(offset.into()),
+ global_type: self.pointer_type(),
+ flags: bound_flags,
+ });
+ self.gc_heap_bound = Some(bound);
+ bound
+ }
+
+ /// Get the GC heap's bound.
+ pub(crate) fn get_gc_heap_bound(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ ) -> WasmResult<ir::Value> {
+ let global = self.get_gc_heap_bound_global(&mut builder.func);
+ Ok(builder.ins().global_value(self.pointer_type(), global))
+ }
+
+ /// Get or create the `Heap` for our GC heap.
+ fn get_gc_heap(&mut self, func: &mut ir::Function) -> Heap {
+ if let Some(heap) = self.gc_heap {
+ return heap;
+ }
+
+ let base = self.get_gc_heap_base_global(func);
+ let bound = self.get_gc_heap_bound_global(func);
+ let memory = self.tunables.gc_heap_memory_type();
+ let heap = self.heaps.push(HeapData {
+ base,
+ bound,
+ memory,
+ kind: MemoryKind::GcHeap,
+ });
+ self.gc_heap = Some(heap);
+ heap
+ }
+
+ /// Get the raw pointer of `gc_ref[offset]` bounds checked for an access of
+ /// `size` bytes.
+ ///
+ /// The given `gc_ref` must be a non-null, non-i31 GC reference.
+ ///
+ /// If `check` is a `BoundsCheck::Object`, then it is the callers
+ /// responsibility to ensure that `offset + access_size <= object_size`.
+ ///
+ /// Returns a raw pointer to `gc_ref[offset]` -- not a raw pointer to the GC
+ /// object itself (unless `offset` happens to be `0`). This raw pointer may
+ /// be used to read or write up to as many bytes as described by `bound`. Do
+ /// NOT attempt accesses bytes outside of `bound`; that may lead to
+ /// unchecked out-of-bounds accesses.
+ ///
+ /// This method is collector-agnostic.
+ fn prepare_gc_ref_access(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ gc_ref: ir::Value,
+ bounds_check: BoundsCheck,
+ ) -> ir::Value {
+ log::trace!("prepare_gc_ref_access({gc_ref:?}, {bounds_check:?})");
+ assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);
+
+ let gc_heap = self.get_gc_heap(&mut builder.func);
+ let gc_heap = self.heaps[gc_heap].clone();
+ let result = match crate::bounds_checks::bounds_check_and_compute_addr(
+ builder,
+ self,
+ &gc_heap,
+ gc_ref,
+ bounds_check,
+ crate::TRAP_GC_HEAP_CORRUPT,
+ ) {
+ Reachability::Reachable(v) => v,
+ Reachability::Unreachable => {
+ // We are now in unreachable code, but we don't want to plumb
+ // through a bunch of `Reachability` through all of our callers,
+ // so just assert we won't reach here and return `null`
+ let null = builder.ins().iconst(self.pointer_type(), 0);
+ builder.ins().trapz(null, crate::TRAP_INTERNAL_ASSERT);
+ null
+ }
+ };
+ log::trace!("prepare_gc_ref_access(..) -> {result:?}");
+ result
+ }
+
+ /// Emit checks (if necessary) for whether the given `gc_ref` is null or is
+ /// an `i31ref`.
+ ///
+ /// Takes advantage of static information based on `ty` as to whether the GC
+ /// reference is nullable or can ever be an `i31`.
+ ///
+ /// Returns an `ir::Value` that is an `i32` will be non-zero if the GC
+ /// reference is null or is an `i31ref`; otherwise, it will be zero.
+ ///
+ /// This method is collector-agnostic.
+ fn gc_ref_is_null_or_i31(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ ty: WasmRefType,
+ gc_ref: ir::Value,
+ ) -> ir::Value {
+ assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);
+ assert!(ty.is_vmgcref_type_and_not_i31());
+
+ let might_be_i31 = match ty.heap_type {
+ // If we are definitely dealing with an i31, we shouldn't be
+ // emitting dynamic checks for it, and the caller shouldn't call
+ // this function. Should have been caught by the assertion at the
+ // start of the function.
+ WasmHeapType::I31 => unreachable!(),
+
+ // Could potentially be an i31.
+ WasmHeapType::Any | WasmHeapType::Eq => true,
+
+ // If it is definitely a struct, array, or uninhabited type, then it
+ // is definitely not an i31.
+ WasmHeapType::Array
+ | WasmHeapType::ConcreteArray(_)
+ | WasmHeapType::Struct
+ | WasmHeapType::ConcreteStruct(_)
+ | WasmHeapType::None => false,
+
+ // Despite being a different type hierarchy, this *could* be an
+ // `i31` if it is the result of
+ //
+ // (extern.convert_any (ref.i31 ...))
+ WasmHeapType::Extern => true,
+
+ // Can only ever be `null`.
+ WasmHeapType::NoExtern => false,
+
+ WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn => false,
+
+ // Wrong type hierarchy, and also funcrefs are not GC-managed
+ // types. Should have been caught by the assertion at the start of
+ // the function.
+ WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => {
+ unreachable!()
+ }
+ WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => {
+ unreachable!()
+ }
+ };
+
+ match (ty.nullable, might_be_i31) {
+ // This GC reference statically cannot be null nor an i31. (Let
+ // Cranelift's optimizer const-propagate this value and erase any
+ // unnecessary control flow resulting from branching on this value.)
+ (false, false) => builder.ins().iconst(ir::types::I32, 0),
+
+ // This GC reference is always non-null, but might be an i31.
+ (false, true) => builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT)),
+
+ // This GC reference might be null, but can never be an i31.
+ (true, false) => builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0),
+
+ // Fully general case: this GC reference could be either null or an
+ // i31.
+ (true, true) => {
+ let is_i31 = builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT));
+ let is_null = builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0);
+ let is_null = builder.ins().uextend(ir::types::I32, is_null);
+ builder.ins().bor(is_i31, is_null)
+ }
+ }
+ }
+
+ // Emit code to check whether `a <: b` for two `VMSharedTypeIndex`es.
+ pub(crate) fn is_subtype(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ a: ir::Value,
+ b: ir::Value,
+ ) -> ir::Value {
+ log::trace!("is_subtype({a:?}, {b:?})");
+
+ let diff_tys_block = builder.create_block();
+ let continue_block = builder.create_block();
+
+ // Current block: fast path for when `a == b`.
+ log::trace!("is_subtype: fast path check for exact same types");
+ let same_ty = builder.ins().icmp(IntCC::Equal, a, b);
+ let same_ty = builder.ins().uextend(ir::types::I32, same_ty);
+ builder.ins().brif(
+ same_ty,
+ continue_block,
+ &[same_ty.into()],
+ diff_tys_block,
+ &[],
+ );
+
+ // Different types block: fall back to the `is_subtype` libcall.
+ builder.switch_to_block(diff_tys_block);
+ log::trace!("is_subtype: slow path to do full `is_subtype` libcall");
+ let is_subtype = self.builtin_functions.is_subtype(builder.func);
+ let vmctx = self.vmctx_val(&mut builder.cursor());
+ let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]);
+ let result = builder.func.dfg.first_result(call_inst);
+ builder.ins().jump(continue_block, &[result.into()]);
+
+ // Continue block: join point for the result.
+ builder.switch_to_block(continue_block);
+ let result = builder.append_block_param(continue_block, ir::types::I32);
+ log::trace!("is_subtype(..) -> {result:?}");
+
+ builder.seal_block(diff_tys_block);
+ builder.seal_block(continue_block);
+
+ result
+ }
+}
+
+fn stack_switching_unsupported<T>() -> WasmResult<T> {
+ Err(wasmtime_environ::WasmError::Unsupported(
+ "Stack switching feature not compatible with GC, yet".to_string(),
+ ))
+}
+
+pub fn translate_array_new_entity(
+ env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ array_type_index: TypeIndex,
+ entity: CheckedEntity,
+ entity_offset: ir::Value,
+ len: ir::Value,
+) -> WasmResult<ir::Value> {
+ // Before actually allocating this array first do a bounds-check on the
+ // passive entity itself.
+ let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();
+ env.translate_entity_bounds_check(builder, entity, entity_offset, len)?;
+
+ let array = gc_compiler(env)?.alloc_uninit_array(env, builder, array_type_index, len)?;
+ let dst = builder.ins().iconst(ir::types::I32, 0);
+ env.translate_entity_copy(
+ builder,
+ CheckedEntity::Array {
+ array,
+ ty: interned_type_index,
+ initialized: false,
+ },
+ entity,
+ dst,
+ entity_offset,
+ len,
+ )?;
+
+ Ok(array)
+}
diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs
similarity index 100%
rename from crates/cranelift/src/func_environ/gc/enabled/copying.rs
rename to crates/cranelift/src/func_environ/gc/copying.rs
diff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rs
deleted file mode 100644
index e659ff6f74..0000000000
--- a/crates/cranelift/src/func_environ/gc/disabled.rs
+++ /dev/null
@@ -1,229 +0,0 @@
-//! `GcCompiler` implementation when GC support is disabled.
-
-use super::GcCompiler;
-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};
-use cranelift_codegen::ir;
-use cranelift_frontend::FunctionBuilder;
-use smallvec::SmallVec;
-use wasmtime_environ::{
- GcArrayLayout, ModuleInternedTypeIndex, TagIndex, TypeIndex, WasmRefType, WasmResult,
- WasmStorageType, wasm_unsupported,
-};
-
-fn disabled<T>() -> WasmResult<T> {
- Err(wasm_unsupported!(
- "support for Wasm GC disabled at compile time because the `gc` cargo \
- feature was not enabled"
- ))
-}
-
-/// Get the default GC compiler.
-pub fn gc_compiler(_: &FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {
- disabled()
-}
-
-pub fn translate_struct_new(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _struct_type_index: TypeIndex,
- _fields: &[ir::Value],
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_struct_new_default(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _struct_type_index: TypeIndex,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_struct_get(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _struct_type_index: TypeIndex,
- _field_index: u32,
- _struct_ref: ir::Value,
- _extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_struct_set(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _struct_type_index: TypeIndex,
- _field_index: u32,
- _struct_ref: ir::Value,
- _new_val: ir::Value,
-) -> WasmResult<()> {
- disabled()
-}
-
-pub fn translate_exn_unbox(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _tag_index: TagIndex,
- _exn_ref: ir::Value,
-) -> WasmResult<SmallVec<[ir::Value; 4]>> {
- disabled()
-}
-
-pub fn translate_exn_throw(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _tag_index: TagIndex,
- _args: &[ir::Value],
-) -> WasmResult<()> {
- disabled()
-}
-
-pub fn translate_exn_throw_ref(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _exnref: ir::Value,
-) -> WasmResult<()> {
- disabled()
-}
-
-pub fn translate_array_new(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _elem: ir::Value,
- _len: ir::Value,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_new_default(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _len: ir::Value,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_new_fixed(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _elems: &[ir::Value],
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_len(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array: ir::Value,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_get(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _array: ir::Value,
- _index: ir::Value,
- _extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_set(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _array: ir::Value,
- _index: ir::Value,
- _value: ir::Value,
-) -> WasmResult<()> {
- disabled()
-}
-
-pub fn translate_ref_test(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _test_ty: WasmRefType,
- _val: ir::Value,
- _val_ty: WasmRefType,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn translate_array_new_entity(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder,
- _array_type_index: TypeIndex,
- _entity: CheckedEntity,
- _data_offset: ir::Value,
- _len: ir::Value,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn read_field_at_addr(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _ty: WasmStorageType,
- _addr: ir::Value,
- _extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- disabled()
-}
-
-pub fn write_field_at_addr(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _field_ty: WasmStorageType,
- _field_addr: ir::Value,
- _new_val: ir::Value,
-) -> WasmResult<()> {
- disabled()
-}
-
-pub fn init_field_at_addr(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _field_ty: WasmStorageType,
- _field_addr: ir::Value,
- _new_val: ir::Value,
-) -> WasmResult<()> {
- disabled()
-}
-
-impl FuncEnvironment<'_> {
- pub(crate) fn array_layout(
- &mut self,
- _type_index: ModuleInternedTypeIndex,
- ) -> WasmResult<&GcArrayLayout> {
- disabled()
- }
-
- pub(crate) fn get_gc_heap_base(
- &mut self,
- _builder: &mut FunctionBuilder<'_>,
- ) -> WasmResult<ir::Value> {
- disabled()
- }
-
- pub(crate) fn get_gc_heap_bound(
- &mut self,
- _builder: &mut FunctionBuilder<'_>,
- ) -> WasmResult<ir::Value> {
- disabled()
- }
-}
-
-pub fn intern_func_ref(
- _func_env: &mut FuncEnvironment<'_>,
- _builder: &mut FunctionBuilder<'_>,
- _ref_type: WasmRefType,
- _func_ref: ir::Value,
-) -> WasmResult<ir::Value> {
- disabled()
-}
diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs
similarity index 100%
rename from crates/cranelift/src/func_environ/gc/enabled/drc.rs
rename to crates/cranelift/src/func_environ/gc/drc.rs
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs
deleted file mode 100644
index 68a6cc33a0..0000000000
--- a/crates/cranelift/src/func_environ/gc/enabled.rs
+++ /dev/null
@@ -1,1620 +0,0 @@
-use super::GcCompiler;
-use crate::TRAP_ARRAY_OUT_OF_BOUNDS;
-use crate::alias_region_key::AliasRegionKey;
-use crate::bounds_checks::BoundsCheck;
-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};
-use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};
-use crate::trap::TranslateTrap;
-use crate::{Reachability, TRAP_GC_HEAP_CORRUPT, TRAP_INTERNAL_ASSERT};
-use cranelift_codegen::ir::immediates::Offset32;
-use cranelift_codegen::ir::{BlockArg, ExceptionTableData, ExceptionTableItem};
-use cranelift_codegen::{
- cursor::FuncCursor,
- ir::{self, InstBuilder, condcodes::IntCC},
-};
-use cranelift_entity::packed_option::ReservedValue;
-use cranelift_frontend::FunctionBuilder;
-use smallvec::{SmallVec, smallvec};
-use wasmtime_environ::{
- Collector, GcArrayLayout, GcLayout, GcStructLayout, I31_DISCRIMINANT, ModuleInternedTypeIndex,
- PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType, WasmHeapTopType, WasmHeapType,
- WasmRefType, WasmResult, WasmStorageType, WasmValType, wasm_unsupported,
-};
-
-#[cfg(feature = "gc-drc")]
-mod drc;
-#[cfg(feature = "gc-null")]
-mod null;
-
-#[cfg(feature = "gc-copying")]
-mod copying;
-
-/// Get the default GC compiler.
-pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {
- // If this function requires a GC compiler, that is not too bad of an
- // over-approximation for it requiring a GC heap.
- func_env.needs_gc_heap = true;
-
- match func_env.tunables.collector {
- #[cfg(feature = "gc-drc")]
- Some(Collector::DeferredReferenceCounting) => Ok(Box::new(drc::DrcCompiler::default())),
- #[cfg(not(feature = "gc-drc"))]
- Some(Collector::DeferredReferenceCounting) => Err(wasm_unsupported!(
- "the DRC collector is unavailable because the `gc-drc` feature \
- was disabled at compile time",
- )),
-
- #[cfg(feature = "gc-null")]
- Some(Collector::Null) => Ok(Box::new(null::NullCompiler::default())),
- #[cfg(not(feature = "gc-null"))]
- Some(Collector::Null) => Err(wasm_unsupported!(
- "the null collector is unavailable because the `gc-null` feature \
- was disabled at compile time",
- )),
-
- #[cfg(feature = "gc-copying")]
- Some(Collector::Copying) => Ok(Box::new(copying::CopyingCompiler::default())),
- #[cfg(not(feature = "gc-copying"))]
- Some(Collector::Copying) => Err(wasm_unsupported!(
- "the copying collector is unavailable because the `gc-copying` feature \
- was disabled at compile time",
- )),
-
- #[cfg(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying"))]
- None => Err(wasm_unsupported!(
- "support for GC types disabled at configuration time"
- )),
- #[cfg(not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")))]
- None => Err(wasm_unsupported!(
- "support for GC types disabled because no collector implementation \
- was selected at compile time; enable one of the `gc-drc` or \
- `gc-null` features",
- )),
- }
-}
-
-#[cfg_attr(
- not(any(feature = "gc-drc", feature = "gc-copying")),
- expect(dead_code, reason = "easier to define")
-)]
-fn unbarriered_load_gc_ref(
- builder: &mut FunctionBuilder,
- ty: WasmHeapType,
- ptr_to_gc_ref: ir::Value,
- flags: ir::MemFlagsData,
-) -> WasmResult<ir::Value> {
- debug_assert!(ty.is_vmgcref_type());
- let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);
- if ty != WasmHeapType::I31 {
- builder.declare_value_needs_stack_map(gc_ref);
- }
- Ok(gc_ref)
-}
-
-#[cfg_attr(
- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
- expect(dead_code, reason = "easier to define")
-)]
-fn unbarriered_store_gc_ref(
- builder: &mut FunctionBuilder,
- ty: WasmHeapType,
- dst: ir::Value,
- gc_ref: ir::Value,
- flags: ir::MemFlagsData,
-) -> WasmResult<()> {
- debug_assert!(ty.is_vmgcref_type());
- builder.ins().store(flags, gc_ref, dst, 0);
- Ok(())
-}
-
-/// Emit CLIF to call the `gc_alloc_raw` libcall.
-#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]
-fn emit_gc_raw_alloc(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- kind: VMGcKind,
- ty: ModuleInternedTypeIndex,
- size: ir::Value,
- align: u32,
- reserved_bits: u32,
-) -> ir::Value {
- let gc_alloc_raw_builtin = func_env.builtin_functions.gc_alloc_raw(builder.func);
- let vmctx = func_env.vmctx_val(&mut builder.cursor());
-
- let kind = builder
- .ins()
- .iconst(ir::types::I32, i64::from(kind.as_u32() | reserved_bits));
-
- let ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);
-
- assert!(align.is_power_of_two());
- let align = builder.ins().iconst(ir::types::I32, i64::from(align));
-
- let call_inst = builder
- .ins()
- .call(gc_alloc_raw_builtin, &[vmctx, kind, ty, size, align]);
-
- let gc_ref = builder.func.dfg.first_result(call_inst);
- builder.declare_value_needs_stack_map(gc_ref);
- gc_ref
-}
-
-/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the
-/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.
-///
-/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).
-fn emit_gc_kind_assert(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- gc_ref: ir::Value,
- expected_kind: VMGcKind,
-) {
- if !cfg!(gc_zeal) {
- return;
- }
-
- func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);
-
- let kind_addr = func_env.prepare_gc_ref_access(
- builder,
- gc_ref,
- BoundsCheck::StaticObjectField {
- offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,
- access_size: wasmtime_environ::VM_GC_KIND_SIZE,
- object_size: wasmtime_environ::VM_GC_HEADER_SIZE,
- },
- );
- let flags = func_env.gc_memflags(&mut builder.func).with_readonly();
- let kind_and_reserved_bits = builder.ins().load(ir::types::I32, flags, kind_addr, 0);
- let kind_mask = builder
- .ins()
- .iconst(ir::types::I32, i64::from(VMGcKind::MASK));
- let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);
-
- let expected_kind = builder
- .ins()
- .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));
-
- // NB: Do a subtype check rather than a strict equality check. See
- // `VMGcKind::matches` for details.
- let and = builder.ins().band(actual_kind, expected_kind);
- let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);
-
- builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);
-}
-
-/// Read a struct field or array element from its raw address in the GC heap.
-///
-/// The given address MUST have already been bounds-checked via
-/// `prepare_gc_ref_access`.
-pub fn read_field_at_addr(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- ty: WasmStorageType,
- addr: ir::Value,
- extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- assert_eq!(extension.is_none(), matches!(ty, WasmStorageType::Val(_)));
- assert_eq!(
- extension.is_some(),
- matches!(ty, WasmStorageType::I8 | WasmStorageType::I16)
- );
-
- // Data inside GC objects is always little endian.
- let flags = func_env
- .gc_memflags(&mut builder.func)
- .with_endianness(ir::Endianness::Little);
-
- let value = match ty {
- WasmStorageType::I8 => builder.ins().load(ir::types::I8, flags, addr, 0),
- WasmStorageType::I16 => builder.ins().load(ir::types::I16, flags, addr, 0),
- WasmStorageType::Val(v) => match v {
- WasmValType::I32 => builder.ins().load(ir::types::I32, flags, addr, 0),
- WasmValType::I64 => builder.ins().load(ir::types::I64, flags, addr, 0),
- WasmValType::F32 => builder.ins().load(ir::types::F32, flags, addr, 0),
- WasmValType::F64 => builder.ins().load(ir::types::F64, flags, addr, 0),
- WasmValType::V128 => builder.ins().load(ir::types::I8X16, flags, addr, 0),
- WasmValType::Ref(r) => match r.heap_type.top() {
- WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
- gc_compiler(func_env)?
- .translate_read_gc_reference(func_env, builder, r, addr, flags)?
- }
- WasmHeapTopType::Func => {
- let expected_ty = match r.heap_type {
- WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),
- WasmHeapType::ConcreteFunc(ty) => ty.unwrap_module_type_index(),
- WasmHeapType::NoFunc => {
- let null = builder.ins().iconst(func_env.pointer_type(), 0);
- if !r.nullable {
- // Because `nofunc` is uninhabited, and this
- // reference is non-null, this is unreachable
- // code. Unconditionally trap via conditional
- // trap instructions to avoid inserting block
- // terminators in the middle of this block.
- builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);
- }
- return Ok(null);
- }
- _ => unreachable!("not a function heap type"),
- };
- let expected_ty = builder
- .ins()
- .iconst(ir::types::I32, i64::from(expected_ty.as_bits()));
-
- let vmctx = func_env.vmctx_val(&mut builder.cursor());
-
- let func_ref_id = builder.ins().load(ir::types::I32, flags, addr, 0);
- let get_interned_func_ref = func_env
- .builtin_functions
- .get_interned_func_ref(builder.func);
-
- let call_inst = builder
- .ins()
- .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);
- builder.func.dfg.first_result(call_inst)
- }
- WasmHeapTopType::Cont => {
- // TODO(#10248) GC integration for stack switching
- return stack_switching_unsupported();
- }
- },
- },
- };
-
- let value = match extension {
- Some(Extension::Sign) => builder.ins().sextend(ir::types::I32, value),
- Some(Extension::Zero) => builder.ins().uextend(ir::types::I32, value),
- None => value,
- };
-
- Ok(value)
-}
-
-pub fn intern_func_ref(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- ref_type: WasmRefType,
- func_ref: ir::Value,
-) -> WasmResult<ir::Value> {
- assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);
-
- let vmctx = func_env.vmctx_val(&mut builder.cursor());
-
- let intern_func_ref_for_gc_heap = func_env
- .builtin_functions
- .intern_func_ref_for_gc_heap(builder.func);
-
- let func_ref = if ref_type.heap_type == WasmHeapType::NoFunc {
- let null = builder.ins().iconst(func_env.pointer_type(), 0);
- if !ref_type.nullable {
- // Because `nofunc` is uninhabited, and this reference is
- // non-null, this is unreachable code. Unconditionally trap
- // via conditional trap instructions to avoid inserting
- // block terminators in the middle of this block.
- builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);
- }
- null
- } else {
- func_ref
- };
-
- // Convert the raw `funcref` into a `FuncRefTableId` for use in the
- // GC heap.
- let call_inst = builder
- .ins()
- .call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);
- let func_ref_id = builder.func.dfg.first_result(call_inst);
- Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))
-}
-
-fn write_func_ref_at_addr(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- ref_type: WasmRefType,
- flags: ir::MemFlagsData,
- field_addr: ir::Value,
- func_ref: ir::Value,
-) -> WasmResult<()> {
- // Convert the raw `funcref` into a `FuncRefTableId` for use in the
- // GC heap.
- let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;
-
- // Store the id in the field.
- builder.ins().store(flags, func_ref_id, field_addr, 0);
-
- Ok(())
-}
-
-pub fn init_field_at_addr(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- field_ty: WasmStorageType,
- field_addr: ir::Value,
- new_val: ir::Value,
-) -> WasmResult<()> {
- gc_compiler(func_env)?.init_field(func_env, builder, field_ty, field_addr, new_val)
-}
-
-pub fn write_field_at_addr(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- field_ty: WasmStorageType,
- field_addr: ir::Value,
- new_val: ir::Value,
-) -> WasmResult<()> {
- // Data inside GC objects is always little endian.
- let flags = func_env
- .gc_memflags(&mut builder.func)
- .with_endianness(ir::Endianness::Little);
-
- match field_ty {
- WasmStorageType::I8 => {
- builder.ins().istore8(flags, new_val, field_addr, 0);
- }
- WasmStorageType::I16 => {
- builder.ins().istore16(flags, new_val, field_addr, 0);
- }
- WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {
- WasmHeapTopType::Func => {
- write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?
- }
- WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {
- gc_compiler(func_env)?.translate_write_gc_reference(
- func_env, builder, r, field_addr, new_val, flags,
- )?;
- }
- WasmHeapTopType::Cont => return stack_switching_unsupported(),
- },
- WasmStorageType::Val(_) => {
- assert_eq!(
- builder.func.dfg.value_type(new_val).bytes(),
- wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty)
- );
- builder.ins().store(flags, new_val, field_addr, 0);
- }
- }
- Ok(())
-}
-
-pub fn translate_struct_new(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- struct_type_index: TypeIndex,
- fields: &[ir::Value],
-) -> WasmResult<ir::Value> {
- gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)
-}
-
-fn default_value(
- cursor: &mut FuncCursor,
- func_env: &FuncEnvironment<'_>,
- ty: &WasmStorageType,
-) -> ir::Value {
- match ty {
- WasmStorageType::I8 | WasmStorageType::I16 => cursor.ins().iconst(ir::types::I32, 0),
- WasmStorageType::Val(v) => match v {
- WasmValType::I32 => cursor.ins().iconst(ir::types::I32, 0),
- WasmValType::I64 => cursor.ins().iconst(ir::types::I64, 0),
- WasmValType::F32 => cursor.ins().f32const(0.0),
- WasmValType::F64 => cursor.ins().f64const(0.0),
- WasmValType::V128 => {
- let c = cursor.func.dfg.constants.insert(vec![0; 16].into());
- cursor.ins().vconst(ir::types::I8X16, c)
- }
- WasmValType::Ref(r) => {
- assert!(r.nullable);
- let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);
-
- // NB: The collector doesn't need to know about null references.
- let _ = needs_stack_map;
-
- cursor.ins().iconst(ty, 0)
- }
- },
- }
-}
-
-pub fn translate_struct_new_default(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- struct_type_index: TypeIndex,
-) -> WasmResult<ir::Value> {
- let interned_ty = func_env.module.types[struct_type_index].unwrap_module_type_index();
- let struct_ty = func_env.types.unwrap_struct(interned_ty)?;
- let fields = struct_ty
- .fields
- .iter()
- .map(|f| default_value(&mut builder.cursor(), func_env, &f.element_type))
- .collect::<StructFieldsVec>();
- gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)
-}
-
-pub fn translate_struct_get(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- struct_type_index: TypeIndex,
- field_index: u32,
- struct_ref: ir::Value,
- extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- log::trace!(
- "translate_struct_get({struct_type_index:?}, {field_index:?}, {struct_ref:?}, {extension:?})"
- );
-
- // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a
- // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that
- // type info from `wasmparser` and through to here is a bit funky.
- func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
-
- emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
-
- let field_index = usize::try_from(field_index).unwrap();
- let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
-
- let struct_layout = func_env.struct_or_exn_layout(interned_type_index);
- let struct_size = struct_layout.size;
-
- let field_offset = struct_layout.fields[field_index].offset;
- let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];
- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);
- assert!(field_offset + field_size <= struct_size);
-
- let field_addr = func_env.prepare_gc_ref_access(
- builder,
- struct_ref,
- BoundsCheck::StaticObjectField {
- offset: field_offset,
- access_size: u8::try_from(field_size).unwrap(),
- object_size: struct_size,
- },
- );
-
- let result = read_field_at_addr(
- func_env,
- builder,
- field_ty.element_type,
- field_addr,
- extension,
- );
- log::trace!("translate_struct_get(..) -> {result:?}");
- result
-}
-
-pub fn translate_struct_set(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- struct_type_index: TypeIndex,
- field_index: u32,
- struct_ref: ir::Value,
- new_val: ir::Value,
-) -> WasmResult<()> {
- log::trace!(
- "translate_struct_set({struct_type_index:?}, {field_index:?}, struct_ref: {struct_ref:?}, new_val: {new_val:?})"
- );
-
- // TODO: See comment in `translate_struct_get` about the `trapz`.
- func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
-
- emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
-
- let field_index = usize::try_from(field_index).unwrap();
- let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
-
- let struct_layout = func_env.struct_or_exn_layout(interned_type_index);
- let struct_size = struct_layout.size;
-
- let field_offset = struct_layout.fields[field_index].offset;
- let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];
- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);
- assert!(field_offset + field_size <= struct_size);
-
- let field_addr = func_env.prepare_gc_ref_access(
- builder,
- struct_ref,
- BoundsCheck::StaticObjectField {
- offset: field_offset,
- access_size: u8::try_from(field_size).unwrap(),
- object_size: struct_size,
- },
- );
-
- write_field_at_addr(
- func_env,
- builder,
- field_ty.element_type,
- field_addr,
- new_val,
- )?;
-
- log::trace!("translate_struct_set: finished");
- Ok(())
-}
-
-pub fn translate_exn_unbox(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- tag_index: TagIndex,
- exn_ref: ir::Value,
-) -> WasmResult<SmallVec<[ir::Value; 4]>> {
- log::trace!("translate_exn_unbox({tag_index:?}, {exn_ref:?})");
-
- // We know that the `exn_ref` is not null because we reach this
- // operation only in catch blocks, and throws are initiated from
- // runtime code that checks for nulls first.
-
- // Get the GcExceptionLayout associated with this tag's
- // function type, and generate loads for each field.
- let exception_ty_idx = func_env
- .exception_type_from_tag(tag_index)
- .unwrap_module_type_index();
- let exception_ty = func_env.types.unwrap_exn(exception_ty_idx)?;
- let exn_layout = func_env.struct_or_exn_layout(exception_ty_idx);
- let exn_size = exn_layout.size;
-
- // Gather accesses first because these require a borrow on
- // `func_env`, which we later mutate below via
- // `prepare_gc_ref_access()`.
- let mut accesses: SmallVec<[_; 4]> = smallvec![];
- for (field_ty, field_layout) in exception_ty.fields.iter().zip(exn_layout.fields.iter()) {
- accesses.push((field_layout.offset, field_ty.element_type));
- }
-
- let mut result = smallvec![];
- for (field_offset, field_ty) in accesses {
- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty);
- assert!(field_offset + field_size <= exn_size);
- let field_addr = func_env.prepare_gc_ref_access(
- builder,
- exn_ref,
- BoundsCheck::StaticObjectField {
- offset: field_offset,
- access_size: u8::try_from(field_size).unwrap(),
- object_size: exn_size,
- },
- );
-
- let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None)?;
- result.push(value);
- }
-
- log::trace!("translate_exn_unbox(..) -> {result:?}");
- Ok(result)
-}
-
-pub fn translate_exn_throw(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- tag_index: TagIndex,
- args: &[ir::Value],
-) -> WasmResult<()> {
- let (instance_id, defined_tag_id) = func_env.get_instance_and_tag(builder, tag_index);
- let exnref = gc_compiler(func_env)?.alloc_exn(
- func_env,
- builder,
- tag_index,
- args,
- instance_id,
- defined_tag_id,
- )?;
- translate_exn_throw_ref(func_env, builder, exnref)
-}
-
-pub fn translate_exn_throw_ref(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- exnref: ir::Value,
-) -> WasmResult<()> {
- let builtin = func_env.builtin_functions.throw_ref(builder.func);
- let sig = builder.func.dfg.ext_funcs[builtin].signature;
- let vmctx = func_env.vmctx_val(&mut builder.cursor());
-
- // Generate a `try_call` with handlers from the current
- // stack. This libcall is unique among libcall implementations of
- // opcodes: we know the others will not throw, but `throw_ref`'s
- // entire purpose is to throw. So if there are any handlers in the
- // local function body, we need to attach them to this callsite
- // like any other.
- let continuation = builder.create_block();
- let current_block = builder.current_block().unwrap();
- builder.insert_block_after(continuation, current_block);
- let continuation_call = builder.func.dfg.block_call(continuation, &[]);
- let mut table_items = vec![ExceptionTableItem::Context(vmctx)];
- for (tag, block) in func_env.stacks.handlers.handlers() {
- let block_call = builder
- .func
- .dfg
- .block_call(block, &[BlockArg::TryCallExn(0)]);
- table_items.push(match tag {
- Some(tag) => ExceptionTableItem::Tag(tag, block_call),
- None => ExceptionTableItem::Default(block_call),
- });
- }
- let etd = ExceptionTableData::new(sig, continuation_call, table_items);
- let et = builder.func.dfg.exception_tables.push(etd);
-
- builder.ins().try_call(builtin, &[vmctx, exnref], et);
-
- builder.switch_to_block(continuation);
- builder.seal_block(continuation);
- func_env.trap(builder, crate::TRAP_UNREACHABLE);
-
- Ok(())
-}
-
-pub fn translate_array_new(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- elem: ir::Value,
- len: ir::Value,
-) -> WasmResult<ir::Value> {
- log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");
- let result =
- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
- let zero = builder.ins().iconst(ir::types::I32, 0);
- let ty = func_env.module.types[array_type_index].unwrap_module_type_index();
- func_env.translate_entity_fill(
- builder,
- CheckedEntity::Array {
- array: result,
- ty,
- initialized: false,
- },
- zero,
- elem,
- len,
- )?;
- log::trace!("translate_array_new(..) -> {result:?}");
- Ok(result)
-}
-
-pub fn translate_array_new_default(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- len: ir::Value,
-) -> WasmResult<ir::Value> {
- log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");
-
- let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();
- let array_ty = func_env.types.unwrap_array(interned_ty)?;
- let result =
- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
- log::trace!("translate_array_new_default(..) -> {result:?}");
- let elem = default_value(&mut builder.cursor(), func_env, &array_ty.0.element_type);
- let zero = builder.ins().iconst(ir::types::I32, 0);
- func_env.translate_entity_fill(
- builder,
- CheckedEntity::Array {
- array: result,
- ty: interned_ty,
- initialized: false,
- },
- zero,
- elem,
- len,
- )?;
- Ok(result)
-}
-
-pub fn translate_array_new_fixed(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- elems: &[ir::Value],
-) -> WasmResult<ir::Value> {
- log::trace!("translate_array_new_fixed({array_type_index:?}, {elems:?})");
- let len = builder
- .ins()
- .iconst(ir::types::I32, i64::try_from(elems.len()).unwrap());
- let result =
- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
- log::trace!("translate_array_new_fixed(..) -> {result:?}");
- let ty = func_env.module.types[array_type_index].unwrap_module_type_index();
- let array_ty = func_env.types.unwrap_array(ty)?;
-
- for (i, elem) in elems.iter().enumerate() {
- let index = builder
- .ins()
- .iconst(ir::types::I32, i64::try_from(i).unwrap());
- let addr = array_elem_addr(func_env, builder, ty, result, index)?;
- gc_compiler(func_env)?.init_field(
- func_env,
- builder,
- array_ty.0.element_type,
- addr,
- *elem,
- )?;
- }
- Ok(result)
-}
-
-pub fn translate_array_len(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_ref: ir::Value,
-) -> WasmResult<ir::Value> {
- log::trace!("translate_array_len({array_ref:?})");
-
- func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE);
-
- let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();
- let len_field = func_env.prepare_gc_ref_access(
- builder,
- array_ref,
- // Note: We can't bounds check the whole array object's size because we
- // don't know its length yet. Chicken and egg problem.
- BoundsCheck::StaticOffset {
- offset: len_offset,
- access_size: u8::try_from(ir::types::I32.bytes()).unwrap(),
- },
- );
- let flags = func_env.gc_memflags(&mut builder.func).with_readonly();
- let result = builder.ins().load(ir::types::I32, flags, len_field, 0);
- log::trace!("translate_array_len(..) -> {result:?}");
- Ok(result)
-}
-
-struct ArraySizeInfo {
- /// The `i32` size of the whole array object, in bytes.
- obj_size: ir::Value,
-
- /// The `i32` size of each one of the array's elements, in bytes.
- one_elem_size: ir::Value,
-
- /// The `i32` size of the array's base object, in bytes. This is also the
- /// offset from the start of the array object to its elements.
- base_size: ir::Value,
-}
-
-/// Emit code to get the dynamic size (in bytes) of a whole array object, along
-/// with some other related bits.
-fn emit_array_size_info(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- array_type_index: ModuleInternedTypeIndex,
- // `i32` value containing the array's length.
- array_len: ir::Value,
-) -> WasmResult<ArraySizeInfo> {
- let array_layout = func_env.array_layout(array_type_index)?;
-
- // Note that we check for overflow below because we can't trust the array's
- // length: it came from inside the GC heap.
- //
- // We check for 32-bit multiplication overflow by performing a 64-bit
- // multiplication and testing the high bits.
- let one_elem_size = builder
- .ins()
- .iconst(ir::types::I64, i64::from(array_layout.elem_size));
- let array_len = builder.ins().uextend(ir::types::I64, array_len);
- let all_elems_size = builder.ins().imul(one_elem_size, array_len);
-
- let high_bits = builder.ins().ushr_imm(all_elems_size, 32);
- builder.ins().trapnz(high_bits, TRAP_GC_HEAP_CORRUPT);
-
- let all_elems_size = builder.ins().ireduce(ir::types::I32, all_elems_size);
- let base_size = builder
- .ins()
- .iconst(ir::types::I32, i64::from(array_layout.base_size));
- let obj_size =
- builder
- .ins()
- .uadd_overflow_trap(all_elems_size, base_size, TRAP_GC_HEAP_CORRUPT);
-
- let one_elem_size = builder.ins().ireduce(ir::types::I32, one_elem_size);
-
- Ok(ArraySizeInfo {
- obj_size,
- one_elem_size,
- base_size,
- })
-}
-
-/// Get the bounds-checked address of an element in an array.
-///
-/// The emitted code will trap if `index >= array.length`.
-///
-/// Returns the `ir::Value` containing the address of the `index`th element in
-/// the array. You may read or write a value of the array's element type at this
-/// address. You may not use it for any other kind of access, nor reuse this
-/// value across GC safepoints.
-fn array_elem_addr(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- array_type_index: ModuleInternedTypeIndex,
- array_ref: ir::Value,
- index: ir::Value,
-) -> WasmResult<ir::Value> {
- // First, assert that `index < array.length`.
- //
- // This check is visible at the Wasm-semantics level.
- //
- // TODO: We should emit spectre-safe bounds checks for array accesses (if
- // configured) but we don't currently have a great way to do that here. The
- // proper solution is to use linear memories to back GC heaps and reuse the
- // code in `bounds_check.rs` to implement these bounds checks. That is all
- // planned, but not yet implemented.
-
- let len = translate_array_len(func_env, builder, array_ref).unwrap();
-
- let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len);
- func_env.trapz(builder, in_bounds, TRAP_ARRAY_OUT_OF_BOUNDS);
-
- // Compute the size (in bytes) of the whole array object.
- let ArraySizeInfo {
- obj_size,
- one_elem_size,
- base_size,
- } = emit_array_size_info(func_env, builder, array_type_index, len)?;
-
- // Compute the offset of the `index`th element within the array object.
- //
- // NB: no need to check for overflow here, since at this point we know that
- // `len * elem_size + base_size` did not overflow and `i < len`.
- let offset_in_elems = builder.ins().imul(index, one_elem_size);
- let offset_in_array = builder.ins().iadd(offset_in_elems, base_size);
-
- // Finally, use the object size and element offset we just computed to
- // perform our implementation-internal bounds checks.
- //
- // Checking the whole object's size, rather than the `index`th element's
- // size allows these bounds checks to be deduplicated across repeated
- // accesses to the same array at different indices.
- //
- // This check should not be visible to Wasm, and serve to protect us from
- // our own implementation bugs. The goal is to keep any potential widgets
- // confined within the GC heap, and turn what would otherwise be a security
- // vulnerability into a simple bug.
- //
- // TODO: Ideally we should fold the first Wasm-visible bounds check into
- // this internal bounds check, so that we aren't performing multiple,
- // redundant bounds checks. But we should figure out how to do this in a way
- // that doesn't defeat the object-size bounds checking's deduplication
- // mentioned above.
- Ok(func_env.prepare_gc_ref_access(
- builder,
- array_ref,
- BoundsCheck::DynamicObjectField {
- offset: offset_in_array,
- object_size: obj_size,
- },
- ))
-}
-
-pub fn translate_array_get(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- array_ref: ir::Value,
- index: ir::Value,
- extension: Option<Extension>,
-) -> WasmResult<ir::Value> {
- log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");
-
- emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
-
- let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
- let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;
-
- let array_ty = func_env.types.unwrap_array(array_type_index)?;
- let elem_ty = array_ty.0.element_type;
-
- let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension)?;
- log::trace!("translate_array_get(..) -> {result:?}");
- Ok(result)
-}
-
-pub fn translate_array_set(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- array_ref: ir::Value,
- index: ir::Value,
- value: ir::Value,
-) -> WasmResult<()> {
- log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");
-
- emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
-
- let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
- let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;
-
- let array_ty = func_env.types.unwrap_array(array_type_index)?;
- let elem_ty = array_ty.0.element_type;
-
- write_field_at_addr(func_env, builder, elem_ty, elem_addr, value)?;
-
- log::trace!("translate_array_set: finished");
- Ok(())
-}
-
-pub fn translate_ref_test(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- test_ty: WasmRefType,
- val: ir::Value,
- val_ty: WasmRefType,
-) -> WasmResult<ir::Value> {
- log::trace!("translate_ref_test({test_ty:?}, {val:?})");
-
- // First special case: testing for references to bottom types.
- if test_ty.heap_type.is_bottom() {
- let result = if test_ty.nullable {
- // All null references (within the same type hierarchy) match null
- // references to the bottom type.
- func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?
- } else {
- // `ref.test` is always false for non-nullable bottom types, as the
- // bottom types are uninhabited.
- builder.ins().iconst(ir::types::I32, 0)
- };
- log::trace!("translate_ref_test(..) -> {result:?}");
- return Ok(result);
- }
-
- // And because `ref.test heap_ty` is only valid on operands whose type is in
- // the same type hierarchy as `heap_ty`, if `heap_ty` is its hierarchy's top
- // type, we only need to worry about whether we are testing for nullability
- // or not.
- if test_ty.heap_type.is_top() {
- let result = if test_ty.nullable {
- builder.ins().iconst(ir::types::I32, 1)
- } else {
- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
- let zero = builder.ins().iconst(ir::types::I32, 0);
- let one = builder.ins().iconst(ir::types::I32, 1);
- builder.ins().select(is_null, zero, one)
- };
- log::trace!("translate_ref_test(..) -> {result:?}");
- return Ok(result);
- }
-
- // `i31ref`s are a little interesting because they don't point to GC
- // objects; we test the bit pattern of the reference itself.
- if test_ty.heap_type == WasmHeapType::I31 {
- let i31_mask = builder.ins().iconst(
- ir::types::I32,
- i64::from(wasmtime_environ::I31_DISCRIMINANT),
- );
- let is_i31 = builder.ins().band(val, i31_mask);
- let result = if test_ty.nullable {
- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
- builder.ins().bor(is_null, is_i31)
- } else {
- is_i31
- };
- log::trace!("translate_ref_test(..) -> {result:?}");
- return Ok(result);
- }
-
- // Otherwise, in the general case, we need to inspect our given object's
- // actual type, which also requires null-checking and i31-checking it.
-
- let is_any_hierarchy = test_ty.heap_type.top() == WasmHeapTopType::Any;
-
- let non_null_block = builder.create_block();
- let non_null_non_i31_block = builder.create_block();
- let continue_block = builder.create_block();
-
- // Current block: check if the reference is null and branch appropriately.
- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;
- let result_when_is_null = builder
- .ins()
- .iconst(ir::types::I32, test_ty.nullable as i64);
- builder.ins().brif(
- is_null,
- continue_block,
- &[result_when_is_null.into()],
- non_null_block,
- &[],
- );
-
- // Non-null block: We know the GC ref is non-null, but we need to also check
- // for `i31` references that don't point to GC objects.
- builder.switch_to_block(non_null_block);
- log::trace!("translate_ref_test: non-null ref block");
- if is_any_hierarchy {
- let i31_mask = builder.ins().iconst(
- ir::types::I32,
- i64::from(wasmtime_environ::I31_DISCRIMINANT),
- );
- let is_i31 = builder.ins().band(val, i31_mask);
- // If it is an `i31`, then create the result value based on whether we
- // want `i31`s to pass the test or not.
- let result_when_is_i31 = builder.ins().iconst(
- ir::types::I32,
- matches!(
- test_ty.heap_type,
- WasmHeapType::Any | WasmHeapType::Eq | WasmHeapType::I31
- ) as i64,
- );
- builder.ins().brif(
- is_i31,
- continue_block,
- &[result_when_is_i31.into()],
- non_null_non_i31_block,
- &[],
- );
- } else {
- // If we aren't testing the `any` hierarchy, the reference cannot be an
- // `i31ref`. Jump directly to the non-null and non-i31 block; rely on
- // branch folding during lowering to clean this up.
- builder.ins().jump(non_null_non_i31_block, &[]);
- }
-
- // Non-null and non-i31 block: Read the actual `VMGcKind` or
- // `VMSharedTypeIndex` out of the object's header and check whether it
- // matches the expected type.
- builder.switch_to_block(non_null_non_i31_block);
- log::trace!("translate_ref_test: non-null and non-i31 ref block");
- let check_header_kind = |func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- val: ir::Value,
- expected_kind: VMGcKind|
- -> ir::Value {
- let kind_addr = func_env.prepare_gc_ref_access(
- builder,
- val,
- BoundsCheck::StaticObjectField {
- offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,
- access_size: wasmtime_environ::VM_GC_KIND_SIZE,
- object_size: wasmtime_environ::VM_GC_HEADER_SIZE,
- },
- );
- let gc_memflags = func_env.gc_memflags(&mut builder.func);
- let actual_kind =
- builder
- .ins()
- .load(ir::types::I32, gc_memflags.with_readonly(), kind_addr, 0);
- let expected_kind = builder
- .ins()
- .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));
- // Inline version of `VMGcKind::matches`.
- let and = builder.ins().band(actual_kind, expected_kind);
- let kind_matches = builder
- .ins()
- .icmp(ir::condcodes::IntCC::Equal, and, expected_kind);
- builder.ins().uextend(ir::types::I32, kind_matches)
- };
- let result = match test_ty.heap_type {
- WasmHeapType::Any
- | WasmHeapType::None
- | WasmHeapType::Extern
- | WasmHeapType::NoExtern
- | WasmHeapType::Func
- | WasmHeapType::NoFunc
- | WasmHeapType::Cont
- | WasmHeapType::NoCont
- | WasmHeapType::Exn
- | WasmHeapType::NoExn
- | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"),
-
- // For these abstract but non-top and non-bottom types, we check the
- // `VMGcKind` that is in the object's header.
- WasmHeapType::Eq => check_header_kind(func_env, builder, val, VMGcKind::EqRef),
- WasmHeapType::Struct => check_header_kind(func_env, builder, val, VMGcKind::StructRef),
- WasmHeapType::Array => check_header_kind(func_env, builder, val, VMGcKind::ArrayRef),
-
- // For concrete types, we need to do a full subtype check between the
- // `VMSharedTypeIndex` in the object's header and the
- // `ModuleInternedTypeIndex` we have here.
- //
- // TODO: This check should ideally be done inline, but we don't have a
- // good way to access the `TypeRegistry`'s supertypes arrays from Wasm
- // code at the moment.
- WasmHeapType::ConcreteArray(ty)
- | WasmHeapType::ConcreteStruct(ty)
- | WasmHeapType::ConcreteExn(ty) => {
- let expected_interned_ty = ty.unwrap_module_type_index();
- let expected_shared_ty =
- func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
-
- let ty_addr = func_env.prepare_gc_ref_access(
- builder,
- val,
- BoundsCheck::StaticOffset {
- offset: wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET,
- access_size: func_env.offsets.size_of_vmshared_type_index(),
- },
- );
- let gc_memflags = func_env.gc_memflags(&mut builder.func);
- let actual_shared_ty =
- builder
- .ins()
- .load(ir::types::I32, gc_memflags.with_readonly(), ty_addr, 0);
-
- func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)
- }
-
- // Same as for concrete arrays and structs except that a `VMFuncRef`
- // doesn't begin with a `VMGcHeader` and is a raw pointer rather than GC
- // heap index.
- WasmHeapType::ConcreteFunc(ty) => {
- let expected_interned_ty = ty.unwrap_module_type_index();
- let expected_shared_ty =
- func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
-
- let gc_memflags = func_env.gc_memflags(&mut builder.func);
- let actual_shared_ty = func_env.load_funcref_type_index(
- &mut builder.cursor(),
- gc_memflags.with_readonly(),
- val,
- );
-
- func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)
- }
- WasmHeapType::ConcreteCont(_) => {
- // TODO(#10248) GC integration for stack switching
- return stack_switching_unsupported();
- }
- };
- builder.ins().jump(continue_block, &[result.into()]);
-
- // Control flow join point with the result.
- builder.switch_to_block(continue_block);
- let result = builder.append_block_param(continue_block, ir::types::I32);
- log::trace!("translate_ref_test(..) -> {result:?}");
-
- builder.seal_block(non_null_block);
- builder.seal_block(non_null_non_i31_block);
- builder.seal_block(continue_block);
-
- Ok(result)
-}
-
-#[cfg(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying"))]
-fn uextend_i32_to_pointer_type(
- builder: &mut FunctionBuilder,
- pointer_type: ir::Type,
- value: ir::Value,
-) -> ir::Value {
- assert_eq!(builder.func.dfg.value_type(value), ir::types::I32);
- match pointer_type {
- ir::types::I32 => value,
- ir::types::I64 => builder.ins().uextend(ir::types::I64, value),
- _ => unreachable!(),
- }
-}
-
-/// Emit CLIF to compute an array object's total size, given the dynamic length
-/// in its initialization.
-///
-/// Traps if the size overflows.
-#[cfg_attr(
- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
- expect(dead_code, reason = "easier to define")
-)]
-fn emit_array_size(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- array_layout: &GcArrayLayout,
- len: ir::Value,
-) -> ir::Value {
- let base_size = builder
- .ins()
- .iconst(ir::types::I32, i64::from(array_layout.base_size));
-
- // `elems_size = len * elem_size`
- //
- // Check for multiplication overflow and trap if it occurs, since that
- // means Wasm is attempting to allocate an array that is larger than our
- // implementation limits. (Note: there is no standard implementation
- // limit for array length beyond `u32::MAX`.)
- //
- // We implement this check by encoding our logically-32-bit operands as
- // i64 values, doing a 64-bit multiplication, and then checking the high
- // 32 bits of the multiplication's result. If the high 32 bits are not
- // all zeros, then the multiplication overflowed.
- debug_assert_eq!(builder.func.dfg.value_type(len), ir::types::I32);
- let len = builder.ins().uextend(ir::types::I64, len);
- let elems_size_64 = builder
- .ins()
- .imul_imm(len, i64::from(array_layout.elem_size));
- let high_bits = builder.ins().ushr_imm(elems_size_64, 32);
- func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE);
- let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64);
-
- // And if adding the base size and elements size overflows, then the
- // allocation is too large.
- let size = func_env.uadd_overflow_trap(
- builder,
- base_size,
- elems_size,
- crate::TRAP_ALLOCATION_TOO_LARGE,
- );
-
- size
-}
-
-/// Common helper for struct-field initialization that can be reused across
-/// collectors.
-#[cfg_attr(
- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
- expect(dead_code, reason = "easier to define")
-)]
-fn initialize_struct_fields(
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder<'_>,
- struct_ty: ModuleInternedTypeIndex,
- raw_ptr_to_struct: ir::Value,
- field_values: &[ir::Value],
-) -> WasmResult<()> {
- let struct_layout = func_env.struct_or_exn_layout(struct_ty);
- let struct_size = struct_layout.size;
- let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().map(|f| f.offset).collect();
- assert_eq!(field_offsets.len(), field_values.len());
-
- assert!(!func_env.types[struct_ty].composite_type.shared);
- let fields = match &func_env.types[struct_ty].composite_type.inner {
- WasmCompositeInnerType::Struct(s) => &s.fields,
- WasmCompositeInnerType::Exn(e) => &e.fields,
- _ => panic!("Not a struct or exception type"),
- };
-
- let field_types: SmallVec<[_; 8]> = fields.iter().cloned().collect();
- assert_eq!(field_types.len(), field_values.len());
-
- for ((ty, val), offset) in field_types.into_iter().zip(field_values).zip(field_offsets) {
- let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty.element_type);
- assert!(offset + size_of_access <= struct_size);
- let field_addr = builder.ins().iadd_imm(raw_ptr_to_struct, i64::from(offset));
- gc_compiler(func_env)?.init_field(func_env, builder, ty.element_type, field_addr, *val)?;
- }
-
- Ok(())
-}
-
-impl FuncEnvironment<'_> {
- pub(crate) fn gc_heap_alias_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
- self.alias_region(func, AliasRegionKey::GcHeap)
- }
-
- /// Flags to use for general-purpose GC loads/stores.
- ///
- /// This is used for accesses to the GC heap which aren't expected to trap, but
- /// retain internal assertion metadata to report if such a trap happens. This
- /// is here to ensure that in the face of heap corruption that there's no
- /// possible UB within Cranelift and/or the runtime.
- fn gc_memflags(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {
- let region = self.gc_heap_alias_region(func);
- ir::MemFlagsData::new()
- .with_trap_code(Some(crate::TRAP_GC_HEAP_CORRUPT))
- .with_alias_region(Some(region))
- }
-
- fn gc_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcLayout {
- // Lazily compute and cache the layout.
- if !self.ty_to_gc_layout.contains_key(&type_index) {
- let ty = &self.types[type_index].composite_type;
- let layout = gc_compiler(self)
- .unwrap()
- .layouts()
- .gc_layout(ty)
- .expect("gc_layout should not OOM at compile time")
- .expect("should only call `FuncEnvironment::gc_layout` for GC types");
- self.ty_to_gc_layout.insert(type_index, layout);
- }
-
- self.ty_to_gc_layout.get(&type_index).unwrap()
- }
-
- /// Get the `GcArrayLayout` for the array type at the given `type_index`.
- pub(crate) fn array_layout(
- &mut self,
- type_index: ModuleInternedTypeIndex,
- ) -> WasmResult<&GcArrayLayout> {
- Ok(self.gc_layout(type_index).unwrap_array())
- }
-
- /// Get the `GcStructLayout` for the struct or exception type at the given `type_index`.
- fn struct_or_exn_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcStructLayout {
- let result = self.gc_layout(type_index).unwrap_struct();
- result
- }
-
- /// Get or create the global for our GC heap's base pointer.
- fn get_gc_heap_base_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {
- if let Some(base) = self.gc_heap_base {
- return base;
- }
-
- let store_context_ptr = self.get_vmstore_context_ptr_global(func);
- let offset = self.offsets.ptr.vmstore_context_gc_heap_base();
-
- let mut flags = ir::MemFlagsData::trusted();
- let memory_tunables =
- wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);
- if !self
- .tunables
- .gc_heap_memory_type()
- .memory_may_move(&memory_tunables)
- {
- flags.set_readonly();
- flags.set_can_move();
- }
-
- let base_flags = func.dfg.mem_flags.insert(flags).unwrap();
- let base = func.create_global_value(ir::GlobalValueData::Load {
- base: store_context_ptr,
- offset: Offset32::new(offset.into()),
- global_type: self.pointer_type(),
- flags: base_flags,
- });
-
- self.gc_heap_base = Some(base);
- base
- }
-
- /// Get the GC heap's base.
- pub(crate) fn get_gc_heap_base(
- &mut self,
- builder: &mut FunctionBuilder,
- ) -> WasmResult<ir::Value> {
- let global = self.get_gc_heap_base_global(&mut builder.func);
- Ok(builder.ins().global_value(self.pointer_type(), global))
- }
-
- fn get_gc_heap_bound_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {
- if let Some(bound) = self.gc_heap_bound {
- return bound;
- }
- let store_context_ptr = self.get_vmstore_context_ptr_global(func);
- let offset = self.offsets.ptr.vmstore_context_gc_heap_current_length();
- let bound_flags = func
- .dfg
- .mem_flags
- .insert(ir::MemFlagsData::trusted())
- .unwrap();
- let bound = func.create_global_value(ir::GlobalValueData::Load {
- base: store_context_ptr,
- offset: Offset32::new(offset.into()),
- global_type: self.pointer_type(),
- flags: bound_flags,
- });
- self.gc_heap_bound = Some(bound);
- bound
- }
-
- /// Get the GC heap's bound.
- pub(crate) fn get_gc_heap_bound(
- &mut self,
- builder: &mut FunctionBuilder,
- ) -> WasmResult<ir::Value> {
- let global = self.get_gc_heap_bound_global(&mut builder.func);
- Ok(builder.ins().global_value(self.pointer_type(), global))
- }
-
- /// Get or create the `Heap` for our GC heap.
- fn get_gc_heap(&mut self, func: &mut ir::Function) -> Heap {
- if let Some(heap) = self.gc_heap {
- return heap;
- }
-
- let base = self.get_gc_heap_base_global(func);
- let bound = self.get_gc_heap_bound_global(func);
- let memory = self.tunables.gc_heap_memory_type();
- let heap = self.heaps.push(HeapData {
- base,
- bound,
- memory,
- kind: MemoryKind::GcHeap,
- });
- self.gc_heap = Some(heap);
- heap
- }
-
- /// Get the raw pointer of `gc_ref[offset]` bounds checked for an access of
- /// `size` bytes.
- ///
- /// The given `gc_ref` must be a non-null, non-i31 GC reference.
- ///
- /// If `check` is a `BoundsCheck::Object`, then it is the callers
- /// responsibility to ensure that `offset + access_size <= object_size`.
- ///
- /// Returns a raw pointer to `gc_ref[offset]` -- not a raw pointer to the GC
- /// object itself (unless `offset` happens to be `0`). This raw pointer may
- /// be used to read or write up to as many bytes as described by `bound`. Do
- /// NOT attempt accesses bytes outside of `bound`; that may lead to
- /// unchecked out-of-bounds accesses.
- ///
- /// This method is collector-agnostic.
- fn prepare_gc_ref_access(
- &mut self,
- builder: &mut FunctionBuilder,
- gc_ref: ir::Value,
- bounds_check: BoundsCheck,
- ) -> ir::Value {
- log::trace!("prepare_gc_ref_access({gc_ref:?}, {bounds_check:?})");
- assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);
-
- let gc_heap = self.get_gc_heap(&mut builder.func);
- let gc_heap = self.heaps[gc_heap].clone();
- let result = match crate::bounds_checks::bounds_check_and_compute_addr(
- builder,
- self,
- &gc_heap,
- gc_ref,
- bounds_check,
- crate::TRAP_GC_HEAP_CORRUPT,
- ) {
- Reachability::Reachable(v) => v,
- Reachability::Unreachable => {
- // We are now in unreachable code, but we don't want to plumb
- // through a bunch of `Reachability` through all of our callers,
- // so just assert we won't reach here and return `null`
- let null = builder.ins().iconst(self.pointer_type(), 0);
- builder.ins().trapz(null, crate::TRAP_INTERNAL_ASSERT);
- null
- }
- };
- log::trace!("prepare_gc_ref_access(..) -> {result:?}");
- result
- }
-
- /// Emit checks (if necessary) for whether the given `gc_ref` is null or is
- /// an `i31ref`.
- ///
- /// Takes advantage of static information based on `ty` as to whether the GC
- /// reference is nullable or can ever be an `i31`.
- ///
- /// Returns an `ir::Value` that is an `i32` will be non-zero if the GC
- /// reference is null or is an `i31ref`; otherwise, it will be zero.
- ///
- /// This method is collector-agnostic.
- #[cfg_attr(
- not(feature = "gc-drc"),
- expect(dead_code, reason = "easier to define")
- )]
- fn gc_ref_is_null_or_i31(
- &mut self,
- builder: &mut FunctionBuilder,
- ty: WasmRefType,
- gc_ref: ir::Value,
- ) -> ir::Value {
- assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);
- assert!(ty.is_vmgcref_type_and_not_i31());
-
- let might_be_i31 = match ty.heap_type {
- // If we are definitely dealing with an i31, we shouldn't be
- // emitting dynamic checks for it, and the caller shouldn't call
- // this function. Should have been caught by the assertion at the
- // start of the function.
- WasmHeapType::I31 => unreachable!(),
-
- // Could potentially be an i31.
- WasmHeapType::Any | WasmHeapType::Eq => true,
-
- // If it is definitely a struct, array, or uninhabited type, then it
- // is definitely not an i31.
- WasmHeapType::Array
- | WasmHeapType::ConcreteArray(_)
- | WasmHeapType::Struct
- | WasmHeapType::ConcreteStruct(_)
- | WasmHeapType::None => false,
-
- // Despite being a different type hierarchy, this *could* be an
- // `i31` if it is the result of
- //
- // (extern.convert_any (ref.i31 ...))
- WasmHeapType::Extern => true,
-
- // Can only ever be `null`.
- WasmHeapType::NoExtern => false,
-
- WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn => false,
-
- // Wrong type hierarchy, and also funcrefs are not GC-managed
- // types. Should have been caught by the assertion at the start of
- // the function.
- WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => {
- unreachable!()
- }
- WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => {
- unreachable!()
- }
- };
-
- match (ty.nullable, might_be_i31) {
- // This GC reference statically cannot be null nor an i31. (Let
- // Cranelift's optimizer const-propagate this value and erase any
- // unnecessary control flow resulting from branching on this value.)
- (false, false) => builder.ins().iconst(ir::types::I32, 0),
-
- // This GC reference is always non-null, but might be an i31.
- (false, true) => builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT)),
-
- // This GC reference might be null, but can never be an i31.
- (true, false) => builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0),
-
- // Fully general case: this GC reference could be either null or an
- // i31.
- (true, true) => {
- let is_i31 = builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT));
- let is_null = builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0);
- let is_null = builder.ins().uextend(ir::types::I32, is_null);
- builder.ins().bor(is_i31, is_null)
- }
- }
- }
-
- // Emit code to check whether `a <: b` for two `VMSharedTypeIndex`es.
- pub(crate) fn is_subtype(
- &mut self,
- builder: &mut FunctionBuilder<'_>,
- a: ir::Value,
- b: ir::Value,
- ) -> ir::Value {
- log::trace!("is_subtype({a:?}, {b:?})");
-
- let diff_tys_block = builder.create_block();
- let continue_block = builder.create_block();
-
- // Current block: fast path for when `a == b`.
- log::trace!("is_subtype: fast path check for exact same types");
- let same_ty = builder.ins().icmp(IntCC::Equal, a, b);
- let same_ty = builder.ins().uextend(ir::types::I32, same_ty);
- builder.ins().brif(
- same_ty,
- continue_block,
- &[same_ty.into()],
- diff_tys_block,
- &[],
- );
-
- // Different types block: fall back to the `is_subtype` libcall.
- builder.switch_to_block(diff_tys_block);
- log::trace!("is_subtype: slow path to do full `is_subtype` libcall");
- let is_subtype = self.builtin_functions.is_subtype(builder.func);
- let vmctx = self.vmctx_val(&mut builder.cursor());
- let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]);
- let result = builder.func.dfg.first_result(call_inst);
- builder.ins().jump(continue_block, &[result.into()]);
-
- // Continue block: join point for the result.
- builder.switch_to_block(continue_block);
- let result = builder.append_block_param(continue_block, ir::types::I32);
- log::trace!("is_subtype(..) -> {result:?}");
-
- builder.seal_block(diff_tys_block);
- builder.seal_block(continue_block);
-
- result
- }
-}
-
-fn stack_switching_unsupported<T>() -> WasmResult<T> {
- Err(wasmtime_environ::WasmError::Unsupported(
- "Stack switching feature not compatible with GC, yet".to_string(),
- ))
-}
-
-pub fn translate_array_new_entity(
- env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- array_type_index: TypeIndex,
- entity: CheckedEntity,
- entity_offset: ir::Value,
- len: ir::Value,
-) -> WasmResult<ir::Value> {
- // Before actually allocating this array first do a bounds-check on the
- // passive entity itself.
- let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();
- env.translate_entity_bounds_check(builder, entity, entity_offset, len)?;
-
- let array = gc_compiler(env)?.alloc_uninit_array(env, builder, array_type_index, len)?;
- let dst = builder.ins().iconst(ir::types::I32, 0);
- env.translate_entity_copy(
- builder,
- CheckedEntity::Array {
- array,
- ty: interned_type_index,
- initialized: false,
- },
- entity,
- dst,
- entity_offset,
- len,
- )?;
-
- Ok(array)
-}
diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/null.rs
similarity index 100%
rename from crates/cranelift/src/func_environ/gc/enabled/null.rs
rename to crates/cranelift/src/func_environ/gc/null.rs
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
index 5da5c88623..41c9dd38b9 100644
--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -1221,7 +1221,7 @@ pub(crate) fn translate_cont_new<'a>(
.ins()
.iconst(I32, i64::try_from(return_types.len()).unwrap());
- let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?;
+ let cont_new_func = env.builtin_functions.cont_new(&mut builder.func);
let vmctx = env.vmctx_val(&mut builder.cursor());
let call_inst = builder
.ins()
diff --git a/crates/cranelift/src/func_environ/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rs
index 15b36d110b..653191a769 100644
--- a/crates/cranelift/src/func_environ/stack_switching/mod.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/mod.rs
@@ -1,35 +1,3 @@
mod control_effect;
pub(crate) mod fatpointer;
pub(crate) mod instructions;
-
-pub(crate) mod builtins {
- macro_rules! define_builtin_accessors {
- ( $( $name:ident , )* ) => {
- $(
- #[inline]
- pub fn $name(
- func_env: &mut crate::func_environ::FuncEnvironment<'_>,
- func: &mut crate::ir::Function,
- ) -> wasmtime_environ::WasmResult<crate::ir::FuncRef> {
- #[cfg(feature = "stack-switching")]
- {
- return Ok(func_env.builtin_functions.$name(func));
- }
-
- #[cfg(not(feature = "stack-switching"))]
- {
- let _ = (func, func_env);
- return Err(wasmtime_environ::wasm_unsupported!(
- "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \
- feature was not enabled"
- ));
- }
- }
- )*
- };
- }
-
- define_builtin_accessors! {
- cont_new,
- }
-}
diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml
index b56304c229..0d08121831 100644
--- a/crates/wasmtime/Cargo.toml
+++ b/crates/wasmtime/Cargo.toml
@@ -188,7 +188,7 @@ pulley = [
]
# Enables support for incremental compilation cache to be enabled in `Config`.
-incremental-cache = ["wasmtime-cranelift?/incremental-cache", "std"]
+incremental-cache = ["std"]
# Enables support for profiling guest modules.
profiling = [
@@ -231,8 +231,6 @@ all-arch = ["wasmtime-cranelift?/all-arch", "wasmtime-winch?/all-arch", "pulley"
# testing purposes.
component-model = [
"wasmtime-environ/component-model",
- "wasmtime-cranelift?/component-model",
- "wasmtime-winch?/component-model",
"dep:wasmtime-component-macro",
"dep:wasmtime-component-util",
"dep:encoding_rs",
@@ -241,8 +239,6 @@ component-model = [
wmemcheck = [
"dep:wasmtime-wmemcheck",
- "wasmtime-cranelift?/wmemcheck",
- "wasmtime-winch?/wmemcheck",
"wasmtime-environ/wmemcheck",
"std",
]
@@ -293,49 +289,24 @@ runtime = [
#
# You can additionally configure which GC implementations are enabled via the
# `gc-drc`, `gc-null`, and `gc-copying` features.
-gc = [
- "wasmtime-environ/gc",
- "wasmtime-cranelift?/gc",
- "wasmtime-winch?/gc",
-]
+gc = ["wasmtime-environ/gc"]
# Enable the deferred reference counting garbage collector.
-gc-drc = [
- "gc",
- "wasmtime-environ/gc-drc",
- "wasmtime-cranelift?/gc-drc",
- "wasmtime-winch?/gc-drc",
-]
+gc-drc = ["gc", "wasmtime-environ/gc-drc"]
# Enable the null garbage collector.
-gc-null = [
- "gc",
- "wasmtime-environ/gc-null",
- "wasmtime-cranelift?/gc-null",
- "wasmtime-winch?/gc-null",
-]
+gc-null = ["gc", "wasmtime-environ/gc-null"]
# Enable the copying garbage collector.
-gc-copying = [
- "gc",
- "wasmtime-environ/gc-copying",
- "wasmtime-cranelift?/gc-copying",
- "wasmtime-winch?/gc-copying",
-]
+gc-copying = ["gc", "wasmtime-environ/gc-copying"]
# Enable runtime support for the WebAssembly threads proposal.
-threads = [
- "wasmtime-cranelift?/threads",
- "wasmtime-winch?/threads",
- "std",
-]
+threads = ["std"]
stack-switching = [
"runtime",
"std",
"wasmtime-environ/stack-switching",
- "wasmtime-cranelift?/stack-switching",
- "wasmtime-winch?/stack-switching",
]
# Controls whether backtraces will attempt to parse DWARF information in
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index 11070cce43..077f475360 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -957,7 +957,6 @@ fn is_inlining_function(key: FuncKey) -> bool {
FuncKey::DefinedWasmFunction(..) => true,
// Intrinsics can be inlined into other functions.
- #[cfg(feature = "component-model")]
FuncKey::UnsafeIntrinsic(..) => true,
// Trampolines cannot participate in inlining since our
@@ -968,7 +967,6 @@ fn is_inlining_function(key: FuncKey) -> bool {
| FuncKey::WasmToBuiltinTrampoline(..)
| FuncKey::PatchableToBuiltinTrampoline(..)
| FuncKey::ModuleStartup(..) => false,
- #[cfg(feature = "component-model")]
FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,
FuncKey::PulleyHostCall(_) => {
diff --git a/crates/winch/Cargo.toml b/crates/winch/Cargo.toml
index 80f6992168..14dc41d408 100644
--- a/crates/winch/Cargo.toml
+++ b/crates/winch/Cargo.toml
@@ -23,15 +23,4 @@ gimli = { workspace = true, features = ['std'] }
log = { workspace = true }
[features]
-component-model = [
- "wasmtime-environ/component-model",
- "wasmtime-cranelift/component-model",
-]
all-arch = ["winch-codegen/all-arch"]
-gc = ['winch-codegen/gc']
-gc-drc = ['winch-codegen/gc-drc']
-gc-null = ['winch-codegen/gc-null']
-gc-copying = ['winch-codegen/gc-copying']
-stack-switching = ['winch-codegen/stack-switching']
-threads = ['winch-codegen/threads']
-wmemcheck = ['winch-codegen/wmemcheck']
diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs
index 6a9a7f2982..ad34b8e2a9 100644
--- a/crates/winch/src/compiler.rs
+++ b/crates/winch/src/compiler.rs
@@ -5,7 +5,6 @@ use std::mem;
use std::sync::Mutex;
use wasmparser::FuncValidatorAllocations;
use wasmtime_cranelift::CompiledFunction;
-#[cfg(feature = "component-model")]
use wasmtime_environ::component::ComponentTranslation;
use wasmtime_environ::error::Result;
use wasmtime_environ::{
@@ -203,7 +202,6 @@ impl wasmtime_environ::Compiler for Compiler {
self.isa.is_branch_protection_enabled()
}
- #[cfg(feature = "component-model")]
fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {
self.trampolines.component_compiler()
}
@@ -309,7 +307,6 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {
self.0.is_branch_protection_enabled()
}
- #[cfg(feature = "component-model")]
fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {
self
}
@@ -330,7 +327,6 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {
}
}
-#[cfg(feature = "component-model")]
impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
fn compile_component_trampoline(
&self,
diff --git a/winch/codegen/Cargo.toml b/winch/codegen/Cargo.toml
index e9d2dba097..07d348f715 100644
--- a/winch/codegen/Cargo.toml
+++ b/winch/codegen/Cargo.toml
@@ -35,10 +35,3 @@ all-arch = [
"x64",
"arm64",
]
-gc = ['wasmtime-environ/gc']
-gc-drc = ['wasmtime-environ/gc-drc']
-gc-null = ['wasmtime-environ/gc-null']
-gc-copying = ['wasmtime-environ/gc-copying']
-stack-switching = ['wasmtime-environ/stack-switching']
-threads = ['wasmtime-environ/threads']
-wmemcheck = ['wasmtime-environ/wmemcheck']
diff --git a/winch/codegen/build.rs b/winch/codegen/build.rs
index 378ac51732..1d9fb5db21 100644
--- a/winch/codegen/build.rs
+++ b/winch/codegen/build.rs
@@ -1,4 +1,29 @@
fn main() {
+ println!("cargo:rerun-if-changed=build.rs");
+
+ enable_default_arch();
+
+ // Pretend that Winch has these features. These are injected via the
+ // `wasmtime_environ::foreach_builtin_function!` macro so to ensure that we
+ // don't get spurious warnings about unused cfg's these are printed out
+ // here.
+ faux_feature("stack-switching");
+ faux_feature("gc");
+ faux_feature("gc-copying");
+ faux_feature("gc-null");
+ faux_feature("gc-drc");
+ faux_feature("threads");
+ faux_feature("wmemcheck");
+ faux_feature("component-model");
+ faux_feature("incremental-cache");
+}
+
+fn faux_feature(feature: &str) {
+ println!("cargo:rustc-check-cfg=cfg(feature, values(\"{feature}\"))");
+ println!("cargo:rustc-cfg=feature=\"{feature}\"");
+}
+
+fn enable_default_arch() {
if cfg!(feature = "x64") || cfg!(feature = "arm64") || cfg!(feature = "all-arch") {
return;
}
@@ -12,5 +37,4 @@ fn main() {
} else {
println!("cargo:rustc-cfg=feature=\"{arch}\"");
}
- println!("cargo:rerun-if-changed=build.rs");
}
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index e993017276..24805f79d0 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -2391,13 +2391,7 @@ where
Ok(())
}
- #[cfg(not(feature = "threads"))]
- pub fn emit_atomic_wait(&mut self, _arg: &MemArg, _kind: AtomicWaitKind) -> Result<()> {
- Err(CodeGenError::unimplemented_wasm_instruction().into())
- }
-
/// Emit the sequence of instruction for a `memory.atomic.wait*`.
- #[cfg(feature = "threads")]
pub fn emit_atomic_wait(&mut self, arg: &MemArg, kind: AtomicWaitKind) -> Result<()> {
// The `memory_atomic_wait*` builtins expect the following arguments:
// - `memory`, as u32
@@ -2448,12 +2442,6 @@ where
Ok(())
}
- #[cfg(not(feature = "threads"))]
- pub fn emit_atomic_notify(&mut self, _arg: &MemArg) -> Result<()> {
- Err(CodeGenError::unimplemented_wasm_instruction().into())
- }
-
- #[cfg(feature = "threads")]
pub fn emit_atomic_notify(&mut self, arg: &MemArg) -> Result<()> {
// The memory `memory_atomic_notify` builtin expects the following arguments:
// - `memory`, as u32