A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove Cargo features from `wasmtime-internal-cranelift` (#13560)
Alex Crichton07db6feef273
diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.tomlindex 0aadcedf23..381e304a2f 100644--- a/crates/cranelift/Cargo.toml+++ b/crates/cranelift/Cargo.toml@@ -16,8 +16,7 @@ workspace = true[dependencies]log = { workspace = true }-wasmtime-environ = { workspace = true, features = ['compile'] }-cranelift-codegen = { workspace = true, features = ["host-arch", "timing"] }+cranelift-codegen = { workspace = true, features = ["host-arch", "timing", "incremental-cache"] }cranelift-frontend = { workspace = true }cranelift-entity = { workspace = true }cranelift-native = { workspace = true }@@ -35,17 +34,20 @@ pulley-interpreter = { workspace = true, optional = true }wasmtime-core = { workspace = true }wasmtime-unwinder = { workspace = true, features = ["cranelift"] }+[dependencies.wasmtime-environ]+workspace = true+features = [+ "compile",+ "component-model",+ "gc-drc",+ "gc-null",+ "gc-copying",+ "stack-switching",+ "threads",+]+[features]all-arch = ["cranelift-codegen/all-arch"]host-arch = ["cranelift-codegen/host-arch"]pulley = ["cranelift-codegen/pulley", "dep:pulley-interpreter"]trace-log = ["cranelift-codegen/trace-log"]-component-model = ["wasmtime-environ/component-model"]-incremental-cache = ["cranelift-codegen/incremental-cache"]-wmemcheck = ["wasmtime-environ/wmemcheck"]-gc = ["wasmtime-environ/gc"]-gc-drc = ["gc", "wasmtime-environ/gc-drc"]-gc-null = ["gc", "wasmtime-environ/gc-null"]-gc-copying = ["gc", "wasmtime-environ/gc-copying"]-stack-switching = []-threads = ["wasmtime-environ/threads"]diff --git a/crates/cranelift/build.rs b/crates/cranelift/build.rsindex d70c2572c8..dfb67d82f7 100644--- a/crates/cranelift/build.rs+++ b/crates/cranelift/build.rs@@ -2,4 +2,23 @@ fn main() {if std::env::var("CARGO_CFG_FUZZING").is_ok() {println!("cargo:rustc-cfg=gc_zeal");}++ // Pretend that Cranelift has these features. These are injected via the+ // `wasmtime_environ::foreach_builtin_function!` macro so to ensure that we+ // don't get spurious warnings about unused cfg's these are printed out+ // here.+ faux_feature("stack-switching");+ faux_feature("gc");+ faux_feature("gc-copying");+ faux_feature("gc-null");+ faux_feature("gc-drc");+ faux_feature("threads");+ faux_feature("wmemcheck");+ faux_feature("component-model");+ faux_feature("incremental-cache");+}++fn faux_feature(feature: &str) {+ println!("cargo:rustc-check-cfg=cfg(feature, values(\"{feature}\"))");+ println!("cargo:rustc-cfg=feature=\"{feature}\"");}diff --git a/crates/cranelift/src/alias_region_key.rs b/crates/cranelift/src/alias_region_key.rsindex 00fd51ebc1..115ccae0b9 100644--- a/crates/cranelift/src/alias_region_key.rs+++ b/crates/cranelift/src/alias_region_key.rs@@ -21,10 +21,6 @@ pub(crate) enum AliasRegionKey {},/// A `VMStoreContext` field access.- #[cfg_attr(- not(feature = "component-model"),- expect(dead_code, reason = "easier not to cfg off")- )]VMStoreContext {/// The offset of the `VMStoreContext` field being accessed.offset: u32,diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rsindex ab9bfc3dc9..9c6c1dd75d 100644--- a/crates/cranelift/src/bounds_checks.rs+++ b/crates/cranelift/src/bounds_checks.rs@@ -53,7 +53,6 @@ pub enum BoundsCheck {/// ```ignore/// index + object_size <= bound/// ```- #[cfg(feature = "gc")]StaticObjectField {offset: u32,access_size: u8,@@ -64,7 +63,6 @@ pub enum BoundsCheck {////// It is *your* responsibility to ensure that the `offset + access_size <=/// object_size` precondition holds.- #[cfg(feature = "gc")]DynamicObjectField {offset: ir::Value,object_size: ir::Value,@@ -91,7 +89,6 @@ pub fn bounds_check_and_compute_addr(access_size,} => bounds_check_field_access(builder, env, heap, index, offset, access_size, trap),- #[cfg(feature = "gc")]BoundsCheck::StaticObjectField {offset,access_size,@@ -129,7 +126,6 @@ pub fn bounds_check_and_compute_addr(// Compute the index of the end of the object, bounds check that and get// a pointer to just after the object, and then reverse offset from that// to get the pointer to the field being accessed.- #[cfg(feature = "gc")]BoundsCheck::DynamicObjectField {offset,object_size,diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 5cb1f76eca..c71e50eaae 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -45,11 +45,9 @@ use wasmtime_environ::{};use wasmtime_unwinder::ExceptionTableBuilder;-#[cfg(feature = "component-model")]pub(crate) mod component;struct IncrementalCacheContext {- #[cfg(feature = "incremental-cache")]cache_store: Arc<dyn CacheStore>,num_hits: usize,num_cached: usize,@@ -87,7 +85,6 @@ pub struct Compiler {linkopts: LinkOptions,cache_store: Option<Arc<dyn CacheStore>>,clif_dir: Option<path::PathBuf>,- #[cfg(feature = "wmemcheck")]pub(crate) wmemcheck: bool,}@@ -138,7 +135,6 @@ impl Compiler {linkopts,cache_store,clif_dir,- #[cfg(feature = "wmemcheck")]wmemcheck,}}@@ -628,7 +624,6 @@ impl wasmtime_environ::Compiler for Compiler {let ty = types[ty].unwrap_func();self.compile_wasm_to_array_trampoline(ty, key, symbol)}- #[cfg(feature = "component-model")]FuncKey::ResourceDropTrampoline => {let ty = types.find_resource_drop_signature().unwrap();let ty = types[ty].unwrap_func();@@ -645,7 +640,6 @@ impl wasmtime_environ::Compiler for Compiler {FuncKey::PulleyHostCall(_) => unreachable!(),- #[cfg(feature = "component-model")]FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => {unreachable!()}@@ -856,7 +850,6 @@ impl wasmtime_environ::Compiler for Compiler {self.isa.is_branch_protection_enabled()}- #[cfg(feature = "component-model")]fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {self}@@ -969,7 +962,6 @@ impl InliningCompiler for Compiler {.map(|name| FuncKey::from_raw_parts(name.namespace, name.index)).filter(|key| match key {FuncKey::DefinedWasmFunction(..) => true,- #[cfg(feature = "component-model")]FuncKey::UnsafeIntrinsic(..) => true,_ => false,}),@@ -1029,7 +1021,6 @@ impl InliningCompiler for Compiler {let callee = FuncKey::from_raw_parts(callee.namespace, callee.index);match callee {FuncKey::DefinedWasmFunction(..) => {}- #[cfg(feature = "component-model")]FuncKey::UnsafeIntrinsic(..) => {}_ => return InlineCommand::KeepCall,}@@ -1094,56 +1085,39 @@ impl InliningCompiler for Compiler {}}-#[cfg(feature = "incremental-cache")]-mod incremental_cache {- use super::*;+struct CraneliftCacheStore(Arc<dyn CacheStore>);- struct CraneliftCacheStore(Arc<dyn CacheStore>);-- impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore {- fn get(&self, key: &[u8]) -> Option<std::borrow::Cow<'_, [u8]>> {- self.0.get(key)- }- fn insert(&mut self, key: &[u8], val: Vec<u8>) {- self.0.insert(key, val);- }+impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore {+ fn get(&self, key: &[u8]) -> Option<std::borrow::Cow<'_, [u8]>> {+ self.0.get(key)}-- pub(super) fn compile_maybe_cached<'a>(- context: &'a mut Context,- isa: &dyn TargetIsa,- cache_ctx: Option<&mut IncrementalCacheContext>,- ) -> Result<CompiledCode, CompileError> {- let cache_ctx = match cache_ctx {- Some(ctx) => ctx,- None => return compile_uncached(context, isa),- };-- let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone());- let (_compiled_code, from_cache) = context- .compile_with_cache(isa, &mut cache_store, &mut Default::default())- .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?;-- if from_cache {- cache_ctx.num_hits += 1;- } else {- cache_ctx.num_cached += 1;- }-- Ok(context.take_compiled_code().unwrap())+ fn insert(&mut self, key: &[u8], val: Vec<u8>) {+ self.0.insert(key, val);}}-#[cfg(feature = "incremental-cache")]-use incremental_cache::*;--#[cfg(not(feature = "incremental-cache"))]fn compile_maybe_cached<'a>(context: &'a mut Context,isa: &dyn TargetIsa,- _cache_ctx: Option<&mut IncrementalCacheContext>,+ cache_ctx: Option<&mut IncrementalCacheContext>,) -> Result<CompiledCode, CompileError> {- compile_uncached(context, isa)+ let cache_ctx = match cache_ctx {+ Some(ctx) => ctx,+ None => return compile_uncached(context, isa),+ };++ let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone());+ let (_compiled_code, from_cache) = context+ .compile_with_cache(isa, &mut cache_store, &mut Default::default())+ .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?;++ if from_cache {+ cache_ctx.num_hits += 1;+ } else {+ cache_ctx.num_cached += 1;+ }++ Ok(context.take_compiled_code().unwrap())}fn compile_uncached<'a>(@@ -1285,7 +1259,6 @@ impl Compiler {ctx}).unwrap_or_else(|| CompilerContext {- #[cfg(feature = "incremental-cache")]incremental_cache_ctx: self.cache_store.as_ref().map(|cache_store| {IncrementalCacheContext {cache_store: cache_store.clone(),diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex c0f2759c77..1b32303155 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -153,21 +153,17 @@ pub struct FuncEnvironment<'module_environment> {/// Translation state at the given point.pub(crate) stacks: FuncTranslationStacks,- #[cfg(feature = "gc")]ty_to_gc_layout: std::collections::HashMap<wasmtime_environ::ModuleInternedTypeIndex,wasmtime_environ::GcLayout,>,- #[cfg(feature = "gc")]gc_heap: Option<Heap>,/// The Cranelift global holding the GC heap's base address.- #[cfg(feature = "gc")]gc_heap_base: Option<ir::GlobalValue>,/// The Cranelift global holding the GC heap's base address.- #[cfg(feature = "gc")]gc_heap_bound: Option<ir::GlobalValue>,translation: &'module_environment ModuleTranslation<'module_environment>,@@ -285,13 +281,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {entities: WasmEntities::default(),stacks: FuncTranslationStacks::new(),- #[cfg(feature = "gc")]ty_to_gc_layout: std::collections::HashMap::new(),- #[cfg(feature = "gc")]gc_heap: None,- #[cfg(feature = "gc")]gc_heap_base: None,- #[cfg(feature = "gc")]gc_heap_bound: None,heaps: PrimaryMap::default(),@@ -441,10 +433,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {self.alias_region(func, key)}- #[cfg_attr(- not(any(feature = "gc-copying", feature = "gc-drc", feature = "gc-null")),- expect(dead_code, reason = "easier not to cfg off; used in upcoming commits")- )]pub(crate) fn vmctx_alias_region(&mut self,func: &mut Function,@@ -453,7 +441,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {self.alias_region(func, AliasRegionKey::VMContext { offset })}- #[cfg(feature = "threads")]fn get_memory_atomic_wait(&mut self, func: &mut Function, ty: ir::Type) -> ir::FuncRef {match ty {I32 => self.builtin_functions.memory_atomic_wait32(func),@@ -773,7 +760,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {self.epoch_check_full(builder, cur_epoch_value, continuation_block);}- #[cfg(feature = "wmemcheck")]fn hook_malloc_exit(&mut self, builder: &mut FunctionBuilder, retvals: &[ir::Value]) {let check_malloc = self.builtin_functions.check_malloc(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());@@ -796,7 +782,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {builder.ins().call(check_malloc, &[vmctx, retval, len]);}- #[cfg(feature = "wmemcheck")]fn hook_free_exit(&mut self, builder: &mut FunctionBuilder) {let check_free = self.builtin_functions.check_free(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());@@ -1078,21 +1063,18 @@ impl<'module_environment> FuncEnvironment<'module_environment> {result_param}- #[cfg(feature = "wmemcheck")]fn check_malloc_start(&mut self, builder: &mut FunctionBuilder) {let malloc_start = self.builtin_functions.malloc_start(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());builder.ins().call(malloc_start, &[vmctx]);}- #[cfg(feature = "wmemcheck")]fn check_free_start(&mut self, builder: &mut FunctionBuilder) {let free_start = self.builtin_functions.free_start(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());builder.ins().call(free_start, &[vmctx]);}- #[cfg(feature = "wmemcheck")]fn current_func_name(&self, builder: &mut FunctionBuilder) -> Option<&str> {let func_index = match &builder.func.name {ir::UserFuncName::User(user) => FuncIndex::from_u32(user.index),@@ -1665,7 +1647,6 @@ impl FuncEnvironment<'_> {let key = match self.translation.known_imported_functions[func_index] {Some(key @ FuncKey::DefinedWasmFunction(..)) => key,- #[cfg(feature = "component-model")]Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,Some(key) => {@@ -1866,7 +1847,6 @@ impl FuncEnvironment<'_> {}/// Get the type index associated with an exception object.- #[cfg(feature = "gc")]pub(crate) fn exception_type_from_tag(&self, tag: TagIndex) -> EngineOrModuleTypeIndex {self.module.tags[tag].exception}@@ -1883,7 +1863,6 @@ impl FuncEnvironment<'_> {/// Get the runtime instance ID and defined-tag ID in that/// instance for a particular static tag ID.- #[cfg(feature = "gc")]pub(crate) fn get_instance_and_tag(&mut self,builder: &mut FunctionBuilder<'_>,@@ -2042,7 +2021,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// The import is always a compile-time builtin intrinsic. Make a// direct call to that function (presumably it will eventually be// inlined).- #[cfg(feature = "component-model")]Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {let callee = self.env@@ -2097,7 +2075,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {/// intrinsics. The fallback of issuing a `call` to the intrinsic is always/// suitable to do and is used in situations where the call instruction may/// have extra context.- #[cfg(feature = "component-model")]fn can_directly_inline_unsafe_intrinsic(&self, abi: wasmtime_environ::Abi) -> bool {abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest}@@ -2303,15 +2280,8 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// Check that they match: in the case of Wasm GC, this means doing a// full subtype check. Otherwise, we do a simple equality check.let matches = if features.gc() {- #[cfg(feature = "gc")]- {- self.env- .is_subtype(self.builder, callee_sig_id, caller_sig_id)- }- #[cfg(not(feature = "gc"))]- {- unreachable!()- }+ self.env+ .is_subtype(self.builder, callee_sig_id, caller_sig_id)} else {self.builder.ins()@@ -5085,30 +5055,20 @@ impl FuncEnvironment<'_> {expected: ir::Value,timeout: ir::Value,) -> WasmResult<ir::Value> {- #[cfg(feature = "threads")]- {- let mut pos = builder.cursor();- let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);- let implied_ty = pos.func.dfg.value_type(expected);- let wait_func = self.get_memory_atomic_wait(&mut pos.func, implied_ty);+ let mut pos = builder.cursor();+ let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);+ let implied_ty = pos.func.dfg.value_type(expected);+ let wait_func = self.get_memory_atomic_wait(&mut pos.func, implied_ty);- let (memory_vmctx, defined_memory_index) =- self.memory_vmctx_and_defined_index(&mut pos, memory_index);+ let (memory_vmctx, defined_memory_index) =+ self.memory_vmctx_and_defined_index(&mut pos, memory_index);- let call_inst = pos.ins().call(- wait_func,- &[memory_vmctx, defined_memory_index, addr, expected, timeout],- );- let ret = pos.func.dfg.inst_results(call_inst)[0];- Ok(builder.ins().ireduce(ir::types::I32, ret))- }- #[cfg(not(feature = "threads"))]- {- let _ = (builder, memory_index, addr, expected, timeout);- Err(wasmtime_environ::WasmError::Unsupported(- "threads support disabled at compile time".to_string(),- ))- }+ let call_inst = pos.ins().call(+ wait_func,+ &[memory_vmctx, defined_memory_index, addr, expected, timeout],+ );+ let ret = pos.func.dfg.inst_results(call_inst)[0];+ Ok(builder.ins().ireduce(ir::types::I32, ret))}pub fn translate_atomic_notify(@@ -5119,28 +5079,18 @@ impl FuncEnvironment<'_> {addr: ir::Value,count: ir::Value,) -> WasmResult<ir::Value> {- #[cfg(feature = "threads")]- {- let mut pos = builder.cursor();- let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);- let atomic_notify = self.builtin_functions.memory_atomic_notify(&mut pos.func);-- let (memory_vmctx, defined_memory_index) =- self.memory_vmctx_and_defined_index(&mut pos, memory_index);- let call_inst = pos.ins().call(- atomic_notify,- &[memory_vmctx, defined_memory_index, addr, count],- );- let ret = pos.func.dfg.inst_results(call_inst)[0];- Ok(builder.ins().ireduce(ir::types::I32, ret))- }- #[cfg(not(feature = "threads"))]- {- let _ = (builder, memory_index, addr, count);- Err(wasmtime_environ::WasmError::Unsupported(- "threads support disabled at compile time".to_string(),- ))- }+ let mut pos = builder.cursor();+ let addr = self.cast_index_to_i64(&mut pos, addr, self.memory(memory_index).idx_type);+ let atomic_notify = self.builtin_functions.memory_atomic_notify(&mut pos.func);++ let (memory_vmctx, defined_memory_index) =+ self.memory_vmctx_and_defined_index(&mut pos, memory_index);+ let call_inst = pos.ins().call(+ atomic_notify,+ &[memory_vmctx, defined_memory_index, addr, count],+ );+ let ret = pos.func.dfg.inst_results(call_inst)[0];+ Ok(builder.ins().ireduce(ir::types::I32, ret))}pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {@@ -5223,7 +5173,6 @@ impl FuncEnvironment<'_> {self.epoch_function_entry(builder);}- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck {let func_name = self.current_func_name(builder);if func_name == Some("malloc") {@@ -5375,7 +5324,6 @@ impl FuncEnvironment<'_> {}pub fn handle_before_return(&mut self, retvals: &[ir::Value], builder: &mut FunctionBuilder) {- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck {let func_name = self.current_func_name(builder);if func_name == Some("malloc") {@@ -5384,8 +5332,6 @@ impl FuncEnvironment<'_> {self.hook_free_exit(builder);}}- #[cfg(not(feature = "wmemcheck"))]- let _ = (retvals, builder);}pub fn before_load(@@ -5395,7 +5341,6 @@ impl FuncEnvironment<'_> {addr: ir::Value,offset: u64,) {- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck {let check_load = self.builtin_functions.check_load(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());@@ -5405,8 +5350,6 @@ impl FuncEnvironment<'_> {.ins().call(check_load, &[vmctx, num_bytes, addr, offset_val]);}- #[cfg(not(feature = "wmemcheck"))]- let _ = (builder, val_size, addr, offset);}pub fn before_store(@@ -5416,7 +5359,6 @@ impl FuncEnvironment<'_> {addr: ir::Value,offset: u64,) {- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck {let check_store = self.builtin_functions.check_store(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());@@ -5426,8 +5368,6 @@ impl FuncEnvironment<'_> {.ins().call(check_store, &[vmctx, num_bytes, addr, offset_val]);}- #[cfg(not(feature = "wmemcheck"))]- let _ = (builder, val_size, addr, offset);}pub fn update_global(@@ -5436,7 +5376,6 @@ impl FuncEnvironment<'_> {global_index: GlobalIndex,value: ir::Value,) {- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck {if global_index.index() == 0 {// We are making the assumption that global 0 is the auxiliary stack pointer.@@ -5446,8 +5385,6 @@ impl FuncEnvironment<'_> {builder.ins().call(update_stack_pointer, &[vmctx, value]);}}- #[cfg(not(feature = "wmemcheck"))]- let _ = (builder, global_index, value);}pub fn before_memory_grow(@@ -5456,14 +5393,11 @@ impl FuncEnvironment<'_> {num_pages: ir::Value,mem_index: MemoryIndex,) {- #[cfg(feature = "wmemcheck")]if self.compiler.wmemcheck && mem_index.as_u32() == 0 {let update_mem_size = self.builtin_functions.update_mem_size(builder.func);let vmctx = self.vmctx_val(&mut builder.cursor());builder.ins().call(update_mem_size, &[vmctx, num_pages]);}- #[cfg(not(feature = "wmemcheck"))]- let _ = (builder, num_pages, mem_index);}/// If the ISA has rounding instructions, let Cranelift use them. But ifdiff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex 7dd7b5905f..cb5ec2bb4f 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -1,35 +1,31 @@//! Interface to compiling GC-related things.-//!-//! This module and its interface are implemented twice: once when the `gc`-//! cargo feature is enabled and once when the feature is disabled. The goal is-//! to have just a single `cfg(feature = "gc")` for the whole crate, which-//! selects between these two implementations.--use crate::func_environ::FuncEnvironment;-use cranelift_codegen::ir;++use crate::TRAP_ARRAY_OUT_OF_BOUNDS;+use crate::alias_region_key::AliasRegionKey;+use crate::bounds_checks::BoundsCheck;+use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};+use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};+use crate::trap::TranslateTrap;+use crate::{Reachability, TRAP_GC_HEAP_CORRUPT, TRAP_INTERNAL_ASSERT};+use cranelift_codegen::ir::immediates::Offset32;+use cranelift_codegen::ir::{BlockArg, ExceptionTableData, ExceptionTableItem};+use cranelift_codegen::{+ cursor::FuncCursor,+ ir::{self, InstBuilder, condcodes::IntCC},+};+use cranelift_entity::packed_option::ReservedValue;use cranelift_frontend::FunctionBuilder;+use smallvec::{SmallVec, smallvec};use wasmtime_environ::{- GcTypeLayouts, TagIndex, TypeIndex, WasmRefType, WasmResult, WasmStorageType,+ Collector, GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, I31_DISCRIMINANT,+ ModuleInternedTypeIndex, PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,+ WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,+ wasm_unsupported,};-#[cfg(feature = "gc")]-mod enabled;-#[cfg(feature = "gc")]-use enabled as imp;--#[cfg(not(feature = "gc"))]-mod disabled;-#[cfg(not(feature = "gc"))]-use disabled as imp;--// Re-export the GC compilation interface from the implementation that we chose-// based on the compile-time features enabled.-pub use imp::*;-/// A trait for different collectors to emit any GC barriers they might require.pub trait GcCompiler {/// Get the GC type layouts for this GC compiler.- #[cfg_attr(not(feature = "gc"), allow(dead_code))]fn layouts(&self) -> &dyn GcTypeLayouts;/// Whether or not this `GcCompiler`'s collector can move objects.@@ -39,7 +35,6 @@ pub trait GcCompiler {////// The array should be of the given type and its elements initialized as/// described by the given `ArrayInit`.- #[cfg_attr(not(feature = "gc"), allow(dead_code))]fn alloc_uninit_array(&mut self,func_env: &mut FuncEnvironment<'_>,@@ -52,7 +47,6 @@ pub trait GcCompiler {////// The struct should be of the given type and its fields initialized to the/// given values.- #[cfg_attr(not(feature = "gc"), allow(dead_code))]fn alloc_struct(&mut self,func_env: &mut FuncEnvironment<'_>,@@ -68,7 +62,6 @@ pub trait GcCompiler {/// uninitialized; that is the responsibility of generated code to/// fill in. `tag_index` is used only to look up the appropriate/// exception object type.- #[cfg_attr(not(feature = "gc"), allow(dead_code))]fn alloc_exn(&mut self,func_env: &mut FuncEnvironment<'_>,@@ -180,7 +173,6 @@ pub trait GcCompiler {////// This initializes a previously-uninitialized field, so barriers on the/// value previously stored at `addr`, if necessary, should not be executed.- #[cfg_attr(not(feature = "gc"), allow(dead_code))]fn init_field(&mut self,func_env: &mut FuncEnvironment<'_>,@@ -190,3 +182,1550 @@ pub trait GcCompiler {val: ir::Value,) -> WasmResult<()>;}++mod copying;+mod drc;+mod null;++/// Get the default GC compiler.+pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {+ // If this function requires a GC compiler, that is not too bad of an+ // over-approximation for it requiring a GC heap.+ func_env.needs_gc_heap = true;++ match func_env.tunables.collector {+ Some(Collector::DeferredReferenceCounting) => Ok(Box::new(drc::DrcCompiler::default())),++ Some(Collector::Null) => Ok(Box::new(null::NullCompiler::default())),++ Some(Collector::Copying) => Ok(Box::new(copying::CopyingCompiler::default())),++ None => Err(wasm_unsupported!(+ "support for GC types disabled at configuration time"+ )),+ }+}++fn unbarriered_load_gc_ref(+ builder: &mut FunctionBuilder,+ ty: WasmHeapType,+ ptr_to_gc_ref: ir::Value,+ flags: ir::MemFlagsData,+) -> WasmResult<ir::Value> {+ debug_assert!(ty.is_vmgcref_type());+ let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);+ if ty != WasmHeapType::I31 {+ builder.declare_value_needs_stack_map(gc_ref);+ }+ Ok(gc_ref)+}++fn unbarriered_store_gc_ref(+ builder: &mut FunctionBuilder,+ ty: WasmHeapType,+ dst: ir::Value,+ gc_ref: ir::Value,+ flags: ir::MemFlagsData,+) -> WasmResult<()> {+ debug_assert!(ty.is_vmgcref_type());+ builder.ins().store(flags, gc_ref, dst, 0);+ Ok(())+}++/// Emit CLIF to call the `gc_alloc_raw` libcall.+fn emit_gc_raw_alloc(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ kind: VMGcKind,+ ty: ModuleInternedTypeIndex,+ size: ir::Value,+ align: u32,+ reserved_bits: u32,+) -> ir::Value {+ let gc_alloc_raw_builtin = func_env.builtin_functions.gc_alloc_raw(builder.func);+ let vmctx = func_env.vmctx_val(&mut builder.cursor());++ let kind = builder+ .ins()+ .iconst(ir::types::I32, i64::from(kind.as_u32() | reserved_bits));++ let ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);++ assert!(align.is_power_of_two());+ let align = builder.ins().iconst(ir::types::I32, i64::from(align));++ let call_inst = builder+ .ins()+ .call(gc_alloc_raw_builtin, &[vmctx, kind, ty, size, align]);++ let gc_ref = builder.func.dfg.first_result(call_inst);+ builder.declare_value_needs_stack_map(gc_ref);+ gc_ref+}++/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the+/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.+///+/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).+fn emit_gc_kind_assert(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ gc_ref: ir::Value,+ expected_kind: VMGcKind,+) {+ if !cfg!(gc_zeal) {+ return;+ }++ func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);++ let kind_addr = func_env.prepare_gc_ref_access(+ builder,+ gc_ref,+ BoundsCheck::StaticObjectField {+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,+ },+ );+ let flags = func_env.gc_memflags(&mut builder.func).with_readonly();+ let kind_and_reserved_bits = builder.ins().load(ir::types::I32, flags, kind_addr, 0);+ let kind_mask = builder+ .ins()+ .iconst(ir::types::I32, i64::from(VMGcKind::MASK));+ let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);++ let expected_kind = builder+ .ins()+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));++ // NB: Do a subtype check rather than a strict equality check. See+ // `VMGcKind::matches` for details.+ let and = builder.ins().band(actual_kind, expected_kind);+ let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);++ builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);+}++/// Read a struct field or array element from its raw address in the GC heap.+///+/// The given address MUST have already been bounds-checked via+/// `prepare_gc_ref_access`.+pub fn read_field_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ty: WasmStorageType,+ addr: ir::Value,+ extension: Option<Extension>,+) -> WasmResult<ir::Value> {+ assert_eq!(extension.is_none(), matches!(ty, WasmStorageType::Val(_)));+ assert_eq!(+ extension.is_some(),+ matches!(ty, WasmStorageType::I8 | WasmStorageType::I16)+ );++ // Data inside GC objects is always little endian.+ let flags = func_env+ .gc_memflags(&mut builder.func)+ .with_endianness(ir::Endianness::Little);++ let value = match ty {+ WasmStorageType::I8 => builder.ins().load(ir::types::I8, flags, addr, 0),+ WasmStorageType::I16 => builder.ins().load(ir::types::I16, flags, addr, 0),+ WasmStorageType::Val(v) => match v {+ WasmValType::I32 => builder.ins().load(ir::types::I32, flags, addr, 0),+ WasmValType::I64 => builder.ins().load(ir::types::I64, flags, addr, 0),+ WasmValType::F32 => builder.ins().load(ir::types::F32, flags, addr, 0),+ WasmValType::F64 => builder.ins().load(ir::types::F64, flags, addr, 0),+ WasmValType::V128 => builder.ins().load(ir::types::I8X16, flags, addr, 0),+ WasmValType::Ref(r) => match r.heap_type.top() {+ WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {+ gc_compiler(func_env)?+ .translate_read_gc_reference(func_env, builder, r, addr, flags)?+ }+ WasmHeapTopType::Func => {+ let expected_ty = match r.heap_type {+ WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),+ WasmHeapType::ConcreteFunc(ty) => ty.unwrap_module_type_index(),+ WasmHeapType::NoFunc => {+ let null = builder.ins().iconst(func_env.pointer_type(), 0);+ if !r.nullable {+ // Because `nofunc` is uninhabited, and this+ // reference is non-null, this is unreachable+ // code. Unconditionally trap via conditional+ // trap instructions to avoid inserting block+ // terminators in the middle of this block.+ builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);+ }+ return Ok(null);+ }+ _ => unreachable!("not a function heap type"),+ };+ let expected_ty = builder+ .ins()+ .iconst(ir::types::I32, i64::from(expected_ty.as_bits()));++ let vmctx = func_env.vmctx_val(&mut builder.cursor());++ let func_ref_id = builder.ins().load(ir::types::I32, flags, addr, 0);+ let get_interned_func_ref = func_env+ .builtin_functions+ .get_interned_func_ref(builder.func);++ let call_inst = builder+ .ins()+ .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);+ builder.func.dfg.first_result(call_inst)+ }+ WasmHeapTopType::Cont => {+ // TODO(#10248) GC integration for stack switching+ return stack_switching_unsupported();+ }+ },+ },+ };++ let value = match extension {+ Some(Extension::Sign) => builder.ins().sextend(ir::types::I32, value),+ Some(Extension::Zero) => builder.ins().uextend(ir::types::I32, value),+ None => value,+ };++ Ok(value)+}++pub fn intern_func_ref(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ref_type: WasmRefType,+ func_ref: ir::Value,+) -> WasmResult<ir::Value> {+ assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);++ let vmctx = func_env.vmctx_val(&mut builder.cursor());++ let intern_func_ref_for_gc_heap = func_env+ .builtin_functions+ .intern_func_ref_for_gc_heap(builder.func);++ let func_ref = if ref_type.heap_type == WasmHeapType::NoFunc {+ let null = builder.ins().iconst(func_env.pointer_type(), 0);+ if !ref_type.nullable {+ // Because `nofunc` is uninhabited, and this reference is+ // non-null, this is unreachable code. Unconditionally trap+ // via conditional trap instructions to avoid inserting+ // block terminators in the middle of this block.+ builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);+ }+ null+ } else {+ func_ref+ };++ // Convert the raw `funcref` into a `FuncRefTableId` for use in the+ // GC heap.+ let call_inst = builder+ .ins()+ .call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);+ let func_ref_id = builder.func.dfg.first_result(call_inst);+ Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))+}++fn write_func_ref_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ref_type: WasmRefType,+ flags: ir::MemFlagsData,+ field_addr: ir::Value,+ func_ref: ir::Value,+) -> WasmResult<()> {+ // Convert the raw `funcref` into a `FuncRefTableId` for use in the+ // GC heap.+ let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;++ // Store the id in the field.+ builder.ins().store(flags, func_ref_id, field_addr, 0);++ Ok(())+}++pub fn init_field_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ field_ty: WasmStorageType,+ field_addr: ir::Value,+ new_val: ir::Value,+) -> WasmResult<()> {+ gc_compiler(func_env)?.init_field(func_env, builder, field_ty, field_addr, new_val)+}++pub fn write_field_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ field_ty: WasmStorageType,+ field_addr: ir::Value,+ new_val: ir::Value,+) -> WasmResult<()> {+ // Data inside GC objects is always little endian.+ let flags = func_env+ .gc_memflags(&mut builder.func)+ .with_endianness(ir::Endianness::Little);++ match field_ty {+ WasmStorageType::I8 => {+ builder.ins().istore8(flags, new_val, field_addr, 0);+ }+ WasmStorageType::I16 => {+ builder.ins().istore16(flags, new_val, field_addr, 0);+ }+ WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {+ WasmHeapTopType::Func => {+ write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?+ }+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {+ gc_compiler(func_env)?.translate_write_gc_reference(+ func_env, builder, r, field_addr, new_val, flags,+ )?;+ }+ WasmHeapTopType::Cont => return stack_switching_unsupported(),+ },+ WasmStorageType::Val(_) => {+ assert_eq!(+ builder.func.dfg.value_type(new_val).bytes(),+ wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty)+ );+ builder.ins().store(flags, new_val, field_addr, 0);+ }+ }+ Ok(())+}++pub fn translate_struct_new(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ struct_type_index: TypeIndex,+ fields: &[ir::Value],+) -> WasmResult<ir::Value> {+ gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)+}++fn default_value(+ cursor: &mut FuncCursor,+ func_env: &FuncEnvironment<'_>,+ ty: &WasmStorageType,+) -> ir::Value {+ match ty {+ WasmStorageType::I8 | WasmStorageType::I16 => cursor.ins().iconst(ir::types::I32, 0),+ WasmStorageType::Val(v) => match v {+ WasmValType::I32 => cursor.ins().iconst(ir::types::I32, 0),+ WasmValType::I64 => cursor.ins().iconst(ir::types::I64, 0),+ WasmValType::F32 => cursor.ins().f32const(0.0),+ WasmValType::F64 => cursor.ins().f64const(0.0),+ WasmValType::V128 => {+ let c = cursor.func.dfg.constants.insert(vec![0; 16].into());+ cursor.ins().vconst(ir::types::I8X16, c)+ }+ WasmValType::Ref(r) => {+ assert!(r.nullable);+ let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);++ // NB: The collector doesn't need to know about null references.+ let _ = needs_stack_map;++ cursor.ins().iconst(ty, 0)+ }+ },+ }+}++pub fn translate_struct_new_default(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ struct_type_index: TypeIndex,+) -> WasmResult<ir::Value> {+ let interned_ty = func_env.module.types[struct_type_index].unwrap_module_type_index();+ let struct_ty = func_env.types.unwrap_struct(interned_ty)?;+ let fields = struct_ty+ .fields+ .iter()+ .map(|f| default_value(&mut builder.cursor(), func_env, &f.element_type))+ .collect::<StructFieldsVec>();+ gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)+}++pub fn translate_struct_get(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ struct_type_index: TypeIndex,+ field_index: u32,+ struct_ref: ir::Value,+ extension: Option<Extension>,+) -> WasmResult<ir::Value> {+ log::trace!(+ "translate_struct_get({struct_type_index:?}, {field_index:?}, {struct_ref:?}, {extension:?})"+ );++ // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a+ // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that+ // type info from `wasmparser` and through to here is a bit funky.+ func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);++ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);++ let field_index = usize::try_from(field_index).unwrap();+ let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();++ let struct_layout = func_env.struct_or_exn_layout(interned_type_index);+ let struct_size = struct_layout.size;++ let field_offset = struct_layout.fields[field_index].offset;+ let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);+ assert!(field_offset + field_size <= struct_size);++ let field_addr = func_env.prepare_gc_ref_access(+ builder,+ struct_ref,+ BoundsCheck::StaticObjectField {+ offset: field_offset,+ access_size: u8::try_from(field_size).unwrap(),+ object_size: struct_size,+ },+ );++ let result = read_field_at_addr(+ func_env,+ builder,+ field_ty.element_type,+ field_addr,+ extension,+ );+ log::trace!("translate_struct_get(..) -> {result:?}");+ result+}++pub fn translate_struct_set(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ struct_type_index: TypeIndex,+ field_index: u32,+ struct_ref: ir::Value,+ new_val: ir::Value,+) -> WasmResult<()> {+ log::trace!(+ "translate_struct_set({struct_type_index:?}, {field_index:?}, struct_ref: {struct_ref:?}, new_val: {new_val:?})"+ );++ // TODO: See comment in `translate_struct_get` about the `trapz`.+ func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);++ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);++ let field_index = usize::try_from(field_index).unwrap();+ let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();++ let struct_layout = func_env.struct_or_exn_layout(interned_type_index);+ let struct_size = struct_layout.size;++ let field_offset = struct_layout.fields[field_index].offset;+ let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);+ assert!(field_offset + field_size <= struct_size);++ let field_addr = func_env.prepare_gc_ref_access(+ builder,+ struct_ref,+ BoundsCheck::StaticObjectField {+ offset: field_offset,+ access_size: u8::try_from(field_size).unwrap(),+ object_size: struct_size,+ },+ );++ write_field_at_addr(+ func_env,+ builder,+ field_ty.element_type,+ field_addr,+ new_val,+ )?;++ log::trace!("translate_struct_set: finished");+ Ok(())+}++pub fn translate_exn_unbox(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ tag_index: TagIndex,+ exn_ref: ir::Value,+) -> WasmResult<SmallVec<[ir::Value; 4]>> {+ log::trace!("translate_exn_unbox({tag_index:?}, {exn_ref:?})");++ // We know that the `exn_ref` is not null because we reach this+ // operation only in catch blocks, and throws are initiated from+ // runtime code that checks for nulls first.++ // Get the GcExceptionLayout associated with this tag's+ // function type, and generate loads for each field.+ let exception_ty_idx = func_env+ .exception_type_from_tag(tag_index)+ .unwrap_module_type_index();+ let exception_ty = func_env.types.unwrap_exn(exception_ty_idx)?;+ let exn_layout = func_env.struct_or_exn_layout(exception_ty_idx);+ let exn_size = exn_layout.size;++ // Gather accesses first because these require a borrow on+ // `func_env`, which we later mutate below via+ // `prepare_gc_ref_access()`.+ let mut accesses: SmallVec<[_; 4]> = smallvec![];+ for (field_ty, field_layout) in exception_ty.fields.iter().zip(exn_layout.fields.iter()) {+ accesses.push((field_layout.offset, field_ty.element_type));+ }++ let mut result = smallvec![];+ for (field_offset, field_ty) in accesses {+ let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty);+ assert!(field_offset + field_size <= exn_size);+ let field_addr = func_env.prepare_gc_ref_access(+ builder,+ exn_ref,+ BoundsCheck::StaticObjectField {+ offset: field_offset,+ access_size: u8::try_from(field_size).unwrap(),+ object_size: exn_size,+ },+ );++ let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None)?;+ result.push(value);+ }++ log::trace!("translate_exn_unbox(..) -> {result:?}");+ Ok(result)+}++pub fn translate_exn_throw(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ tag_index: TagIndex,+ args: &[ir::Value],+) -> WasmResult<()> {+ let (instance_id, defined_tag_id) = func_env.get_instance_and_tag(builder, tag_index);+ let exnref = gc_compiler(func_env)?.alloc_exn(+ func_env,+ builder,+ tag_index,+ args,+ instance_id,+ defined_tag_id,+ )?;+ translate_exn_throw_ref(func_env, builder, exnref)+}++pub fn translate_exn_throw_ref(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ exnref: ir::Value,+) -> WasmResult<()> {+ let builtin = func_env.builtin_functions.throw_ref(builder.func);+ let sig = builder.func.dfg.ext_funcs[builtin].signature;+ let vmctx = func_env.vmctx_val(&mut builder.cursor());++ // Generate a `try_call` with handlers from the current+ // stack. This libcall is unique among libcall implementations of+ // opcodes: we know the others will not throw, but `throw_ref`'s+ // entire purpose is to throw. So if there are any handlers in the+ // local function body, we need to attach them to this callsite+ // like any other.+ let continuation = builder.create_block();+ let current_block = builder.current_block().unwrap();+ builder.insert_block_after(continuation, current_block);+ let continuation_call = builder.func.dfg.block_call(continuation, &[]);+ let mut table_items = vec![ExceptionTableItem::Context(vmctx)];+ for (tag, block) in func_env.stacks.handlers.handlers() {+ let block_call = builder+ .func+ .dfg+ .block_call(block, &[BlockArg::TryCallExn(0)]);+ table_items.push(match tag {+ Some(tag) => ExceptionTableItem::Tag(tag, block_call),+ None => ExceptionTableItem::Default(block_call),+ });+ }+ let etd = ExceptionTableData::new(sig, continuation_call, table_items);+ let et = builder.func.dfg.exception_tables.push(etd);++ builder.ins().try_call(builtin, &[vmctx, exnref], et);++ builder.switch_to_block(continuation);+ builder.seal_block(continuation);+ func_env.trap(builder, crate::TRAP_UNREACHABLE);++ Ok(())+}++pub fn translate_array_new(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ elem: ir::Value,+ len: ir::Value,+) -> WasmResult<ir::Value> {+ log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");+ let result =+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;+ let zero = builder.ins().iconst(ir::types::I32, 0);+ let ty = func_env.module.types[array_type_index].unwrap_module_type_index();+ func_env.translate_entity_fill(+ builder,+ CheckedEntity::Array {+ array: result,+ ty,+ initialized: false,+ },+ zero,+ elem,+ len,+ )?;+ log::trace!("translate_array_new(..) -> {result:?}");+ Ok(result)+}++pub fn translate_array_new_default(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ len: ir::Value,+) -> WasmResult<ir::Value> {+ log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");++ let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();+ let array_ty = func_env.types.unwrap_array(interned_ty)?;+ let result =+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;+ log::trace!("translate_array_new_default(..) -> {result:?}");+ let elem = default_value(&mut builder.cursor(), func_env, &array_ty.0.element_type);+ let zero = builder.ins().iconst(ir::types::I32, 0);+ func_env.translate_entity_fill(+ builder,+ CheckedEntity::Array {+ array: result,+ ty: interned_ty,+ initialized: false,+ },+ zero,+ elem,+ len,+ )?;+ Ok(result)+}++pub fn translate_array_new_fixed(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ elems: &[ir::Value],+) -> WasmResult<ir::Value> {+ log::trace!("translate_array_new_fixed({array_type_index:?}, {elems:?})");+ let len = builder+ .ins()+ .iconst(ir::types::I32, i64::try_from(elems.len()).unwrap());+ let result =+ gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;+ log::trace!("translate_array_new_fixed(..) -> {result:?}");+ let ty = func_env.module.types[array_type_index].unwrap_module_type_index();+ let array_ty = func_env.types.unwrap_array(ty)?;++ for (i, elem) in elems.iter().enumerate() {+ let index = builder+ .ins()+ .iconst(ir::types::I32, i64::try_from(i).unwrap());+ let addr = array_elem_addr(func_env, builder, ty, result, index)?;+ gc_compiler(func_env)?.init_field(+ func_env,+ builder,+ array_ty.0.element_type,+ addr,+ *elem,+ )?;+ }+ Ok(result)+}++pub fn translate_array_len(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_ref: ir::Value,+) -> WasmResult<ir::Value> {+ log::trace!("translate_array_len({array_ref:?})");++ func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE);++ let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();+ let len_field = func_env.prepare_gc_ref_access(+ builder,+ array_ref,+ // Note: We can't bounds check the whole array object's size because we+ // don't know its length yet. Chicken and egg problem.+ BoundsCheck::StaticOffset {+ offset: len_offset,+ access_size: u8::try_from(ir::types::I32.bytes()).unwrap(),+ },+ );+ let flags = func_env.gc_memflags(&mut builder.func).with_readonly();+ let result = builder.ins().load(ir::types::I32, flags, len_field, 0);+ log::trace!("translate_array_len(..) -> {result:?}");+ Ok(result)+}++struct ArraySizeInfo {+ /// The `i32` size of the whole array object, in bytes.+ obj_size: ir::Value,++ /// The `i32` size of each one of the array's elements, in bytes.+ one_elem_size: ir::Value,++ /// The `i32` size of the array's base object, in bytes. This is also the+ /// offset from the start of the array object to its elements.+ base_size: ir::Value,+}++/// Emit code to get the dynamic size (in bytes) of a whole array object, along+/// with some other related bits.+fn emit_array_size_info(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ array_type_index: ModuleInternedTypeIndex,+ // `i32` value containing the array's length.+ array_len: ir::Value,+) -> WasmResult<ArraySizeInfo> {+ let array_layout = func_env.array_layout(array_type_index)?;++ // Note that we check for overflow below because we can't trust the array's+ // length: it came from inside the GC heap.+ //+ // We check for 32-bit multiplication overflow by performing a 64-bit+ // multiplication and testing the high bits.+ let one_elem_size = builder+ .ins()+ .iconst(ir::types::I64, i64::from(array_layout.elem_size));+ let array_len = builder.ins().uextend(ir::types::I64, array_len);+ let all_elems_size = builder.ins().imul(one_elem_size, array_len);++ let high_bits = builder.ins().ushr_imm(all_elems_size, 32);+ builder.ins().trapnz(high_bits, TRAP_GC_HEAP_CORRUPT);++ let all_elems_size = builder.ins().ireduce(ir::types::I32, all_elems_size);+ let base_size = builder+ .ins()+ .iconst(ir::types::I32, i64::from(array_layout.base_size));+ let obj_size =+ builder+ .ins()+ .uadd_overflow_trap(all_elems_size, base_size, TRAP_GC_HEAP_CORRUPT);++ let one_elem_size = builder.ins().ireduce(ir::types::I32, one_elem_size);++ Ok(ArraySizeInfo {+ obj_size,+ one_elem_size,+ base_size,+ })+}++/// Get the bounds-checked address of an element in an array.+///+/// The emitted code will trap if `index >= array.length`.+///+/// Returns the `ir::Value` containing the address of the `index`th element in+/// the array. You may read or write a value of the array's element type at this+/// address. You may not use it for any other kind of access, nor reuse this+/// value across GC safepoints.+fn array_elem_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ array_type_index: ModuleInternedTypeIndex,+ array_ref: ir::Value,+ index: ir::Value,+) -> WasmResult<ir::Value> {+ // First, assert that `index < array.length`.+ //+ // This check is visible at the Wasm-semantics level.+ //+ // TODO: We should emit spectre-safe bounds checks for array accesses (if+ // configured) but we don't currently have a great way to do that here. The+ // proper solution is to use linear memories to back GC heaps and reuse the+ // code in `bounds_check.rs` to implement these bounds checks. That is all+ // planned, but not yet implemented.++ let len = translate_array_len(func_env, builder, array_ref).unwrap();++ let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len);+ func_env.trapz(builder, in_bounds, TRAP_ARRAY_OUT_OF_BOUNDS);++ // Compute the size (in bytes) of the whole array object.+ let ArraySizeInfo {+ obj_size,+ one_elem_size,+ base_size,+ } = emit_array_size_info(func_env, builder, array_type_index, len)?;++ // Compute the offset of the `index`th element within the array object.+ //+ // NB: no need to check for overflow here, since at this point we know that+ // `len * elem_size + base_size` did not overflow and `i < len`.+ let offset_in_elems = builder.ins().imul(index, one_elem_size);+ let offset_in_array = builder.ins().iadd(offset_in_elems, base_size);++ // Finally, use the object size and element offset we just computed to+ // perform our implementation-internal bounds checks.+ //+ // Checking the whole object's size, rather than the `index`th element's+ // size allows these bounds checks to be deduplicated across repeated+ // accesses to the same array at different indices.+ //+ // This check should not be visible to Wasm, and serve to protect us from+ // our own implementation bugs. The goal is to keep any potential widgets+ // confined within the GC heap, and turn what would otherwise be a security+ // vulnerability into a simple bug.+ //+ // TODO: Ideally we should fold the first Wasm-visible bounds check into+ // this internal bounds check, so that we aren't performing multiple,+ // redundant bounds checks. But we should figure out how to do this in a way+ // that doesn't defeat the object-size bounds checking's deduplication+ // mentioned above.+ Ok(func_env.prepare_gc_ref_access(+ builder,+ array_ref,+ BoundsCheck::DynamicObjectField {+ offset: offset_in_array,+ object_size: obj_size,+ },+ ))+}++pub fn translate_array_get(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ array_ref: ir::Value,+ index: ir::Value,+ extension: Option<Extension>,+) -> WasmResult<ir::Value> {+ log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");++ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);++ let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();+ let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;++ let array_ty = func_env.types.unwrap_array(array_type_index)?;+ let elem_ty = array_ty.0.element_type;++ let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension)?;+ log::trace!("translate_array_get(..) -> {result:?}");+ Ok(result)+}++pub fn translate_array_set(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ array_ref: ir::Value,+ index: ir::Value,+ value: ir::Value,+) -> WasmResult<()> {+ log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");++ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);++ let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();+ let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;++ let array_ty = func_env.types.unwrap_array(array_type_index)?;+ let elem_ty = array_ty.0.element_type;++ write_field_at_addr(func_env, builder, elem_ty, elem_addr, value)?;++ log::trace!("translate_array_set: finished");+ Ok(())+}++pub fn translate_ref_test(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ test_ty: WasmRefType,+ val: ir::Value,+ val_ty: WasmRefType,+) -> WasmResult<ir::Value> {+ log::trace!("translate_ref_test({test_ty:?}, {val:?})");++ // First special case: testing for references to bottom types.+ if test_ty.heap_type.is_bottom() {+ let result = if test_ty.nullable {+ // All null references (within the same type hierarchy) match null+ // references to the bottom type.+ func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?+ } else {+ // `ref.test` is always false for non-nullable bottom types, as the+ // bottom types are uninhabited.+ builder.ins().iconst(ir::types::I32, 0)+ };+ log::trace!("translate_ref_test(..) -> {result:?}");+ return Ok(result);+ }++ // And because `ref.test heap_ty` is only valid on operands whose type is in+ // the same type hierarchy as `heap_ty`, if `heap_ty` is its hierarchy's top+ // type, we only need to worry about whether we are testing for nullability+ // or not.+ if test_ty.heap_type.is_top() {+ let result = if test_ty.nullable {+ builder.ins().iconst(ir::types::I32, 1)+ } else {+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;+ let zero = builder.ins().iconst(ir::types::I32, 0);+ let one = builder.ins().iconst(ir::types::I32, 1);+ builder.ins().select(is_null, zero, one)+ };+ log::trace!("translate_ref_test(..) -> {result:?}");+ return Ok(result);+ }++ // `i31ref`s are a little interesting because they don't point to GC+ // objects; we test the bit pattern of the reference itself.+ if test_ty.heap_type == WasmHeapType::I31 {+ let i31_mask = builder.ins().iconst(+ ir::types::I32,+ i64::from(wasmtime_environ::I31_DISCRIMINANT),+ );+ let is_i31 = builder.ins().band(val, i31_mask);+ let result = if test_ty.nullable {+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;+ builder.ins().bor(is_null, is_i31)+ } else {+ is_i31+ };+ log::trace!("translate_ref_test(..) -> {result:?}");+ return Ok(result);+ }++ // Otherwise, in the general case, we need to inspect our given object's+ // actual type, which also requires null-checking and i31-checking it.++ let is_any_hierarchy = test_ty.heap_type.top() == WasmHeapTopType::Any;++ let non_null_block = builder.create_block();+ let non_null_non_i31_block = builder.create_block();+ let continue_block = builder.create_block();++ // Current block: check if the reference is null and branch appropriately.+ let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;+ let result_when_is_null = builder+ .ins()+ .iconst(ir::types::I32, test_ty.nullable as i64);+ builder.ins().brif(+ is_null,+ continue_block,+ &[result_when_is_null.into()],+ non_null_block,+ &[],+ );++ // Non-null block: We know the GC ref is non-null, but we need to also check+ // for `i31` references that don't point to GC objects.+ builder.switch_to_block(non_null_block);+ log::trace!("translate_ref_test: non-null ref block");+ if is_any_hierarchy {+ let i31_mask = builder.ins().iconst(+ ir::types::I32,+ i64::from(wasmtime_environ::I31_DISCRIMINANT),+ );+ let is_i31 = builder.ins().band(val, i31_mask);+ // If it is an `i31`, then create the result value based on whether we+ // want `i31`s to pass the test or not.+ let result_when_is_i31 = builder.ins().iconst(+ ir::types::I32,+ matches!(+ test_ty.heap_type,+ WasmHeapType::Any | WasmHeapType::Eq | WasmHeapType::I31+ ) as i64,+ );+ builder.ins().brif(+ is_i31,+ continue_block,+ &[result_when_is_i31.into()],+ non_null_non_i31_block,+ &[],+ );+ } else {+ // If we aren't testing the `any` hierarchy, the reference cannot be an+ // `i31ref`. Jump directly to the non-null and non-i31 block; rely on+ // branch folding during lowering to clean this up.+ builder.ins().jump(non_null_non_i31_block, &[]);+ }++ // Non-null and non-i31 block: Read the actual `VMGcKind` or+ // `VMSharedTypeIndex` out of the object's header and check whether it+ // matches the expected type.+ builder.switch_to_block(non_null_non_i31_block);+ log::trace!("translate_ref_test: non-null and non-i31 ref block");+ let check_header_kind = |func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ val: ir::Value,+ expected_kind: VMGcKind|+ -> ir::Value {+ let kind_addr = func_env.prepare_gc_ref_access(+ builder,+ val,+ BoundsCheck::StaticObjectField {+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,+ },+ );+ let gc_memflags = func_env.gc_memflags(&mut builder.func);+ let actual_kind =+ builder+ .ins()+ .load(ir::types::I32, gc_memflags.with_readonly(), kind_addr, 0);+ let expected_kind = builder+ .ins()+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));+ // Inline version of `VMGcKind::matches`.+ let and = builder.ins().band(actual_kind, expected_kind);+ let kind_matches = builder+ .ins()+ .icmp(ir::condcodes::IntCC::Equal, and, expected_kind);+ builder.ins().uextend(ir::types::I32, kind_matches)+ };+ let result = match test_ty.heap_type {+ WasmHeapType::Any+ | WasmHeapType::None+ | WasmHeapType::Extern+ | WasmHeapType::NoExtern+ | WasmHeapType::Func+ | WasmHeapType::NoFunc+ | WasmHeapType::Cont+ | WasmHeapType::NoCont+ | WasmHeapType::Exn+ | WasmHeapType::NoExn+ | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"),++ // For these abstract but non-top and non-bottom types, we check the+ // `VMGcKind` that is in the object's header.+ WasmHeapType::Eq => check_header_kind(func_env, builder, val, VMGcKind::EqRef),+ WasmHeapType::Struct => check_header_kind(func_env, builder, val, VMGcKind::StructRef),+ WasmHeapType::Array => check_header_kind(func_env, builder, val, VMGcKind::ArrayRef),++ // For concrete types, we need to do a full subtype check between the+ // `VMSharedTypeIndex` in the object's header and the+ // `ModuleInternedTypeIndex` we have here.+ //+ // TODO: This check should ideally be done inline, but we don't have a+ // good way to access the `TypeRegistry`'s supertypes arrays from Wasm+ // code at the moment.+ WasmHeapType::ConcreteArray(ty)+ | WasmHeapType::ConcreteStruct(ty)+ | WasmHeapType::ConcreteExn(ty) => {+ let expected_interned_ty = ty.unwrap_module_type_index();+ let expected_shared_ty =+ func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);++ let ty_addr = func_env.prepare_gc_ref_access(+ builder,+ val,+ BoundsCheck::StaticOffset {+ offset: wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET,+ access_size: func_env.offsets.size_of_vmshared_type_index(),+ },+ );+ let gc_memflags = func_env.gc_memflags(&mut builder.func);+ let actual_shared_ty =+ builder+ .ins()+ .load(ir::types::I32, gc_memflags.with_readonly(), ty_addr, 0);++ func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)+ }++ // Same as for concrete arrays and structs except that a `VMFuncRef`+ // doesn't begin with a `VMGcHeader` and is a raw pointer rather than GC+ // heap index.+ WasmHeapType::ConcreteFunc(ty) => {+ let expected_interned_ty = ty.unwrap_module_type_index();+ let expected_shared_ty =+ func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);++ let gc_memflags = func_env.gc_memflags(&mut builder.func);+ let actual_shared_ty = func_env.load_funcref_type_index(+ &mut builder.cursor(),+ gc_memflags.with_readonly(),+ val,+ );++ func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)+ }+ WasmHeapType::ConcreteCont(_) => {+ // TODO(#10248) GC integration for stack switching+ return stack_switching_unsupported();+ }+ };+ builder.ins().jump(continue_block, &[result.into()]);++ // Control flow join point with the result.+ builder.switch_to_block(continue_block);+ let result = builder.append_block_param(continue_block, ir::types::I32);+ log::trace!("translate_ref_test(..) -> {result:?}");++ builder.seal_block(non_null_block);+ builder.seal_block(non_null_non_i31_block);+ builder.seal_block(continue_block);++ Ok(result)+}++fn uextend_i32_to_pointer_type(+ builder: &mut FunctionBuilder,+ pointer_type: ir::Type,+ value: ir::Value,+) -> ir::Value {+ assert_eq!(builder.func.dfg.value_type(value), ir::types::I32);+ match pointer_type {+ ir::types::I32 => value,+ ir::types::I64 => builder.ins().uextend(ir::types::I64, value),+ _ => unreachable!(),+ }+}++/// Emit CLIF to compute an array object's total size, given the dynamic length+/// in its initialization.+///+/// Traps if the size overflows.+fn emit_array_size(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ array_layout: &GcArrayLayout,+ len: ir::Value,+) -> ir::Value {+ let base_size = builder+ .ins()+ .iconst(ir::types::I32, i64::from(array_layout.base_size));++ // `elems_size = len * elem_size`+ //+ // Check for multiplication overflow and trap if it occurs, since that+ // means Wasm is attempting to allocate an array that is larger than our+ // implementation limits. (Note: there is no standard implementation+ // limit for array length beyond `u32::MAX`.)+ //+ // We implement this check by encoding our logically-32-bit operands as+ // i64 values, doing a 64-bit multiplication, and then checking the high+ // 32 bits of the multiplication's result. If the high 32 bits are not+ // all zeros, then the multiplication overflowed.+ debug_assert_eq!(builder.func.dfg.value_type(len), ir::types::I32);+ let len = builder.ins().uextend(ir::types::I64, len);+ let elems_size_64 = builder+ .ins()+ .imul_imm(len, i64::from(array_layout.elem_size));+ let high_bits = builder.ins().ushr_imm(elems_size_64, 32);+ func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE);+ let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64);++ // And if adding the base size and elements size overflows, then the+ // allocation is too large.+ let size = func_env.uadd_overflow_trap(+ builder,+ base_size,+ elems_size,+ crate::TRAP_ALLOCATION_TOO_LARGE,+ );++ size+}++/// Common helper for struct-field initialization that can be reused across+/// collectors.+fn initialize_struct_fields(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ struct_ty: ModuleInternedTypeIndex,+ raw_ptr_to_struct: ir::Value,+ field_values: &[ir::Value],+) -> WasmResult<()> {+ let struct_layout = func_env.struct_or_exn_layout(struct_ty);+ let struct_size = struct_layout.size;+ let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().map(|f| f.offset).collect();+ assert_eq!(field_offsets.len(), field_values.len());++ assert!(!func_env.types[struct_ty].composite_type.shared);+ let fields = match &func_env.types[struct_ty].composite_type.inner {+ WasmCompositeInnerType::Struct(s) => &s.fields,+ WasmCompositeInnerType::Exn(e) => &e.fields,+ _ => panic!("Not a struct or exception type"),+ };++ let field_types: SmallVec<[_; 8]> = fields.iter().cloned().collect();+ assert_eq!(field_types.len(), field_values.len());++ for ((ty, val), offset) in field_types.into_iter().zip(field_values).zip(field_offsets) {+ let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty.element_type);+ assert!(offset + size_of_access <= struct_size);+ let field_addr = builder.ins().iadd_imm(raw_ptr_to_struct, i64::from(offset));+ gc_compiler(func_env)?.init_field(func_env, builder, ty.element_type, field_addr, *val)?;+ }++ Ok(())+}++impl FuncEnvironment<'_> {+ pub(crate) fn gc_heap_alias_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.alias_region(func, AliasRegionKey::GcHeap)+ }++ /// Flags to use for general-purpose GC loads/stores.+ ///+ /// This is used for accesses to the GC heap which aren't expected to trap, but+ /// retain internal assertion metadata to report if such a trap happens. This+ /// is here to ensure that in the face of heap corruption that there's no+ /// possible UB within Cranelift and/or the runtime.+ fn gc_memflags(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {+ let region = self.gc_heap_alias_region(func);+ ir::MemFlagsData::new()+ .with_trap_code(Some(crate::TRAP_GC_HEAP_CORRUPT))+ .with_alias_region(Some(region))+ }++ fn gc_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcLayout {+ // Lazily compute and cache the layout.+ if !self.ty_to_gc_layout.contains_key(&type_index) {+ let ty = &self.types[type_index].composite_type;+ let layout = gc_compiler(self)+ .unwrap()+ .layouts()+ .gc_layout(ty)+ .expect("gc_layout should not OOM at compile time")+ .expect("should only call `FuncEnvironment::gc_layout` for GC types");+ self.ty_to_gc_layout.insert(type_index, layout);+ }++ self.ty_to_gc_layout.get(&type_index).unwrap()+ }++ /// Get the `GcArrayLayout` for the array type at the given `type_index`.+ pub(crate) fn array_layout(+ &mut self,+ type_index: ModuleInternedTypeIndex,+ ) -> WasmResult<&GcArrayLayout> {+ Ok(self.gc_layout(type_index).unwrap_array())+ }++ /// Get the `GcStructLayout` for the struct or exception type at the given `type_index`.+ fn struct_or_exn_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcStructLayout {+ let result = self.gc_layout(type_index).unwrap_struct();+ result+ }++ /// Get or create the global for our GC heap's base pointer.+ fn get_gc_heap_base_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {+ if let Some(base) = self.gc_heap_base {+ return base;+ }++ let store_context_ptr = self.get_vmstore_context_ptr_global(func);+ let offset = self.offsets.ptr.vmstore_context_gc_heap_base();++ let mut flags = ir::MemFlagsData::trusted();+ let memory_tunables =+ wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);+ if !self+ .tunables+ .gc_heap_memory_type()+ .memory_may_move(&memory_tunables)+ {+ flags.set_readonly();+ flags.set_can_move();+ }++ let base_flags = func.dfg.mem_flags.insert(flags).unwrap();+ let base = func.create_global_value(ir::GlobalValueData::Load {+ base: store_context_ptr,+ offset: Offset32::new(offset.into()),+ global_type: self.pointer_type(),+ flags: base_flags,+ });++ self.gc_heap_base = Some(base);+ base+ }++ /// Get the GC heap's base.+ pub(crate) fn get_gc_heap_base(+ &mut self,+ builder: &mut FunctionBuilder,+ ) -> WasmResult<ir::Value> {+ let global = self.get_gc_heap_base_global(&mut builder.func);+ Ok(builder.ins().global_value(self.pointer_type(), global))+ }++ fn get_gc_heap_bound_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {+ if let Some(bound) = self.gc_heap_bound {+ return bound;+ }+ let store_context_ptr = self.get_vmstore_context_ptr_global(func);+ let offset = self.offsets.ptr.vmstore_context_gc_heap_current_length();+ let bound_flags = func+ .dfg+ .mem_flags+ .insert(ir::MemFlagsData::trusted())+ .unwrap();+ let bound = func.create_global_value(ir::GlobalValueData::Load {+ base: store_context_ptr,+ offset: Offset32::new(offset.into()),+ global_type: self.pointer_type(),+ flags: bound_flags,+ });+ self.gc_heap_bound = Some(bound);+ bound+ }++ /// Get the GC heap's bound.+ pub(crate) fn get_gc_heap_bound(+ &mut self,+ builder: &mut FunctionBuilder,+ ) -> WasmResult<ir::Value> {+ let global = self.get_gc_heap_bound_global(&mut builder.func);+ Ok(builder.ins().global_value(self.pointer_type(), global))+ }++ /// Get or create the `Heap` for our GC heap.+ fn get_gc_heap(&mut self, func: &mut ir::Function) -> Heap {+ if let Some(heap) = self.gc_heap {+ return heap;+ }++ let base = self.get_gc_heap_base_global(func);+ let bound = self.get_gc_heap_bound_global(func);+ let memory = self.tunables.gc_heap_memory_type();+ let heap = self.heaps.push(HeapData {+ base,+ bound,+ memory,+ kind: MemoryKind::GcHeap,+ });+ self.gc_heap = Some(heap);+ heap+ }++ /// Get the raw pointer of `gc_ref[offset]` bounds checked for an access of+ /// `size` bytes.+ ///+ /// The given `gc_ref` must be a non-null, non-i31 GC reference.+ ///+ /// If `check` is a `BoundsCheck::Object`, then it is the callers+ /// responsibility to ensure that `offset + access_size <= object_size`.+ ///+ /// Returns a raw pointer to `gc_ref[offset]` -- not a raw pointer to the GC+ /// object itself (unless `offset` happens to be `0`). This raw pointer may+ /// be used to read or write up to as many bytes as described by `bound`. Do+ /// NOT attempt accesses bytes outside of `bound`; that may lead to+ /// unchecked out-of-bounds accesses.+ ///+ /// This method is collector-agnostic.+ fn prepare_gc_ref_access(+ &mut self,+ builder: &mut FunctionBuilder,+ gc_ref: ir::Value,+ bounds_check: BoundsCheck,+ ) -> ir::Value {+ log::trace!("prepare_gc_ref_access({gc_ref:?}, {bounds_check:?})");+ assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);++ let gc_heap = self.get_gc_heap(&mut builder.func);+ let gc_heap = self.heaps[gc_heap].clone();+ let result = match crate::bounds_checks::bounds_check_and_compute_addr(+ builder,+ self,+ &gc_heap,+ gc_ref,+ bounds_check,+ crate::TRAP_GC_HEAP_CORRUPT,+ ) {+ Reachability::Reachable(v) => v,+ Reachability::Unreachable => {+ // We are now in unreachable code, but we don't want to plumb+ // through a bunch of `Reachability` through all of our callers,+ // so just assert we won't reach here and return `null`+ let null = builder.ins().iconst(self.pointer_type(), 0);+ builder.ins().trapz(null, crate::TRAP_INTERNAL_ASSERT);+ null+ }+ };+ log::trace!("prepare_gc_ref_access(..) -> {result:?}");+ result+ }++ /// Emit checks (if necessary) for whether the given `gc_ref` is null or is+ /// an `i31ref`.+ ///+ /// Takes advantage of static information based on `ty` as to whether the GC+ /// reference is nullable or can ever be an `i31`.+ ///+ /// Returns an `ir::Value` that is an `i32` will be non-zero if the GC+ /// reference is null or is an `i31ref`; otherwise, it will be zero.+ ///+ /// This method is collector-agnostic.+ fn gc_ref_is_null_or_i31(+ &mut self,+ builder: &mut FunctionBuilder,+ ty: WasmRefType,+ gc_ref: ir::Value,+ ) -> ir::Value {+ assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);+ assert!(ty.is_vmgcref_type_and_not_i31());++ let might_be_i31 = match ty.heap_type {+ // If we are definitely dealing with an i31, we shouldn't be+ // emitting dynamic checks for it, and the caller shouldn't call+ // this function. Should have been caught by the assertion at the+ // start of the function.+ WasmHeapType::I31 => unreachable!(),++ // Could potentially be an i31.+ WasmHeapType::Any | WasmHeapType::Eq => true,++ // If it is definitely a struct, array, or uninhabited type, then it+ // is definitely not an i31.+ WasmHeapType::Array+ | WasmHeapType::ConcreteArray(_)+ | WasmHeapType::Struct+ | WasmHeapType::ConcreteStruct(_)+ | WasmHeapType::None => false,++ // Despite being a different type hierarchy, this *could* be an+ // `i31` if it is the result of+ //+ // (extern.convert_any (ref.i31 ...))+ WasmHeapType::Extern => true,++ // Can only ever be `null`.+ WasmHeapType::NoExtern => false,++ WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn => false,++ // Wrong type hierarchy, and also funcrefs are not GC-managed+ // types. Should have been caught by the assertion at the start of+ // the function.+ WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => {+ unreachable!()+ }+ WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => {+ unreachable!()+ }+ };++ match (ty.nullable, might_be_i31) {+ // This GC reference statically cannot be null nor an i31. (Let+ // Cranelift's optimizer const-propagate this value and erase any+ // unnecessary control flow resulting from branching on this value.)+ (false, false) => builder.ins().iconst(ir::types::I32, 0),++ // This GC reference is always non-null, but might be an i31.+ (false, true) => builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT)),++ // This GC reference might be null, but can never be an i31.+ (true, false) => builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0),++ // Fully general case: this GC reference could be either null or an+ // i31.+ (true, true) => {+ let is_i31 = builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT));+ let is_null = builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0);+ let is_null = builder.ins().uextend(ir::types::I32, is_null);+ builder.ins().bor(is_i31, is_null)+ }+ }+ }++ // Emit code to check whether `a <: b` for two `VMSharedTypeIndex`es.+ pub(crate) fn is_subtype(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ a: ir::Value,+ b: ir::Value,+ ) -> ir::Value {+ log::trace!("is_subtype({a:?}, {b:?})");++ let diff_tys_block = builder.create_block();+ let continue_block = builder.create_block();++ // Current block: fast path for when `a == b`.+ log::trace!("is_subtype: fast path check for exact same types");+ let same_ty = builder.ins().icmp(IntCC::Equal, a, b);+ let same_ty = builder.ins().uextend(ir::types::I32, same_ty);+ builder.ins().brif(+ same_ty,+ continue_block,+ &[same_ty.into()],+ diff_tys_block,+ &[],+ );++ // Different types block: fall back to the `is_subtype` libcall.+ builder.switch_to_block(diff_tys_block);+ log::trace!("is_subtype: slow path to do full `is_subtype` libcall");+ let is_subtype = self.builtin_functions.is_subtype(builder.func);+ let vmctx = self.vmctx_val(&mut builder.cursor());+ let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]);+ let result = builder.func.dfg.first_result(call_inst);+ builder.ins().jump(continue_block, &[result.into()]);++ // Continue block: join point for the result.+ builder.switch_to_block(continue_block);+ let result = builder.append_block_param(continue_block, ir::types::I32);+ log::trace!("is_subtype(..) -> {result:?}");++ builder.seal_block(diff_tys_block);+ builder.seal_block(continue_block);++ result+ }+}++fn stack_switching_unsupported<T>() -> WasmResult<T> {+ Err(wasmtime_environ::WasmError::Unsupported(+ "Stack switching feature not compatible with GC, yet".to_string(),+ ))+}++pub fn translate_array_new_entity(+ env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ array_type_index: TypeIndex,+ entity: CheckedEntity,+ entity_offset: ir::Value,+ len: ir::Value,+) -> WasmResult<ir::Value> {+ // Before actually allocating this array first do a bounds-check on the+ // passive entity itself.+ let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();+ env.translate_entity_bounds_check(builder, entity, entity_offset, len)?;++ let array = gc_compiler(env)?.alloc_uninit_array(env, builder, array_type_index, len)?;+ let dst = builder.ins().iconst(ir::types::I32, 0);+ env.translate_entity_copy(+ builder,+ CheckedEntity::Array {+ array,+ ty: interned_type_index,+ initialized: false,+ },+ entity,+ dst,+ entity_offset,+ len,+ )?;++ Ok(array)+}diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rssimilarity index 100%rename from crates/cranelift/src/func_environ/gc/enabled/copying.rsrename to crates/cranelift/src/func_environ/gc/copying.rsdiff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rsdeleted file mode 100644index e659ff6f74..0000000000--- a/crates/cranelift/src/func_environ/gc/disabled.rs+++ /dev/null@@ -1,229 +0,0 @@-//! `GcCompiler` implementation when GC support is disabled.--use super::GcCompiler;-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};-use cranelift_codegen::ir;-use cranelift_frontend::FunctionBuilder;-use smallvec::SmallVec;-use wasmtime_environ::{- GcArrayLayout, ModuleInternedTypeIndex, TagIndex, TypeIndex, WasmRefType, WasmResult,- WasmStorageType, wasm_unsupported,-};--fn disabled<T>() -> WasmResult<T> {- Err(wasm_unsupported!(- "support for Wasm GC disabled at compile time because the `gc` cargo \- feature was not enabled"- ))-}--/// Get the default GC compiler.-pub fn gc_compiler(_: &FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {- disabled()-}--pub fn translate_struct_new(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _struct_type_index: TypeIndex,- _fields: &[ir::Value],-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_struct_new_default(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _struct_type_index: TypeIndex,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_struct_get(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _struct_type_index: TypeIndex,- _field_index: u32,- _struct_ref: ir::Value,- _extension: Option<Extension>,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_struct_set(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _struct_type_index: TypeIndex,- _field_index: u32,- _struct_ref: ir::Value,- _new_val: ir::Value,-) -> WasmResult<()> {- disabled()-}--pub fn translate_exn_unbox(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _tag_index: TagIndex,- _exn_ref: ir::Value,-) -> WasmResult<SmallVec<[ir::Value; 4]>> {- disabled()-}--pub fn translate_exn_throw(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _tag_index: TagIndex,- _args: &[ir::Value],-) -> WasmResult<()> {- disabled()-}--pub fn translate_exn_throw_ref(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _exnref: ir::Value,-) -> WasmResult<()> {- disabled()-}--pub fn translate_array_new(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _elem: ir::Value,- _len: ir::Value,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_new_default(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _len: ir::Value,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_new_fixed(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _elems: &[ir::Value],-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_len(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array: ir::Value,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_get(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _array: ir::Value,- _index: ir::Value,- _extension: Option<Extension>,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_set(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _array: ir::Value,- _index: ir::Value,- _value: ir::Value,-) -> WasmResult<()> {- disabled()-}--pub fn translate_ref_test(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _test_ty: WasmRefType,- _val: ir::Value,- _val_ty: WasmRefType,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn translate_array_new_entity(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder,- _array_type_index: TypeIndex,- _entity: CheckedEntity,- _data_offset: ir::Value,- _len: ir::Value,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn read_field_at_addr(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _ty: WasmStorageType,- _addr: ir::Value,- _extension: Option<Extension>,-) -> WasmResult<ir::Value> {- disabled()-}--pub fn write_field_at_addr(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _field_ty: WasmStorageType,- _field_addr: ir::Value,- _new_val: ir::Value,-) -> WasmResult<()> {- disabled()-}--pub fn init_field_at_addr(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _field_ty: WasmStorageType,- _field_addr: ir::Value,- _new_val: ir::Value,-) -> WasmResult<()> {- disabled()-}--impl FuncEnvironment<'_> {- pub(crate) fn array_layout(- &mut self,- _type_index: ModuleInternedTypeIndex,- ) -> WasmResult<&GcArrayLayout> {- disabled()- }-- pub(crate) fn get_gc_heap_base(- &mut self,- _builder: &mut FunctionBuilder<'_>,- ) -> WasmResult<ir::Value> {- disabled()- }-- pub(crate) fn get_gc_heap_bound(- &mut self,- _builder: &mut FunctionBuilder<'_>,- ) -> WasmResult<ir::Value> {- disabled()- }-}--pub fn intern_func_ref(- _func_env: &mut FuncEnvironment<'_>,- _builder: &mut FunctionBuilder<'_>,- _ref_type: WasmRefType,- _func_ref: ir::Value,-) -> WasmResult<ir::Value> {- disabled()-}diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rssimilarity index 100%rename from crates/cranelift/src/func_environ/gc/enabled/drc.rsrename to crates/cranelift/src/func_environ/gc/drc.rsdiff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rsdeleted file mode 100644index 68a6cc33a0..0000000000--- a/crates/cranelift/src/func_environ/gc/enabled.rs+++ /dev/null@@ -1,1620 +0,0 @@-use super::GcCompiler;-use crate::TRAP_ARRAY_OUT_OF_BOUNDS;-use crate::alias_region_key::AliasRegionKey;-use crate::bounds_checks::BoundsCheck;-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};-use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};-use crate::trap::TranslateTrap;-use crate::{Reachability, TRAP_GC_HEAP_CORRUPT, TRAP_INTERNAL_ASSERT};-use cranelift_codegen::ir::immediates::Offset32;-use cranelift_codegen::ir::{BlockArg, ExceptionTableData, ExceptionTableItem};-use cranelift_codegen::{- cursor::FuncCursor,- ir::{self, InstBuilder, condcodes::IntCC},-};-use cranelift_entity::packed_option::ReservedValue;-use cranelift_frontend::FunctionBuilder;-use smallvec::{SmallVec, smallvec};-use wasmtime_environ::{- Collector, GcArrayLayout, GcLayout, GcStructLayout, I31_DISCRIMINANT, ModuleInternedTypeIndex,- PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType, WasmHeapTopType, WasmHeapType,- WasmRefType, WasmResult, WasmStorageType, WasmValType, wasm_unsupported,-};--#[cfg(feature = "gc-drc")]-mod drc;-#[cfg(feature = "gc-null")]-mod null;--#[cfg(feature = "gc-copying")]-mod copying;--/// Get the default GC compiler.-pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcCompiler>> {- // If this function requires a GC compiler, that is not too bad of an- // over-approximation for it requiring a GC heap.- func_env.needs_gc_heap = true;-- match func_env.tunables.collector {- #[cfg(feature = "gc-drc")]- Some(Collector::DeferredReferenceCounting) => Ok(Box::new(drc::DrcCompiler::default())),- #[cfg(not(feature = "gc-drc"))]- Some(Collector::DeferredReferenceCounting) => Err(wasm_unsupported!(- "the DRC collector is unavailable because the `gc-drc` feature \- was disabled at compile time",- )),-- #[cfg(feature = "gc-null")]- Some(Collector::Null) => Ok(Box::new(null::NullCompiler::default())),- #[cfg(not(feature = "gc-null"))]- Some(Collector::Null) => Err(wasm_unsupported!(- "the null collector is unavailable because the `gc-null` feature \- was disabled at compile time",- )),-- #[cfg(feature = "gc-copying")]- Some(Collector::Copying) => Ok(Box::new(copying::CopyingCompiler::default())),- #[cfg(not(feature = "gc-copying"))]- Some(Collector::Copying) => Err(wasm_unsupported!(- "the copying collector is unavailable because the `gc-copying` feature \- was disabled at compile time",- )),-- #[cfg(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying"))]- None => Err(wasm_unsupported!(- "support for GC types disabled at configuration time"- )),- #[cfg(not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")))]- None => Err(wasm_unsupported!(- "support for GC types disabled because no collector implementation \- was selected at compile time; enable one of the `gc-drc` or \- `gc-null` features",- )),- }-}--#[cfg_attr(- not(any(feature = "gc-drc", feature = "gc-copying")),- expect(dead_code, reason = "easier to define")-)]-fn unbarriered_load_gc_ref(- builder: &mut FunctionBuilder,- ty: WasmHeapType,- ptr_to_gc_ref: ir::Value,- flags: ir::MemFlagsData,-) -> WasmResult<ir::Value> {- debug_assert!(ty.is_vmgcref_type());- let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);- if ty != WasmHeapType::I31 {- builder.declare_value_needs_stack_map(gc_ref);- }- Ok(gc_ref)-}--#[cfg_attr(- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),- expect(dead_code, reason = "easier to define")-)]-fn unbarriered_store_gc_ref(- builder: &mut FunctionBuilder,- ty: WasmHeapType,- dst: ir::Value,- gc_ref: ir::Value,- flags: ir::MemFlagsData,-) -> WasmResult<()> {- debug_assert!(ty.is_vmgcref_type());- builder.ins().store(flags, gc_ref, dst, 0);- Ok(())-}--/// Emit CLIF to call the `gc_alloc_raw` libcall.-#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]-fn emit_gc_raw_alloc(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- kind: VMGcKind,- ty: ModuleInternedTypeIndex,- size: ir::Value,- align: u32,- reserved_bits: u32,-) -> ir::Value {- let gc_alloc_raw_builtin = func_env.builtin_functions.gc_alloc_raw(builder.func);- let vmctx = func_env.vmctx_val(&mut builder.cursor());-- let kind = builder- .ins()- .iconst(ir::types::I32, i64::from(kind.as_u32() | reserved_bits));-- let ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);-- assert!(align.is_power_of_two());- let align = builder.ins().iconst(ir::types::I32, i64::from(align));-- let call_inst = builder- .ins()- .call(gc_alloc_raw_builtin, &[vmctx, kind, ty, size, align]);-- let gc_ref = builder.func.dfg.first_result(call_inst);- builder.declare_value_needs_stack_map(gc_ref);- gc_ref-}--/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the-/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.-///-/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).-fn emit_gc_kind_assert(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- gc_ref: ir::Value,- expected_kind: VMGcKind,-) {- if !cfg!(gc_zeal) {- return;- }-- func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);-- let kind_addr = func_env.prepare_gc_ref_access(- builder,- gc_ref,- BoundsCheck::StaticObjectField {- offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,- access_size: wasmtime_environ::VM_GC_KIND_SIZE,- object_size: wasmtime_environ::VM_GC_HEADER_SIZE,- },- );- let flags = func_env.gc_memflags(&mut builder.func).with_readonly();- let kind_and_reserved_bits = builder.ins().load(ir::types::I32, flags, kind_addr, 0);- let kind_mask = builder- .ins()- .iconst(ir::types::I32, i64::from(VMGcKind::MASK));- let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);-- let expected_kind = builder- .ins()- .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));-- // NB: Do a subtype check rather than a strict equality check. See- // `VMGcKind::matches` for details.- let and = builder.ins().band(actual_kind, expected_kind);- let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);-- builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);-}--/// Read a struct field or array element from its raw address in the GC heap.-///-/// The given address MUST have already been bounds-checked via-/// `prepare_gc_ref_access`.-pub fn read_field_at_addr(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- ty: WasmStorageType,- addr: ir::Value,- extension: Option<Extension>,-) -> WasmResult<ir::Value> {- assert_eq!(extension.is_none(), matches!(ty, WasmStorageType::Val(_)));- assert_eq!(- extension.is_some(),- matches!(ty, WasmStorageType::I8 | WasmStorageType::I16)- );-- // Data inside GC objects is always little endian.- let flags = func_env- .gc_memflags(&mut builder.func)- .with_endianness(ir::Endianness::Little);-- let value = match ty {- WasmStorageType::I8 => builder.ins().load(ir::types::I8, flags, addr, 0),- WasmStorageType::I16 => builder.ins().load(ir::types::I16, flags, addr, 0),- WasmStorageType::Val(v) => match v {- WasmValType::I32 => builder.ins().load(ir::types::I32, flags, addr, 0),- WasmValType::I64 => builder.ins().load(ir::types::I64, flags, addr, 0),- WasmValType::F32 => builder.ins().load(ir::types::F32, flags, addr, 0),- WasmValType::F64 => builder.ins().load(ir::types::F64, flags, addr, 0),- WasmValType::V128 => builder.ins().load(ir::types::I8X16, flags, addr, 0),- WasmValType::Ref(r) => match r.heap_type.top() {- WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {- gc_compiler(func_env)?- .translate_read_gc_reference(func_env, builder, r, addr, flags)?- }- WasmHeapTopType::Func => {- let expected_ty = match r.heap_type {- WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),- WasmHeapType::ConcreteFunc(ty) => ty.unwrap_module_type_index(),- WasmHeapType::NoFunc => {- let null = builder.ins().iconst(func_env.pointer_type(), 0);- if !r.nullable {- // Because `nofunc` is uninhabited, and this- // reference is non-null, this is unreachable- // code. Unconditionally trap via conditional- // trap instructions to avoid inserting block- // terminators in the middle of this block.- builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);- }- return Ok(null);- }- _ => unreachable!("not a function heap type"),- };- let expected_ty = builder- .ins()- .iconst(ir::types::I32, i64::from(expected_ty.as_bits()));-- let vmctx = func_env.vmctx_val(&mut builder.cursor());-- let func_ref_id = builder.ins().load(ir::types::I32, flags, addr, 0);- let get_interned_func_ref = func_env- .builtin_functions- .get_interned_func_ref(builder.func);-- let call_inst = builder- .ins()- .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);- builder.func.dfg.first_result(call_inst)- }- WasmHeapTopType::Cont => {- // TODO(#10248) GC integration for stack switching- return stack_switching_unsupported();- }- },- },- };-- let value = match extension {- Some(Extension::Sign) => builder.ins().sextend(ir::types::I32, value),- Some(Extension::Zero) => builder.ins().uextend(ir::types::I32, value),- None => value,- };-- Ok(value)-}--pub fn intern_func_ref(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- ref_type: WasmRefType,- func_ref: ir::Value,-) -> WasmResult<ir::Value> {- assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);-- let vmctx = func_env.vmctx_val(&mut builder.cursor());-- let intern_func_ref_for_gc_heap = func_env- .builtin_functions- .intern_func_ref_for_gc_heap(builder.func);-- let func_ref = if ref_type.heap_type == WasmHeapType::NoFunc {- let null = builder.ins().iconst(func_env.pointer_type(), 0);- if !ref_type.nullable {- // Because `nofunc` is uninhabited, and this reference is- // non-null, this is unreachable code. Unconditionally trap- // via conditional trap instructions to avoid inserting- // block terminators in the middle of this block.- builder.ins().trapz(null, TRAP_INTERNAL_ASSERT);- }- null- } else {- func_ref- };-- // Convert the raw `funcref` into a `FuncRefTableId` for use in the- // GC heap.- let call_inst = builder- .ins()- .call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);- let func_ref_id = builder.func.dfg.first_result(call_inst);- Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))-}--fn write_func_ref_at_addr(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- ref_type: WasmRefType,- flags: ir::MemFlagsData,- field_addr: ir::Value,- func_ref: ir::Value,-) -> WasmResult<()> {- // Convert the raw `funcref` into a `FuncRefTableId` for use in the- // GC heap.- let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;-- // Store the id in the field.- builder.ins().store(flags, func_ref_id, field_addr, 0);-- Ok(())-}--pub fn init_field_at_addr(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- field_ty: WasmStorageType,- field_addr: ir::Value,- new_val: ir::Value,-) -> WasmResult<()> {- gc_compiler(func_env)?.init_field(func_env, builder, field_ty, field_addr, new_val)-}--pub fn write_field_at_addr(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- field_ty: WasmStorageType,- field_addr: ir::Value,- new_val: ir::Value,-) -> WasmResult<()> {- // Data inside GC objects is always little endian.- let flags = func_env- .gc_memflags(&mut builder.func)- .with_endianness(ir::Endianness::Little);-- match field_ty {- WasmStorageType::I8 => {- builder.ins().istore8(flags, new_val, field_addr, 0);- }- WasmStorageType::I16 => {- builder.ins().istore16(flags, new_val, field_addr, 0);- }- WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {- WasmHeapTopType::Func => {- write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?- }- WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {- gc_compiler(func_env)?.translate_write_gc_reference(- func_env, builder, r, field_addr, new_val, flags,- )?;- }- WasmHeapTopType::Cont => return stack_switching_unsupported(),- },- WasmStorageType::Val(_) => {- assert_eq!(- builder.func.dfg.value_type(new_val).bytes(),- wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty)- );- builder.ins().store(flags, new_val, field_addr, 0);- }- }- Ok(())-}--pub fn translate_struct_new(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- struct_type_index: TypeIndex,- fields: &[ir::Value],-) -> WasmResult<ir::Value> {- gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)-}--fn default_value(- cursor: &mut FuncCursor,- func_env: &FuncEnvironment<'_>,- ty: &WasmStorageType,-) -> ir::Value {- match ty {- WasmStorageType::I8 | WasmStorageType::I16 => cursor.ins().iconst(ir::types::I32, 0),- WasmStorageType::Val(v) => match v {- WasmValType::I32 => cursor.ins().iconst(ir::types::I32, 0),- WasmValType::I64 => cursor.ins().iconst(ir::types::I64, 0),- WasmValType::F32 => cursor.ins().f32const(0.0),- WasmValType::F64 => cursor.ins().f64const(0.0),- WasmValType::V128 => {- let c = cursor.func.dfg.constants.insert(vec![0; 16].into());- cursor.ins().vconst(ir::types::I8X16, c)- }- WasmValType::Ref(r) => {- assert!(r.nullable);- let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);-- // NB: The collector doesn't need to know about null references.- let _ = needs_stack_map;-- cursor.ins().iconst(ty, 0)- }- },- }-}--pub fn translate_struct_new_default(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- struct_type_index: TypeIndex,-) -> WasmResult<ir::Value> {- let interned_ty = func_env.module.types[struct_type_index].unwrap_module_type_index();- let struct_ty = func_env.types.unwrap_struct(interned_ty)?;- let fields = struct_ty- .fields- .iter()- .map(|f| default_value(&mut builder.cursor(), func_env, &f.element_type))- .collect::<StructFieldsVec>();- gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields)-}--pub fn translate_struct_get(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- struct_type_index: TypeIndex,- field_index: u32,- struct_ref: ir::Value,- extension: Option<Extension>,-) -> WasmResult<ir::Value> {- log::trace!(- "translate_struct_get({struct_type_index:?}, {field_index:?}, {struct_ref:?}, {extension:?})"- );-- // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a- // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that- // type info from `wasmparser` and through to here is a bit funky.- func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);-- emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);-- let field_index = usize::try_from(field_index).unwrap();- let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();-- let struct_layout = func_env.struct_or_exn_layout(interned_type_index);- let struct_size = struct_layout.size;-- let field_offset = struct_layout.fields[field_index].offset;- let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);- assert!(field_offset + field_size <= struct_size);-- let field_addr = func_env.prepare_gc_ref_access(- builder,- struct_ref,- BoundsCheck::StaticObjectField {- offset: field_offset,- access_size: u8::try_from(field_size).unwrap(),- object_size: struct_size,- },- );-- let result = read_field_at_addr(- func_env,- builder,- field_ty.element_type,- field_addr,- extension,- );- log::trace!("translate_struct_get(..) -> {result:?}");- result-}--pub fn translate_struct_set(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- struct_type_index: TypeIndex,- field_index: u32,- struct_ref: ir::Value,- new_val: ir::Value,-) -> WasmResult<()> {- log::trace!(- "translate_struct_set({struct_type_index:?}, {field_index:?}, struct_ref: {struct_ref:?}, new_val: {new_val:?})"- );-- // TODO: See comment in `translate_struct_get` about the `trapz`.- func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);-- emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);-- let field_index = usize::try_from(field_index).unwrap();- let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();-- let struct_layout = func_env.struct_or_exn_layout(interned_type_index);- let struct_size = struct_layout.size;-- let field_offset = struct_layout.fields[field_index].offset;- let field_ty = &func_env.types.unwrap_struct(interned_type_index)?.fields[field_index];- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty.element_type);- assert!(field_offset + field_size <= struct_size);-- let field_addr = func_env.prepare_gc_ref_access(- builder,- struct_ref,- BoundsCheck::StaticObjectField {- offset: field_offset,- access_size: u8::try_from(field_size).unwrap(),- object_size: struct_size,- },- );-- write_field_at_addr(- func_env,- builder,- field_ty.element_type,- field_addr,- new_val,- )?;-- log::trace!("translate_struct_set: finished");- Ok(())-}--pub fn translate_exn_unbox(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- tag_index: TagIndex,- exn_ref: ir::Value,-) -> WasmResult<SmallVec<[ir::Value; 4]>> {- log::trace!("translate_exn_unbox({tag_index:?}, {exn_ref:?})");-- // We know that the `exn_ref` is not null because we reach this- // operation only in catch blocks, and throws are initiated from- // runtime code that checks for nulls first.-- // Get the GcExceptionLayout associated with this tag's- // function type, and generate loads for each field.- let exception_ty_idx = func_env- .exception_type_from_tag(tag_index)- .unwrap_module_type_index();- let exception_ty = func_env.types.unwrap_exn(exception_ty_idx)?;- let exn_layout = func_env.struct_or_exn_layout(exception_ty_idx);- let exn_size = exn_layout.size;-- // Gather accesses first because these require a borrow on- // `func_env`, which we later mutate below via- // `prepare_gc_ref_access()`.- let mut accesses: SmallVec<[_; 4]> = smallvec![];- for (field_ty, field_layout) in exception_ty.fields.iter().zip(exn_layout.fields.iter()) {- accesses.push((field_layout.offset, field_ty.element_type));- }-- let mut result = smallvec![];- for (field_offset, field_ty) in accesses {- let field_size = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&field_ty);- assert!(field_offset + field_size <= exn_size);- let field_addr = func_env.prepare_gc_ref_access(- builder,- exn_ref,- BoundsCheck::StaticObjectField {- offset: field_offset,- access_size: u8::try_from(field_size).unwrap(),- object_size: exn_size,- },- );-- let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None)?;- result.push(value);- }-- log::trace!("translate_exn_unbox(..) -> {result:?}");- Ok(result)-}--pub fn translate_exn_throw(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- tag_index: TagIndex,- args: &[ir::Value],-) -> WasmResult<()> {- let (instance_id, defined_tag_id) = func_env.get_instance_and_tag(builder, tag_index);- let exnref = gc_compiler(func_env)?.alloc_exn(- func_env,- builder,- tag_index,- args,- instance_id,- defined_tag_id,- )?;- translate_exn_throw_ref(func_env, builder, exnref)-}--pub fn translate_exn_throw_ref(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- exnref: ir::Value,-) -> WasmResult<()> {- let builtin = func_env.builtin_functions.throw_ref(builder.func);- let sig = builder.func.dfg.ext_funcs[builtin].signature;- let vmctx = func_env.vmctx_val(&mut builder.cursor());-- // Generate a `try_call` with handlers from the current- // stack. This libcall is unique among libcall implementations of- // opcodes: we know the others will not throw, but `throw_ref`'s- // entire purpose is to throw. So if there are any handlers in the- // local function body, we need to attach them to this callsite- // like any other.- let continuation = builder.create_block();- let current_block = builder.current_block().unwrap();- builder.insert_block_after(continuation, current_block);- let continuation_call = builder.func.dfg.block_call(continuation, &[]);- let mut table_items = vec![ExceptionTableItem::Context(vmctx)];- for (tag, block) in func_env.stacks.handlers.handlers() {- let block_call = builder- .func- .dfg- .block_call(block, &[BlockArg::TryCallExn(0)]);- table_items.push(match tag {- Some(tag) => ExceptionTableItem::Tag(tag, block_call),- None => ExceptionTableItem::Default(block_call),- });- }- let etd = ExceptionTableData::new(sig, continuation_call, table_items);- let et = builder.func.dfg.exception_tables.push(etd);-- builder.ins().try_call(builtin, &[vmctx, exnref], et);-- builder.switch_to_block(continuation);- builder.seal_block(continuation);- func_env.trap(builder, crate::TRAP_UNREACHABLE);-- Ok(())-}--pub fn translate_array_new(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- elem: ir::Value,- len: ir::Value,-) -> WasmResult<ir::Value> {- log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");- let result =- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;- let zero = builder.ins().iconst(ir::types::I32, 0);- let ty = func_env.module.types[array_type_index].unwrap_module_type_index();- func_env.translate_entity_fill(- builder,- CheckedEntity::Array {- array: result,- ty,- initialized: false,- },- zero,- elem,- len,- )?;- log::trace!("translate_array_new(..) -> {result:?}");- Ok(result)-}--pub fn translate_array_new_default(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- len: ir::Value,-) -> WasmResult<ir::Value> {- log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");-- let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();- let array_ty = func_env.types.unwrap_array(interned_ty)?;- let result =- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;- log::trace!("translate_array_new_default(..) -> {result:?}");- let elem = default_value(&mut builder.cursor(), func_env, &array_ty.0.element_type);- let zero = builder.ins().iconst(ir::types::I32, 0);- func_env.translate_entity_fill(- builder,- CheckedEntity::Array {- array: result,- ty: interned_ty,- initialized: false,- },- zero,- elem,- len,- )?;- Ok(result)-}--pub fn translate_array_new_fixed(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- elems: &[ir::Value],-) -> WasmResult<ir::Value> {- log::trace!("translate_array_new_fixed({array_type_index:?}, {elems:?})");- let len = builder- .ins()- .iconst(ir::types::I32, i64::try_from(elems.len()).unwrap());- let result =- gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;- log::trace!("translate_array_new_fixed(..) -> {result:?}");- let ty = func_env.module.types[array_type_index].unwrap_module_type_index();- let array_ty = func_env.types.unwrap_array(ty)?;-- for (i, elem) in elems.iter().enumerate() {- let index = builder- .ins()- .iconst(ir::types::I32, i64::try_from(i).unwrap());- let addr = array_elem_addr(func_env, builder, ty, result, index)?;- gc_compiler(func_env)?.init_field(- func_env,- builder,- array_ty.0.element_type,- addr,- *elem,- )?;- }- Ok(result)-}--pub fn translate_array_len(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_ref: ir::Value,-) -> WasmResult<ir::Value> {- log::trace!("translate_array_len({array_ref:?})");-- func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE);-- let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();- let len_field = func_env.prepare_gc_ref_access(- builder,- array_ref,- // Note: We can't bounds check the whole array object's size because we- // don't know its length yet. Chicken and egg problem.- BoundsCheck::StaticOffset {- offset: len_offset,- access_size: u8::try_from(ir::types::I32.bytes()).unwrap(),- },- );- let flags = func_env.gc_memflags(&mut builder.func).with_readonly();- let result = builder.ins().load(ir::types::I32, flags, len_field, 0);- log::trace!("translate_array_len(..) -> {result:?}");- Ok(result)-}--struct ArraySizeInfo {- /// The `i32` size of the whole array object, in bytes.- obj_size: ir::Value,-- /// The `i32` size of each one of the array's elements, in bytes.- one_elem_size: ir::Value,-- /// The `i32` size of the array's base object, in bytes. This is also the- /// offset from the start of the array object to its elements.- base_size: ir::Value,-}--/// Emit code to get the dynamic size (in bytes) of a whole array object, along-/// with some other related bits.-fn emit_array_size_info(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- array_type_index: ModuleInternedTypeIndex,- // `i32` value containing the array's length.- array_len: ir::Value,-) -> WasmResult<ArraySizeInfo> {- let array_layout = func_env.array_layout(array_type_index)?;-- // Note that we check for overflow below because we can't trust the array's- // length: it came from inside the GC heap.- //- // We check for 32-bit multiplication overflow by performing a 64-bit- // multiplication and testing the high bits.- let one_elem_size = builder- .ins()- .iconst(ir::types::I64, i64::from(array_layout.elem_size));- let array_len = builder.ins().uextend(ir::types::I64, array_len);- let all_elems_size = builder.ins().imul(one_elem_size, array_len);-- let high_bits = builder.ins().ushr_imm(all_elems_size, 32);- builder.ins().trapnz(high_bits, TRAP_GC_HEAP_CORRUPT);-- let all_elems_size = builder.ins().ireduce(ir::types::I32, all_elems_size);- let base_size = builder- .ins()- .iconst(ir::types::I32, i64::from(array_layout.base_size));- let obj_size =- builder- .ins()- .uadd_overflow_trap(all_elems_size, base_size, TRAP_GC_HEAP_CORRUPT);-- let one_elem_size = builder.ins().ireduce(ir::types::I32, one_elem_size);-- Ok(ArraySizeInfo {- obj_size,- one_elem_size,- base_size,- })-}--/// Get the bounds-checked address of an element in an array.-///-/// The emitted code will trap if `index >= array.length`.-///-/// Returns the `ir::Value` containing the address of the `index`th element in-/// the array. You may read or write a value of the array's element type at this-/// address. You may not use it for any other kind of access, nor reuse this-/// value across GC safepoints.-fn array_elem_addr(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- array_type_index: ModuleInternedTypeIndex,- array_ref: ir::Value,- index: ir::Value,-) -> WasmResult<ir::Value> {- // First, assert that `index < array.length`.- //- // This check is visible at the Wasm-semantics level.- //- // TODO: We should emit spectre-safe bounds checks for array accesses (if- // configured) but we don't currently have a great way to do that here. The- // proper solution is to use linear memories to back GC heaps and reuse the- // code in `bounds_check.rs` to implement these bounds checks. That is all- // planned, but not yet implemented.-- let len = translate_array_len(func_env, builder, array_ref).unwrap();-- let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len);- func_env.trapz(builder, in_bounds, TRAP_ARRAY_OUT_OF_BOUNDS);-- // Compute the size (in bytes) of the whole array object.- let ArraySizeInfo {- obj_size,- one_elem_size,- base_size,- } = emit_array_size_info(func_env, builder, array_type_index, len)?;-- // Compute the offset of the `index`th element within the array object.- //- // NB: no need to check for overflow here, since at this point we know that- // `len * elem_size + base_size` did not overflow and `i < len`.- let offset_in_elems = builder.ins().imul(index, one_elem_size);- let offset_in_array = builder.ins().iadd(offset_in_elems, base_size);-- // Finally, use the object size and element offset we just computed to- // perform our implementation-internal bounds checks.- //- // Checking the whole object's size, rather than the `index`th element's- // size allows these bounds checks to be deduplicated across repeated- // accesses to the same array at different indices.- //- // This check should not be visible to Wasm, and serve to protect us from- // our own implementation bugs. The goal is to keep any potential widgets- // confined within the GC heap, and turn what would otherwise be a security- // vulnerability into a simple bug.- //- // TODO: Ideally we should fold the first Wasm-visible bounds check into- // this internal bounds check, so that we aren't performing multiple,- // redundant bounds checks. But we should figure out how to do this in a way- // that doesn't defeat the object-size bounds checking's deduplication- // mentioned above.- Ok(func_env.prepare_gc_ref_access(- builder,- array_ref,- BoundsCheck::DynamicObjectField {- offset: offset_in_array,- object_size: obj_size,- },- ))-}--pub fn translate_array_get(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- array_ref: ir::Value,- index: ir::Value,- extension: Option<Extension>,-) -> WasmResult<ir::Value> {- log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");-- emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);-- let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();- let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;-- let array_ty = func_env.types.unwrap_array(array_type_index)?;- let elem_ty = array_ty.0.element_type;-- let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension)?;- log::trace!("translate_array_get(..) -> {result:?}");- Ok(result)-}--pub fn translate_array_set(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- array_ref: ir::Value,- index: ir::Value,- value: ir::Value,-) -> WasmResult<()> {- log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");-- emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);-- let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();- let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index)?;-- let array_ty = func_env.types.unwrap_array(array_type_index)?;- let elem_ty = array_ty.0.element_type;-- write_field_at_addr(func_env, builder, elem_ty, elem_addr, value)?;-- log::trace!("translate_array_set: finished");- Ok(())-}--pub fn translate_ref_test(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- test_ty: WasmRefType,- val: ir::Value,- val_ty: WasmRefType,-) -> WasmResult<ir::Value> {- log::trace!("translate_ref_test({test_ty:?}, {val:?})");-- // First special case: testing for references to bottom types.- if test_ty.heap_type.is_bottom() {- let result = if test_ty.nullable {- // All null references (within the same type hierarchy) match null- // references to the bottom type.- func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?- } else {- // `ref.test` is always false for non-nullable bottom types, as the- // bottom types are uninhabited.- builder.ins().iconst(ir::types::I32, 0)- };- log::trace!("translate_ref_test(..) -> {result:?}");- return Ok(result);- }-- // And because `ref.test heap_ty` is only valid on operands whose type is in- // the same type hierarchy as `heap_ty`, if `heap_ty` is its hierarchy's top- // type, we only need to worry about whether we are testing for nullability- // or not.- if test_ty.heap_type.is_top() {- let result = if test_ty.nullable {- builder.ins().iconst(ir::types::I32, 1)- } else {- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;- let zero = builder.ins().iconst(ir::types::I32, 0);- let one = builder.ins().iconst(ir::types::I32, 1);- builder.ins().select(is_null, zero, one)- };- log::trace!("translate_ref_test(..) -> {result:?}");- return Ok(result);- }-- // `i31ref`s are a little interesting because they don't point to GC- // objects; we test the bit pattern of the reference itself.- if test_ty.heap_type == WasmHeapType::I31 {- let i31_mask = builder.ins().iconst(- ir::types::I32,- i64::from(wasmtime_environ::I31_DISCRIMINANT),- );- let is_i31 = builder.ins().band(val, i31_mask);- let result = if test_ty.nullable {- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;- builder.ins().bor(is_null, is_i31)- } else {- is_i31- };- log::trace!("translate_ref_test(..) -> {result:?}");- return Ok(result);- }-- // Otherwise, in the general case, we need to inspect our given object's- // actual type, which also requires null-checking and i31-checking it.-- let is_any_hierarchy = test_ty.heap_type.top() == WasmHeapTopType::Any;-- let non_null_block = builder.create_block();- let non_null_non_i31_block = builder.create_block();- let continue_block = builder.create_block();-- // Current block: check if the reference is null and branch appropriately.- let is_null = func_env.translate_ref_is_null(builder.cursor(), val, val_ty)?;- let result_when_is_null = builder- .ins()- .iconst(ir::types::I32, test_ty.nullable as i64);- builder.ins().brif(- is_null,- continue_block,- &[result_when_is_null.into()],- non_null_block,- &[],- );-- // Non-null block: We know the GC ref is non-null, but we need to also check- // for `i31` references that don't point to GC objects.- builder.switch_to_block(non_null_block);- log::trace!("translate_ref_test: non-null ref block");- if is_any_hierarchy {- let i31_mask = builder.ins().iconst(- ir::types::I32,- i64::from(wasmtime_environ::I31_DISCRIMINANT),- );- let is_i31 = builder.ins().band(val, i31_mask);- // If it is an `i31`, then create the result value based on whether we- // want `i31`s to pass the test or not.- let result_when_is_i31 = builder.ins().iconst(- ir::types::I32,- matches!(- test_ty.heap_type,- WasmHeapType::Any | WasmHeapType::Eq | WasmHeapType::I31- ) as i64,- );- builder.ins().brif(- is_i31,- continue_block,- &[result_when_is_i31.into()],- non_null_non_i31_block,- &[],- );- } else {- // If we aren't testing the `any` hierarchy, the reference cannot be an- // `i31ref`. Jump directly to the non-null and non-i31 block; rely on- // branch folding during lowering to clean this up.- builder.ins().jump(non_null_non_i31_block, &[]);- }-- // Non-null and non-i31 block: Read the actual `VMGcKind` or- // `VMSharedTypeIndex` out of the object's header and check whether it- // matches the expected type.- builder.switch_to_block(non_null_non_i31_block);- log::trace!("translate_ref_test: non-null and non-i31 ref block");- let check_header_kind = |func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- val: ir::Value,- expected_kind: VMGcKind|- -> ir::Value {- let kind_addr = func_env.prepare_gc_ref_access(- builder,- val,- BoundsCheck::StaticObjectField {- offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,- access_size: wasmtime_environ::VM_GC_KIND_SIZE,- object_size: wasmtime_environ::VM_GC_HEADER_SIZE,- },- );- let gc_memflags = func_env.gc_memflags(&mut builder.func);- let actual_kind =- builder- .ins()- .load(ir::types::I32, gc_memflags.with_readonly(), kind_addr, 0);- let expected_kind = builder- .ins()- .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));- // Inline version of `VMGcKind::matches`.- let and = builder.ins().band(actual_kind, expected_kind);- let kind_matches = builder- .ins()- .icmp(ir::condcodes::IntCC::Equal, and, expected_kind);- builder.ins().uextend(ir::types::I32, kind_matches)- };- let result = match test_ty.heap_type {- WasmHeapType::Any- | WasmHeapType::None- | WasmHeapType::Extern- | WasmHeapType::NoExtern- | WasmHeapType::Func- | WasmHeapType::NoFunc- | WasmHeapType::Cont- | WasmHeapType::NoCont- | WasmHeapType::Exn- | WasmHeapType::NoExn- | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"),-- // For these abstract but non-top and non-bottom types, we check the- // `VMGcKind` that is in the object's header.- WasmHeapType::Eq => check_header_kind(func_env, builder, val, VMGcKind::EqRef),- WasmHeapType::Struct => check_header_kind(func_env, builder, val, VMGcKind::StructRef),- WasmHeapType::Array => check_header_kind(func_env, builder, val, VMGcKind::ArrayRef),-- // For concrete types, we need to do a full subtype check between the- // `VMSharedTypeIndex` in the object's header and the- // `ModuleInternedTypeIndex` we have here.- //- // TODO: This check should ideally be done inline, but we don't have a- // good way to access the `TypeRegistry`'s supertypes arrays from Wasm- // code at the moment.- WasmHeapType::ConcreteArray(ty)- | WasmHeapType::ConcreteStruct(ty)- | WasmHeapType::ConcreteExn(ty) => {- let expected_interned_ty = ty.unwrap_module_type_index();- let expected_shared_ty =- func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);-- let ty_addr = func_env.prepare_gc_ref_access(- builder,- val,- BoundsCheck::StaticOffset {- offset: wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET,- access_size: func_env.offsets.size_of_vmshared_type_index(),- },- );- let gc_memflags = func_env.gc_memflags(&mut builder.func);- let actual_shared_ty =- builder- .ins()- .load(ir::types::I32, gc_memflags.with_readonly(), ty_addr, 0);-- func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)- }-- // Same as for concrete arrays and structs except that a `VMFuncRef`- // doesn't begin with a `VMGcHeader` and is a raw pointer rather than GC- // heap index.- WasmHeapType::ConcreteFunc(ty) => {- let expected_interned_ty = ty.unwrap_module_type_index();- let expected_shared_ty =- func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);-- let gc_memflags = func_env.gc_memflags(&mut builder.func);- let actual_shared_ty = func_env.load_funcref_type_index(- &mut builder.cursor(),- gc_memflags.with_readonly(),- val,- );-- func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty)- }- WasmHeapType::ConcreteCont(_) => {- // TODO(#10248) GC integration for stack switching- return stack_switching_unsupported();- }- };- builder.ins().jump(continue_block, &[result.into()]);-- // Control flow join point with the result.- builder.switch_to_block(continue_block);- let result = builder.append_block_param(continue_block, ir::types::I32);- log::trace!("translate_ref_test(..) -> {result:?}");-- builder.seal_block(non_null_block);- builder.seal_block(non_null_non_i31_block);- builder.seal_block(continue_block);-- Ok(result)-}--#[cfg(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying"))]-fn uextend_i32_to_pointer_type(- builder: &mut FunctionBuilder,- pointer_type: ir::Type,- value: ir::Value,-) -> ir::Value {- assert_eq!(builder.func.dfg.value_type(value), ir::types::I32);- match pointer_type {- ir::types::I32 => value,- ir::types::I64 => builder.ins().uextend(ir::types::I64, value),- _ => unreachable!(),- }-}--/// Emit CLIF to compute an array object's total size, given the dynamic length-/// in its initialization.-///-/// Traps if the size overflows.-#[cfg_attr(- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),- expect(dead_code, reason = "easier to define")-)]-fn emit_array_size(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- array_layout: &GcArrayLayout,- len: ir::Value,-) -> ir::Value {- let base_size = builder- .ins()- .iconst(ir::types::I32, i64::from(array_layout.base_size));-- // `elems_size = len * elem_size`- //- // Check for multiplication overflow and trap if it occurs, since that- // means Wasm is attempting to allocate an array that is larger than our- // implementation limits. (Note: there is no standard implementation- // limit for array length beyond `u32::MAX`.)- //- // We implement this check by encoding our logically-32-bit operands as- // i64 values, doing a 64-bit multiplication, and then checking the high- // 32 bits of the multiplication's result. If the high 32 bits are not- // all zeros, then the multiplication overflowed.- debug_assert_eq!(builder.func.dfg.value_type(len), ir::types::I32);- let len = builder.ins().uextend(ir::types::I64, len);- let elems_size_64 = builder- .ins()- .imul_imm(len, i64::from(array_layout.elem_size));- let high_bits = builder.ins().ushr_imm(elems_size_64, 32);- func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE);- let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64);-- // And if adding the base size and elements size overflows, then the- // allocation is too large.- let size = func_env.uadd_overflow_trap(- builder,- base_size,- elems_size,- crate::TRAP_ALLOCATION_TOO_LARGE,- );-- size-}--/// Common helper for struct-field initialization that can be reused across-/// collectors.-#[cfg_attr(- not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),- expect(dead_code, reason = "easier to define")-)]-fn initialize_struct_fields(- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder<'_>,- struct_ty: ModuleInternedTypeIndex,- raw_ptr_to_struct: ir::Value,- field_values: &[ir::Value],-) -> WasmResult<()> {- let struct_layout = func_env.struct_or_exn_layout(struct_ty);- let struct_size = struct_layout.size;- let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().map(|f| f.offset).collect();- assert_eq!(field_offsets.len(), field_values.len());-- assert!(!func_env.types[struct_ty].composite_type.shared);- let fields = match &func_env.types[struct_ty].composite_type.inner {- WasmCompositeInnerType::Struct(s) => &s.fields,- WasmCompositeInnerType::Exn(e) => &e.fields,- _ => panic!("Not a struct or exception type"),- };-- let field_types: SmallVec<[_; 8]> = fields.iter().cloned().collect();- assert_eq!(field_types.len(), field_values.len());-- for ((ty, val), offset) in field_types.into_iter().zip(field_values).zip(field_offsets) {- let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty.element_type);- assert!(offset + size_of_access <= struct_size);- let field_addr = builder.ins().iadd_imm(raw_ptr_to_struct, i64::from(offset));- gc_compiler(func_env)?.init_field(func_env, builder, ty.element_type, field_addr, *val)?;- }-- Ok(())-}--impl FuncEnvironment<'_> {- pub(crate) fn gc_heap_alias_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {- self.alias_region(func, AliasRegionKey::GcHeap)- }-- /// Flags to use for general-purpose GC loads/stores.- ///- /// This is used for accesses to the GC heap which aren't expected to trap, but- /// retain internal assertion metadata to report if such a trap happens. This- /// is here to ensure that in the face of heap corruption that there's no- /// possible UB within Cranelift and/or the runtime.- fn gc_memflags(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {- let region = self.gc_heap_alias_region(func);- ir::MemFlagsData::new()- .with_trap_code(Some(crate::TRAP_GC_HEAP_CORRUPT))- .with_alias_region(Some(region))- }-- fn gc_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcLayout {- // Lazily compute and cache the layout.- if !self.ty_to_gc_layout.contains_key(&type_index) {- let ty = &self.types[type_index].composite_type;- let layout = gc_compiler(self)- .unwrap()- .layouts()- .gc_layout(ty)- .expect("gc_layout should not OOM at compile time")- .expect("should only call `FuncEnvironment::gc_layout` for GC types");- self.ty_to_gc_layout.insert(type_index, layout);- }-- self.ty_to_gc_layout.get(&type_index).unwrap()- }-- /// Get the `GcArrayLayout` for the array type at the given `type_index`.- pub(crate) fn array_layout(- &mut self,- type_index: ModuleInternedTypeIndex,- ) -> WasmResult<&GcArrayLayout> {- Ok(self.gc_layout(type_index).unwrap_array())- }-- /// Get the `GcStructLayout` for the struct or exception type at the given `type_index`.- fn struct_or_exn_layout(&mut self, type_index: ModuleInternedTypeIndex) -> &GcStructLayout {- let result = self.gc_layout(type_index).unwrap_struct();- result- }-- /// Get or create the global for our GC heap's base pointer.- fn get_gc_heap_base_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {- if let Some(base) = self.gc_heap_base {- return base;- }-- let store_context_ptr = self.get_vmstore_context_ptr_global(func);- let offset = self.offsets.ptr.vmstore_context_gc_heap_base();-- let mut flags = ir::MemFlagsData::trusted();- let memory_tunables =- wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);- if !self- .tunables- .gc_heap_memory_type()- .memory_may_move(&memory_tunables)- {- flags.set_readonly();- flags.set_can_move();- }-- let base_flags = func.dfg.mem_flags.insert(flags).unwrap();- let base = func.create_global_value(ir::GlobalValueData::Load {- base: store_context_ptr,- offset: Offset32::new(offset.into()),- global_type: self.pointer_type(),- flags: base_flags,- });-- self.gc_heap_base = Some(base);- base- }-- /// Get the GC heap's base.- pub(crate) fn get_gc_heap_base(- &mut self,- builder: &mut FunctionBuilder,- ) -> WasmResult<ir::Value> {- let global = self.get_gc_heap_base_global(&mut builder.func);- Ok(builder.ins().global_value(self.pointer_type(), global))- }-- fn get_gc_heap_bound_global(&mut self, func: &mut ir::Function) -> ir::GlobalValue {- if let Some(bound) = self.gc_heap_bound {- return bound;- }- let store_context_ptr = self.get_vmstore_context_ptr_global(func);- let offset = self.offsets.ptr.vmstore_context_gc_heap_current_length();- let bound_flags = func- .dfg- .mem_flags- .insert(ir::MemFlagsData::trusted())- .unwrap();- let bound = func.create_global_value(ir::GlobalValueData::Load {- base: store_context_ptr,- offset: Offset32::new(offset.into()),- global_type: self.pointer_type(),- flags: bound_flags,- });- self.gc_heap_bound = Some(bound);- bound- }-- /// Get the GC heap's bound.- pub(crate) fn get_gc_heap_bound(- &mut self,- builder: &mut FunctionBuilder,- ) -> WasmResult<ir::Value> {- let global = self.get_gc_heap_bound_global(&mut builder.func);- Ok(builder.ins().global_value(self.pointer_type(), global))- }-- /// Get or create the `Heap` for our GC heap.- fn get_gc_heap(&mut self, func: &mut ir::Function) -> Heap {- if let Some(heap) = self.gc_heap {- return heap;- }-- let base = self.get_gc_heap_base_global(func);- let bound = self.get_gc_heap_bound_global(func);- let memory = self.tunables.gc_heap_memory_type();- let heap = self.heaps.push(HeapData {- base,- bound,- memory,- kind: MemoryKind::GcHeap,- });- self.gc_heap = Some(heap);- heap- }-- /// Get the raw pointer of `gc_ref[offset]` bounds checked for an access of- /// `size` bytes.- ///- /// The given `gc_ref` must be a non-null, non-i31 GC reference.- ///- /// If `check` is a `BoundsCheck::Object`, then it is the callers- /// responsibility to ensure that `offset + access_size <= object_size`.- ///- /// Returns a raw pointer to `gc_ref[offset]` -- not a raw pointer to the GC- /// object itself (unless `offset` happens to be `0`). This raw pointer may- /// be used to read or write up to as many bytes as described by `bound`. Do- /// NOT attempt accesses bytes outside of `bound`; that may lead to- /// unchecked out-of-bounds accesses.- ///- /// This method is collector-agnostic.- fn prepare_gc_ref_access(- &mut self,- builder: &mut FunctionBuilder,- gc_ref: ir::Value,- bounds_check: BoundsCheck,- ) -> ir::Value {- log::trace!("prepare_gc_ref_access({gc_ref:?}, {bounds_check:?})");- assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);-- let gc_heap = self.get_gc_heap(&mut builder.func);- let gc_heap = self.heaps[gc_heap].clone();- let result = match crate::bounds_checks::bounds_check_and_compute_addr(- builder,- self,- &gc_heap,- gc_ref,- bounds_check,- crate::TRAP_GC_HEAP_CORRUPT,- ) {- Reachability::Reachable(v) => v,- Reachability::Unreachable => {- // We are now in unreachable code, but we don't want to plumb- // through a bunch of `Reachability` through all of our callers,- // so just assert we won't reach here and return `null`- let null = builder.ins().iconst(self.pointer_type(), 0);- builder.ins().trapz(null, crate::TRAP_INTERNAL_ASSERT);- null- }- };- log::trace!("prepare_gc_ref_access(..) -> {result:?}");- result- }-- /// Emit checks (if necessary) for whether the given `gc_ref` is null or is- /// an `i31ref`.- ///- /// Takes advantage of static information based on `ty` as to whether the GC- /// reference is nullable or can ever be an `i31`.- ///- /// Returns an `ir::Value` that is an `i32` will be non-zero if the GC- /// reference is null or is an `i31ref`; otherwise, it will be zero.- ///- /// This method is collector-agnostic.- #[cfg_attr(- not(feature = "gc-drc"),- expect(dead_code, reason = "easier to define")- )]- fn gc_ref_is_null_or_i31(- &mut self,- builder: &mut FunctionBuilder,- ty: WasmRefType,- gc_ref: ir::Value,- ) -> ir::Value {- assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);- assert!(ty.is_vmgcref_type_and_not_i31());-- let might_be_i31 = match ty.heap_type {- // If we are definitely dealing with an i31, we shouldn't be- // emitting dynamic checks for it, and the caller shouldn't call- // this function. Should have been caught by the assertion at the- // start of the function.- WasmHeapType::I31 => unreachable!(),-- // Could potentially be an i31.- WasmHeapType::Any | WasmHeapType::Eq => true,-- // If it is definitely a struct, array, or uninhabited type, then it- // is definitely not an i31.- WasmHeapType::Array- | WasmHeapType::ConcreteArray(_)- | WasmHeapType::Struct- | WasmHeapType::ConcreteStruct(_)- | WasmHeapType::None => false,-- // Despite being a different type hierarchy, this *could* be an- // `i31` if it is the result of- //- // (extern.convert_any (ref.i31 ...))- WasmHeapType::Extern => true,-- // Can only ever be `null`.- WasmHeapType::NoExtern => false,-- WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn => false,-- // Wrong type hierarchy, and also funcrefs are not GC-managed- // types. Should have been caught by the assertion at the start of- // the function.- WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => {- unreachable!()- }- WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => {- unreachable!()- }- };-- match (ty.nullable, might_be_i31) {- // This GC reference statically cannot be null nor an i31. (Let- // Cranelift's optimizer const-propagate this value and erase any- // unnecessary control flow resulting from branching on this value.)- (false, false) => builder.ins().iconst(ir::types::I32, 0),-- // This GC reference is always non-null, but might be an i31.- (false, true) => builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT)),-- // This GC reference might be null, but can never be an i31.- (true, false) => builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0),-- // Fully general case: this GC reference could be either null or an- // i31.- (true, true) => {- let is_i31 = builder.ins().band_imm(gc_ref, i64::from(I31_DISCRIMINANT));- let is_null = builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0);- let is_null = builder.ins().uextend(ir::types::I32, is_null);- builder.ins().bor(is_i31, is_null)- }- }- }-- // Emit code to check whether `a <: b` for two `VMSharedTypeIndex`es.- pub(crate) fn is_subtype(- &mut self,- builder: &mut FunctionBuilder<'_>,- a: ir::Value,- b: ir::Value,- ) -> ir::Value {- log::trace!("is_subtype({a:?}, {b:?})");-- let diff_tys_block = builder.create_block();- let continue_block = builder.create_block();-- // Current block: fast path for when `a == b`.- log::trace!("is_subtype: fast path check for exact same types");- let same_ty = builder.ins().icmp(IntCC::Equal, a, b);- let same_ty = builder.ins().uextend(ir::types::I32, same_ty);- builder.ins().brif(- same_ty,- continue_block,- &[same_ty.into()],- diff_tys_block,- &[],- );-- // Different types block: fall back to the `is_subtype` libcall.- builder.switch_to_block(diff_tys_block);- log::trace!("is_subtype: slow path to do full `is_subtype` libcall");- let is_subtype = self.builtin_functions.is_subtype(builder.func);- let vmctx = self.vmctx_val(&mut builder.cursor());- let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]);- let result = builder.func.dfg.first_result(call_inst);- builder.ins().jump(continue_block, &[result.into()]);-- // Continue block: join point for the result.- builder.switch_to_block(continue_block);- let result = builder.append_block_param(continue_block, ir::types::I32);- log::trace!("is_subtype(..) -> {result:?}");-- builder.seal_block(diff_tys_block);- builder.seal_block(continue_block);-- result- }-}--fn stack_switching_unsupported<T>() -> WasmResult<T> {- Err(wasmtime_environ::WasmError::Unsupported(- "Stack switching feature not compatible with GC, yet".to_string(),- ))-}--pub fn translate_array_new_entity(- env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- array_type_index: TypeIndex,- entity: CheckedEntity,- entity_offset: ir::Value,- len: ir::Value,-) -> WasmResult<ir::Value> {- // Before actually allocating this array first do a bounds-check on the- // passive entity itself.- let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();- env.translate_entity_bounds_check(builder, entity, entity_offset, len)?;-- let array = gc_compiler(env)?.alloc_uninit_array(env, builder, array_type_index, len)?;- let dst = builder.ins().iconst(ir::types::I32, 0);- env.translate_entity_copy(- builder,- CheckedEntity::Array {- array,- ty: interned_type_index,- initialized: false,- },- entity,- dst,- entity_offset,- len,- )?;-- Ok(array)-}diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/null.rssimilarity index 100%rename from crates/cranelift/src/func_environ/gc/enabled/null.rsrename to crates/cranelift/src/func_environ/gc/null.rsdiff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsindex 5da5c88623..41c9dd38b9 100644--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -1221,7 +1221,7 @@ pub(crate) fn translate_cont_new<'a>(.ins().iconst(I32, i64::try_from(return_types.len()).unwrap());- let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?;+ let cont_new_func = env.builtin_functions.cont_new(&mut builder.func);let vmctx = env.vmctx_val(&mut builder.cursor());let call_inst = builder.ins()diff --git a/crates/cranelift/src/func_environ/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rsindex 15b36d110b..653191a769 100644--- a/crates/cranelift/src/func_environ/stack_switching/mod.rs+++ b/crates/cranelift/src/func_environ/stack_switching/mod.rs@@ -1,35 +1,3 @@mod control_effect;pub(crate) mod fatpointer;pub(crate) mod instructions;--pub(crate) mod builtins {- macro_rules! define_builtin_accessors {- ( $( $name:ident , )* ) => {- $(- #[inline]- pub fn $name(- func_env: &mut crate::func_environ::FuncEnvironment<'_>,- func: &mut crate::ir::Function,- ) -> wasmtime_environ::WasmResult<crate::ir::FuncRef> {- #[cfg(feature = "stack-switching")]- {- return Ok(func_env.builtin_functions.$name(func));- }-- #[cfg(not(feature = "stack-switching"))]- {- let _ = (func, func_env);- return Err(wasmtime_environ::wasm_unsupported!(- "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \- feature was not enabled"- ));- }- }- )*- };- }-- define_builtin_accessors! {- cont_new,- }-}diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.tomlindex b56304c229..0d08121831 100644--- a/crates/wasmtime/Cargo.toml+++ b/crates/wasmtime/Cargo.toml@@ -188,7 +188,7 @@ pulley = []# Enables support for incremental compilation cache to be enabled in `Config`.-incremental-cache = ["wasmtime-cranelift?/incremental-cache", "std"]+incremental-cache = ["std"]# Enables support for profiling guest modules.profiling = [@@ -231,8 +231,6 @@ all-arch = ["wasmtime-cranelift?/all-arch", "wasmtime-winch?/all-arch", "pulley"# testing purposes.component-model = ["wasmtime-environ/component-model",- "wasmtime-cranelift?/component-model",- "wasmtime-winch?/component-model","dep:wasmtime-component-macro","dep:wasmtime-component-util","dep:encoding_rs",@@ -241,8 +239,6 @@ component-model = [wmemcheck = ["dep:wasmtime-wmemcheck",- "wasmtime-cranelift?/wmemcheck",- "wasmtime-winch?/wmemcheck","wasmtime-environ/wmemcheck","std",]@@ -293,49 +289,24 @@ runtime = [## You can additionally configure which GC implementations are enabled via the# `gc-drc`, `gc-null`, and `gc-copying` features.-gc = [- "wasmtime-environ/gc",- "wasmtime-cranelift?/gc",- "wasmtime-winch?/gc",-]+gc = ["wasmtime-environ/gc"]# Enable the deferred reference counting garbage collector.-gc-drc = [- "gc",- "wasmtime-environ/gc-drc",- "wasmtime-cranelift?/gc-drc",- "wasmtime-winch?/gc-drc",-]+gc-drc = ["gc", "wasmtime-environ/gc-drc"]# Enable the null garbage collector.-gc-null = [- "gc",- "wasmtime-environ/gc-null",- "wasmtime-cranelift?/gc-null",- "wasmtime-winch?/gc-null",-]+gc-null = ["gc", "wasmtime-environ/gc-null"]# Enable the copying garbage collector.-gc-copying = [- "gc",- "wasmtime-environ/gc-copying",- "wasmtime-cranelift?/gc-copying",- "wasmtime-winch?/gc-copying",-]+gc-copying = ["gc", "wasmtime-environ/gc-copying"]# Enable runtime support for the WebAssembly threads proposal.-threads = [- "wasmtime-cranelift?/threads",- "wasmtime-winch?/threads",- "std",-]+threads = ["std"]stack-switching = ["runtime","std","wasmtime-environ/stack-switching",- "wasmtime-cranelift?/stack-switching",- "wasmtime-winch?/stack-switching",]# Controls whether backtraces will attempt to parse DWARF information indiff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 11070cce43..077f475360 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -957,7 +957,6 @@ fn is_inlining_function(key: FuncKey) -> bool {FuncKey::DefinedWasmFunction(..) => true,// Intrinsics can be inlined into other functions.- #[cfg(feature = "component-model")]FuncKey::UnsafeIntrinsic(..) => true,// Trampolines cannot participate in inlining since our@@ -968,7 +967,6 @@ fn is_inlining_function(key: FuncKey) -> bool {| FuncKey::WasmToBuiltinTrampoline(..)| FuncKey::PatchableToBuiltinTrampoline(..)| FuncKey::ModuleStartup(..) => false,- #[cfg(feature = "component-model")]FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,FuncKey::PulleyHostCall(_) => {diff --git a/crates/winch/Cargo.toml b/crates/winch/Cargo.tomlindex 80f6992168..14dc41d408 100644--- a/crates/winch/Cargo.toml+++ b/crates/winch/Cargo.toml@@ -23,15 +23,4 @@ gimli = { workspace = true, features = ['std'] }log = { workspace = true }[features]-component-model = [- "wasmtime-environ/component-model",- "wasmtime-cranelift/component-model",-]all-arch = ["winch-codegen/all-arch"]-gc = ['winch-codegen/gc']-gc-drc = ['winch-codegen/gc-drc']-gc-null = ['winch-codegen/gc-null']-gc-copying = ['winch-codegen/gc-copying']-stack-switching = ['winch-codegen/stack-switching']-threads = ['winch-codegen/threads']-wmemcheck = ['winch-codegen/wmemcheck']diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rsindex 6a9a7f2982..ad34b8e2a9 100644--- a/crates/winch/src/compiler.rs+++ b/crates/winch/src/compiler.rs@@ -5,7 +5,6 @@ use std::mem;use std::sync::Mutex;use wasmparser::FuncValidatorAllocations;use wasmtime_cranelift::CompiledFunction;-#[cfg(feature = "component-model")]use wasmtime_environ::component::ComponentTranslation;use wasmtime_environ::error::Result;use wasmtime_environ::{@@ -203,7 +202,6 @@ impl wasmtime_environ::Compiler for Compiler {self.isa.is_branch_protection_enabled()}- #[cfg(feature = "component-model")]fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {self.trampolines.component_compiler()}@@ -309,7 +307,6 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {self.0.is_branch_protection_enabled()}- #[cfg(feature = "component-model")]fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler {self}@@ -330,7 +327,6 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {}}-#[cfg(feature = "component-model")]impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {fn compile_component_trampoline(&self,diff --git a/winch/codegen/Cargo.toml b/winch/codegen/Cargo.tomlindex e9d2dba097..07d348f715 100644--- a/winch/codegen/Cargo.toml+++ b/winch/codegen/Cargo.toml@@ -35,10 +35,3 @@ all-arch = ["x64","arm64",]-gc = ['wasmtime-environ/gc']-gc-drc = ['wasmtime-environ/gc-drc']-gc-null = ['wasmtime-environ/gc-null']-gc-copying = ['wasmtime-environ/gc-copying']-stack-switching = ['wasmtime-environ/stack-switching']-threads = ['wasmtime-environ/threads']-wmemcheck = ['wasmtime-environ/wmemcheck']diff --git a/winch/codegen/build.rs b/winch/codegen/build.rsindex 378ac51732..1d9fb5db21 100644--- a/winch/codegen/build.rs+++ b/winch/codegen/build.rs@@ -1,4 +1,29 @@fn main() {+ println!("cargo:rerun-if-changed=build.rs");++ enable_default_arch();++ // Pretend that Winch has these features. These are injected via the+ // `wasmtime_environ::foreach_builtin_function!` macro so to ensure that we+ // don't get spurious warnings about unused cfg's these are printed out+ // here.+ faux_feature("stack-switching");+ faux_feature("gc");+ faux_feature("gc-copying");+ faux_feature("gc-null");+ faux_feature("gc-drc");+ faux_feature("threads");+ faux_feature("wmemcheck");+ faux_feature("component-model");+ faux_feature("incremental-cache");+}++fn faux_feature(feature: &str) {+ println!("cargo:rustc-check-cfg=cfg(feature, values(\"{feature}\"))");+ println!("cargo:rustc-cfg=feature=\"{feature}\"");+}++fn enable_default_arch() {if cfg!(feature = "x64") || cfg!(feature = "arm64") || cfg!(feature = "all-arch") {return;}@@ -12,5 +37,4 @@ fn main() {} else {println!("cargo:rustc-cfg=feature=\"{arch}\"");}- println!("cargo:rerun-if-changed=build.rs");}diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex e993017276..24805f79d0 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -2391,13 +2391,7 @@ whereOk(())}- #[cfg(not(feature = "threads"))]- pub fn emit_atomic_wait(&mut self, _arg: &MemArg, _kind: AtomicWaitKind) -> Result<()> {- Err(CodeGenError::unimplemented_wasm_instruction().into())- }-/// Emit the sequence of instruction for a `memory.atomic.wait*`.- #[cfg(feature = "threads")]pub fn emit_atomic_wait(&mut self, arg: &MemArg, kind: AtomicWaitKind) -> Result<()> {// The `memory_atomic_wait*` builtins expect the following arguments:// - `memory`, as u32@@ -2448,12 +2442,6 @@ whereOk(())}- #[cfg(not(feature = "threads"))]- pub fn emit_atomic_notify(&mut self, _arg: &MemArg) -> Result<()> {- Err(CodeGenError::unimplemented_wasm_instruction().into())- }-- #[cfg(feature = "threads")]pub fn emit_atomic_notify(&mut self, arg: &MemArg) -> Result<()> {// The memory `memory_atomic_notify` builtin expects the following arguments:// - `memory`, as u32