A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Move most module initialization to compiled code (#13487)
Alex Crichton5f3b67ea0558
diff --git a/crates/core/src/slab.rs b/crates/core/src/slab.rsindex 38a483a839..030d6294a2 100644--- a/crates/core/src/slab.rs+++ b/crates/core/src/slab.rs@@ -390,15 +390,11 @@ impl<T> Slab<T> {////// Returns `None` if the value has since been deallocated.///- /// If `id` comes from a different `Slab` instance, this method may panic,- /// return `None`, or return an arbitrary value.+ /// If `id` comes from a different `Slab` instance, this method may return+ /// `None`, or return an arbitrary value.#[inline]pub fn get(&self, id: Id) -> Option<&T> {- match self- .entries- .get(id.0.index())- .expect("id from different slab")- {+ match self.entries.get(id.0.index())? {Entry::Occupied(x) => Some(x),Entry::Free { .. } => None,}@@ -408,15 +404,11 @@ impl<T> Slab<T> {////// Returns `None` if the value has since been deallocated.///- /// If `id` comes from a different `Slab` instance, this method may panic,- /// return `None`, or return an arbitrary value.+ /// If `id` comes from a different `Slab` instance, this method may return+ /// `None`, or return an arbitrary value.#[inline]pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {- match self- .entries- .get_mut(id.0.index())- .expect("id from different slab")- {+ match self.entries.get_mut(id.0.index())? {Entry::Occupied(x) => Some(x),Entry::Free { .. } => None,}diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex a2c76c9f8a..5bc5005503 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -39,9 +39,9 @@ use wasmtime_environ::{Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, HostCall, Inlining,- InliningCompiler, ModulePC, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection,- StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType,- WasmValType, prelude::*,+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,+ StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,+ WasmFuncType, WasmValType, prelude::*,};use wasmtime_unwinder::ExceptionTableBuilder;@@ -199,6 +199,230 @@ impl Compiler {builder.ins().call_indirect(sig, addr, args)}++ fn compile_wasm_to_array_trampoline(+ &self,+ wasm_func_ty: &WasmFuncType,+ key: FuncKey,+ symbol: &str,+ ) -> Result<CompiledFunctionBody, CompileError> {+ log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}");++ let isa = &*self.isa;+ let pointer_type = isa.pointer_type();+ let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables);+ let array_call_sig = array_call_signature(isa);++ let mut compiler = self.function_compiler();+ let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig);+ let (mut builder, block0) = compiler.builder(func);++ let args = builder.func.dfg.block_params(block0).to_vec();+ let callee_vmctx = args[0];+ let caller_vmctx = args[1];++ // We are exiting Wasm, so save our PC and FP.+ //+ // Assert that the caller vmctx really is a core Wasm vmctx, since+ // that's what we are assuming with our offsets below.+ self.debug_assert_vmctx_kind(+ &mut builder,+ caller_vmctx,+ wasmtime_environ::VMCONTEXT_MAGIC,+ );+ let ptr = isa.pointer_bytes();+ let vm_store_context = builder.ins().load(+ pointer_type,+ MemFlagsData::trusted(),+ caller_vmctx,+ i32::from(ptr.vmcontext_store_context()),+ );+ save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context);++ // Spill all wasm arguments to the stack in `ValRaw` slots.+ let (args_base, args_len) =+ self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);+ let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));++ // Load the actual callee out of the+ // `VMArrayCallHostFuncContext::host_func`.+ let ptr_size = isa.pointer_bytes();+ let callee = builder.ins().load(+ pointer_type,+ MemFlagsData::trusted(),+ callee_vmctx,+ ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),+ );++ // Do an indirect call to the callee.+ let callee_signature = builder.func.import_signature(array_call_sig);+ let call = self.call_indirect_host(+ &mut builder,+ HostCall::ArrayCall,+ callee_signature,+ callee,+ &[callee_vmctx, caller_vmctx, args_base, args_len],+ );++ // Increment the "execution version" on the VMStoreContext if+ // guest debugging is enabled.+ if self.tunables.debug_guest {+ let vmstore_ctx_ptr = builder.ins().load(+ pointer_type,+ MemFlagsData::trusted().with_readonly(),+ caller_vmctx,+ i32::from(ptr_size.vmctx_store_context()),+ );+ let old_version = builder.ins().load(+ ir::types::I64,+ MemFlagsData::trusted(),+ vmstore_ctx_ptr,+ i32::from(ptr_size.vmstore_context_execution_version()),+ );+ let new_version = builder.ins().iadd_imm(old_version, 1);+ builder.ins().store(+ MemFlagsData::trusted(),+ new_version,+ vmstore_ctx_ptr,+ i32::from(ptr_size.vmstore_context_execution_version()),+ );+ }++ // Invoke `raise` if the callee (host) returned an error.+ let succeeded = builder.func.dfg.inst_results(call)[0];+ self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded);++ // Return results from the array as native return values.+ let results =+ self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);+ builder.ins().return_(&results);+ builder.finalize();++ Ok(CompiledFunctionBody {+ code: box_dyn_any_compiler_context(Some(compiler.cx)),+ needs_gc_heap: false,+ })+ }++ fn compile_wasm_to_builtin(+ &self,+ key: FuncKey,+ symbol: &str,+ ) -> Result<CompiledFunctionBody, CompileError> {+ log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}");++ let isa = &*self.isa;+ let ptr_size = isa.pointer_bytes();+ let pointer_type = isa.pointer_type();+ let sigs = BuiltinFunctionSignatures::new(self);++ let (builtin_func_index, wasm_sig) = match key {+ FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)),+ FuncKey::PatchableToBuiltinTrampoline(builtin) => {+ let mut sig = sigs.wasm_signature(builtin);+ // Patchable functions cannot return anything. We+ // raise any errors that occur below so this is fine.+ sig.returns.clear();+ sig.call_conv = CallConv::PreserveAll;+ (builtin, sig)+ }+ _ => unreachable!(),+ };+ let host_sig = sigs.host_signature(builtin_func_index);++ let mut compiler = self.function_compiler();+ let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone());+ let (mut builder, block0) = compiler.builder(func);+ let vmctx = builder.block_params(block0)[0];++ // Debug-assert that this is the right kind of vmctx, and then+ // additionally perform the "routine of the exit trampoline" of saving+ // fp/pc/etc.+ self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC);+ let vm_store_context = builder.ins().load(+ pointer_type,+ MemFlagsData::trusted(),+ vmctx,+ ptr_size.vmcontext_store_context(),+ );+ save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context);++ // Now it's time to delegate to the actual builtin. Forward all our own+ // arguments to the libcall itself.+ let args = builder.block_params(block0).to_vec();+ let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig);+ let results = builder.func.dfg.inst_results(call).to_vec();++ // Libcalls do not explicitly jump/raise on traps but instead return a+ // code indicating whether they trapped or not. This means that it's the+ // responsibility of the trampoline to check for an trapping return+ // value and raise a trap as appropriate. With the `results` above check+ // what `index` is and for each libcall that has a trapping return value+ // process it here.+ match builtin_func_index.trap_sentinel() {+ Some(TrapSentinel::Falsy) => {+ self.raise_if_host_trapped(&mut builder, vmctx, results[0]);+ }+ Some(TrapSentinel::NegativeTwo) => {+ let ty = builder.func.dfg.value_type(results[0]);+ let trapped = builder.ins().iconst(ty, -2);+ let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped);+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);+ }+ Some(TrapSentinel::Negative) => {+ let ty = builder.func.dfg.value_type(results[0]);+ let zero = builder.ins().iconst(ty, 0);+ let succeeded =+ builder+ .ins()+ .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero);+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);+ }+ Some(TrapSentinel::NegativeOne) => {+ let ty = builder.func.dfg.value_type(results[0]);+ let minus_one = builder.ins().iconst(ty, -1);+ let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one);+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);+ }+ None => {}+ }++ // And finally, return all the results of this libcall.+ if !wasm_sig.returns.is_empty() {+ builder.ins().return_(&results);+ } else {+ builder.ins().return_(&[]);+ }+ builder.finalize();++ Ok(CompiledFunctionBody {+ code: box_dyn_any_compiler_context(Some(compiler.cx)),+ needs_gc_heap: false,+ })+ }++ fn compile_module_startup(+ &self,+ translation: &ModuleTranslation<'_>,+ types: &ModuleTypesBuilder,+ key: FuncKey,+ ty: &WasmFuncType,+ ) -> Result<CompiledFunctionBody, CompileError> {+ let mut compiler = self.function_compiler();+ let context = &mut compiler.cx.codegen_context;+ context.func.signature = wasm_call_signature(&*self.isa, ty, &self.tunables);+ let (namespace, index) = key.into_raw_parts();+ context.func.name = UserFuncName::User(UserExternalName { namespace, index });+ let mut func_env = FuncEnvironment::new(self, translation, types, ty, key);+ compiler+ .cx+ .func_translator+ .translate_module_startup(&mut context.func, &mut func_env)?;+ Ok(CompiledFunctionBody {+ code: box_dyn_any_compiler_context(Some(compiler.cx)),+ needs_gc_heap: func_env.needs_gc_heap(),+ })+ }}fn box_dyn_any_compiled_function(f: CompiledFunction) -> Box<dyn Any + Send + Sync> {@@ -364,130 +588,82 @@ impl wasmtime_environ::Compiler for Compiler {})}- fn compile_array_to_wasm_trampoline(+ fn compile_trampoline(&self,- translation: &ModuleTranslation<'_>,- types: &ModuleTypesBuilder,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- let (module_index, def_func_index) = key.unwrap_array_to_wasm_trampoline();- let func_index = translation.module.func_index(def_func_index);- let sig = translation.module.functions[func_index]- .signature- .unwrap_module_type_index();- self.array_to_wasm_trampoline(- key,- FuncKey::DefinedWasmFunction(module_index, def_func_index),- types[sig].unwrap_func(),- symbol,- self.isa.pointer_bytes().vmctx_store_context().into(),- wasmtime_environ::VMCONTEXT_MAGIC,- )- }-- fn compile_wasm_to_array_trampoline(- &self,- wasm_func_ty: &WasmFuncType,+ translation: Option<&ModuleTranslation<'_>>,key: FuncKey,+ types: &ModuleTypesBuilder,symbol: &str,) -> Result<CompiledFunctionBody, CompileError> {- log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}");-- let isa = &*self.isa;- let pointer_type = isa.pointer_type();- let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables);- let array_call_sig = array_call_signature(isa);-- let mut compiler = self.function_compiler();- let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig);- let (mut builder, block0) = compiler.builder(func);+ match key {+ FuncKey::ArrayToWasmTrampoline(module_index, def_func_index) => {+ let translation = translation.unwrap();+ let func_index = translation.module.func_index(def_func_index);+ let sig = translation.module.functions[func_index]+ .signature+ .unwrap_module_type_index();+ self.array_to_wasm_trampoline(+ key,+ FuncKey::DefinedWasmFunction(module_index, def_func_index),+ types[sig].unwrap_func(),+ symbol,+ self.isa.pointer_bytes().vmctx_store_context().into(),+ wasmtime_environ::VMCONTEXT_MAGIC,+ )+ }- let args = builder.func.dfg.block_params(block0).to_vec();- let callee_vmctx = args[0];- let caller_vmctx = args[1];+ FuncKey::WasmToArrayTrampoline(ty) => {+ let ty = types[ty].unwrap_func();+ self.compile_wasm_to_array_trampoline(ty, key, symbol)+ }+ #[cfg(feature = "component-model")]+ FuncKey::ResourceDropTrampoline => {+ let ty = types.find_resource_drop_signature().unwrap();+ let ty = types[ty].unwrap_func();+ self.compile_wasm_to_array_trampoline(ty, key, symbol)+ }- // We are exiting Wasm, so save our PC and FP.- //- // Assert that the caller vmctx really is a core Wasm vmctx, since- // that's what we are assuming with our offsets below.- self.debug_assert_vmctx_kind(- &mut builder,- caller_vmctx,- wasmtime_environ::VMCONTEXT_MAGIC,- );- let ptr = isa.pointer_bytes();- let vm_store_context = builder.ins().load(- pointer_type,- MemFlagsData::trusted(),- caller_vmctx,- i32::from(ptr.vmcontext_store_context()),- );- save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context);+ FuncKey::DefinedWasmFunction(..) => {+ panic!("use `compile_function` instead")+ }- // Spill all wasm arguments to the stack in `ValRaw` slots.- let (args_base, args_len) =- self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);- let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));+ FuncKey::WasmToBuiltinTrampoline(..) | FuncKey::PatchableToBuiltinTrampoline(_) => {+ self.compile_wasm_to_builtin(key, symbol)+ }- // Load the actual callee out of the- // `VMArrayCallHostFuncContext::host_func`.- let ptr_size = isa.pointer_bytes();- let callee = builder.ins().load(- pointer_type,- MemFlagsData::trusted(),- callee_vmctx,- ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),- );+ FuncKey::PulleyHostCall(_) => unreachable!(),- // Do an indirect call to the callee.- let callee_signature = builder.func.import_signature(array_call_sig);- let call = self.call_indirect_host(- &mut builder,- HostCall::ArrayCall,- callee_signature,- callee,- &[callee_vmctx, caller_vmctx, args_base, args_len],- );+ #[cfg(feature = "component-model")]+ FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => {+ unreachable!()+ }- // Increment the "execution version" on the VMStoreContext if- // guest debugging is enabled.- if self.tunables.debug_guest {- let vmstore_ctx_ptr = builder.ins().load(- pointer_type,- MemFlagsData::trusted().with_readonly(),- caller_vmctx,- i32::from(ptr_size.vmctx_store_context()),- );- let old_version = builder.ins().load(- ir::types::I64,- MemFlagsData::trusted(),- vmstore_ctx_ptr,- i32::from(ptr_size.vmstore_context_execution_version()),- );- let new_version = builder.ins().iadd_imm(old_version, 1);- builder.ins().store(- MemFlagsData::trusted(),- new_version,- vmstore_ctx_ptr,- i32::from(ptr_size.vmstore_context_execution_version()),- );+ FuncKey::ModuleStartup(abi, module) => {+ let translation = translation.unwrap();+ let ty = match translation.module.startup {+ ModuleStartup::None => unreachable!(),+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => {+ t.unwrap_module_type_index()+ }+ };+ let ty = types[ty].unwrap_func();+ match abi {+ // This is a thin array-to-wasm shim around the actual+ // implementation.+ Abi::Array => self.array_to_wasm_trampoline(+ key,+ FuncKey::ModuleStartup(Abi::Wasm, module),+ ty,+ symbol,+ self.isa.pointer_bytes().vmctx_store_context().into(),+ wasmtime_environ::VMCONTEXT_MAGIC,+ ),+ // Delegate to a helper to finish compiling this.+ Abi::Wasm => self.compile_module_startup(translation, types, key, ty),+ Abi::Patchable => unreachable!(),+ }+ }}-- // Invoke `raise` if the callee (host) returned an error.- let succeeded = builder.func.dfg.inst_results(call)[0];- self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded);-- // Return results from the array as native return values.- let results =- self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);- builder.ins().return_(&results);- builder.finalize();-- Ok(CompiledFunctionBody {- code: box_dyn_any_compiler_context(Some(compiler.cx)),- needs_gc_heap: false,- })}fn append_code(@@ -752,103 +928,6 @@ impl wasmtime_environ::Compiler for Compiler {self.isa.create_systemv_cie()}- fn compile_wasm_to_builtin(- &self,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}");-- let isa = &*self.isa;- let ptr_size = isa.pointer_bytes();- let pointer_type = isa.pointer_type();- let sigs = BuiltinFunctionSignatures::new(self);-- let (builtin_func_index, wasm_sig) = match key {- FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)),- FuncKey::PatchableToBuiltinTrampoline(builtin) => {- let mut sig = sigs.wasm_signature(builtin);- // Patchable functions cannot return anything. We- // raise any errors that occur below so this is fine.- sig.returns.clear();- sig.call_conv = CallConv::PreserveAll;- (builtin, sig)- }- _ => unreachable!(),- };- let host_sig = sigs.host_signature(builtin_func_index);-- let mut compiler = self.function_compiler();- let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone());- let (mut builder, block0) = compiler.builder(func);- let vmctx = builder.block_params(block0)[0];-- // Debug-assert that this is the right kind of vmctx, and then- // additionally perform the "routine of the exit trampoline" of saving- // fp/pc/etc.- self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC);- let vm_store_context = builder.ins().load(- pointer_type,- MemFlagsData::trusted(),- vmctx,- ptr_size.vmcontext_store_context(),- );- save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context);-- // Now it's time to delegate to the actual builtin. Forward all our own- // arguments to the libcall itself.- let args = builder.block_params(block0).to_vec();- let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig);- let results = builder.func.dfg.inst_results(call).to_vec();-- // Libcalls do not explicitly jump/raise on traps but instead return a- // code indicating whether they trapped or not. This means that it's the- // responsibility of the trampoline to check for an trapping return- // value and raise a trap as appropriate. With the `results` above check- // what `index` is and for each libcall that has a trapping return value- // process it here.- match builtin_func_index.trap_sentinel() {- Some(TrapSentinel::Falsy) => {- self.raise_if_host_trapped(&mut builder, vmctx, results[0]);- }- Some(TrapSentinel::NegativeTwo) => {- let ty = builder.func.dfg.value_type(results[0]);- let trapped = builder.ins().iconst(ty, -2);- let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped);- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);- }- Some(TrapSentinel::Negative) => {- let ty = builder.func.dfg.value_type(results[0]);- let zero = builder.ins().iconst(ty, 0);- let succeeded =- builder- .ins()- .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero);- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);- }- Some(TrapSentinel::NegativeOne) => {- let ty = builder.func.dfg.value_type(results[0]);- let minus_one = builder.ins().iconst(ty, -1);- let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one);- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);- }- None => {}- }-- // And finally, return all the results of this libcall.- if !wasm_sig.returns.is_empty() {- builder.ins().return_(&results);- } else {- builder.ins().return_(&[]);- }- builder.finalize();-- Ok(CompiledFunctionBody {- code: box_dyn_any_compiler_context(Some(compiler.cx)),- needs_gc_heap: false,- })- }-fn compiled_function_relocation_targets<'a>(&'a self,func: &'a dyn Any,diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex ec972bb7b3..d10c1370d3 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1588,7 +1588,7 @@ impl TranslateTrap for TrapTranslator<'_> {}impl ComponentCompiler for Compiler {- fn compile_trampoline(+ fn compile_component_trampoline(&self,component: &ComponentTranslation,types: &ComponentTypesBuilder,diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex b3568e4070..c6f97a67a5 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -13,7 +13,7 @@ use crate::{};use cranelift_codegen::cursor::FuncCursor;use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};-use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm};+use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, V128Imm};use cranelift_codegen::ir::{self, BlockArg, Endianness, ExceptionTableData, ExceptionTableItem, types,};@@ -29,10 +29,12 @@ use std::mem;use wasmparser::{FuncValidator, Operator, WasmFeatures, WasmModuleResources};use wasmtime_core::math::f64_cvt_to_int_bounds;use wasmtime_environ::{- BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,- EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,- GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,- ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,+ BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex,+ DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex,+ FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,+ IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module,+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize,+ RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements,TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,};@@ -1481,8 +1483,12 @@ impl FuncEnvironment<'_> {if !self.module.globals[index].mutability {if let Some(index) = self.module.defined_global_index(index) {- let init = &self.module.global_initializers[index];- if let Some(value) = init.const_eval() {+ if let Ok(i) = self+ .module+ .global_initializers+ .binary_search_by_key(&index, |(def_index, _)| *def_index)+ {+ let (_, value) = self.module.global_initializers[i];return GlobalVariable::Constant { value };}}@@ -2618,7 +2624,7 @@ impl FuncEnvironment<'_> {) -> WasmResult<()> {let table_data = self.get_or_create_table(builder.func, table_index);let (dst, flags) = table_data.prepare_table_addr(self, builder, index);- self.emit_table_set(builder, table_index, dst, flags, value)+ self.emit_table_set(builder, table_index, dst, flags, value, true)}/// Helper to store `value` into the table address at `addr` using `flags`.@@ -2632,19 +2638,32 @@ impl FuncEnvironment<'_> {addr: ir::Value,flags: ir::MemFlagsData,value: ir::Value,+ initialized: bool,) -> WasmResult<()> {let table = self.module.tables[table_index];match table.ref_type.heap_type.top() {// GC-managed types.WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {- gc::gc_compiler(self)?.translate_write_gc_reference(- self,- builder,- table.ref_type,- addr,- value,- flags,- )+ let mut gc = gc::gc_compiler(self)?;+ if initialized {+ gc.translate_write_gc_reference(+ self,+ builder,+ table.ref_type,+ addr,+ value,+ flags,+ )+ } else {+ gc.translate_init_gc_reference(+ self,+ builder,+ table.ref_type,+ addr,+ value,+ flags,+ )+ }}// Function types.@@ -2693,7 +2712,16 @@ impl FuncEnvironment<'_> {val: ir::Value,len: ir::Value,) -> WasmResult<()> {- self.translate_entity_fill(builder, table_index, dst, val, len)+ self.translate_entity_fill(+ builder,+ CheckedEntity::Table {+ table: table_index,+ initialized: true,+ },+ dst,+ val,+ len,+ )}pub fn translate_ref_i31(@@ -3164,6 +3192,21 @@ impl FuncEnvironment<'_> {builder: &mut FunctionBuilder<'_>,global_index: GlobalIndex,val: ir::Value,+ ) -> WasmResult<()> {+ self.emit_global_set(builder, global_index, val, true)+ }++ /// Helper to translate the `global.set` instruction.+ ///+ /// The main difference with `translate_global_set`, the main entrypoint, is+ /// the `initialized` flag to indicate whether the global was previously+ /// initialized to a valid value.+ fn emit_global_set(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ global_index: GlobalIndex,+ val: ir::Value,+ initialized: bool,) -> WasmResult<()> {match self.get_or_create_global(builder.func, global_index) {GlobalVariable::Constant { .. } => {@@ -3197,14 +3240,26 @@ impl FuncEnvironment<'_> {let gv = builder.ins().global_value(self.pointer_type(), gv);let src = builder.ins().iadd_imm(gv, i64::from(offset));- gc::gc_compiler(self)?.translate_write_gc_reference(- self,- builder,- ty,- src,- val,- ir::MemFlagsData::trusted(),- )?+ let mut gc = gc::gc_compiler(self)?;+ if initialized {+ gc.translate_write_gc_reference(+ self,+ builder,+ ty,+ src,+ val,+ ir::MemFlagsData::trusted(),+ )?;+ } else {+ gc.translate_init_gc_reference(+ self,+ builder,+ ty,+ src,+ val,+ ir::MemFlagsData::trusted(),+ )?;+ }}}Ok(())@@ -3731,7 +3786,7 @@ impl FuncEnvironment<'_> {},);}- CheckedEntity::Table(table) => {+ CheckedEntity::Table { table, initialized } => {self.emit_raw_array_or_table_fill(builder,entity,@@ -3739,7 +3794,14 @@ impl FuncEnvironment<'_> {val,len_ptr,&|env, builder, addr, value| {- env.emit_table_set(builder, table, addr, ir::MemFlagsData::trusted(), value)+ env.emit_table_set(+ builder,+ table,+ addr,+ ir::MemFlagsData::trusted(),+ value,+ initialized,+ )},)?;}@@ -3761,7 +3823,9 @@ impl FuncEnvironment<'_> {)?;}// Not allowed to be written to in wasm.- CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(),+ CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {+ unreachable!()+ }}Ok(())@@ -3993,7 +4057,7 @@ impl FuncEnvironment<'_> {// Lookup the passive data segment corresponding to this data segment.// If this is an active data segment then it already has length 0 so// there's nothing to do.- let passive_index = match self.translation.passive_data_map[seg_index] {+ let runtime_index = match self.translation.runtime_data_map[seg_index] {Some(idx) => idx,None => return Ok(()),};@@ -4006,7 +4070,7 @@ impl FuncEnvironment<'_> {ir::MemFlagsData::trusted(),new_length,vmctx,- i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)).unwrap(),+ i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(),);Ok(())@@ -4080,7 +4144,9 @@ impl FuncEnvironment<'_> {CheckedEntity::Memory(_) => {assert!(matches!(src_entity,- CheckedEntity::Memory(_) | CheckedEntity::Data { .. }+ CheckedEntity::Memory(_)+ | CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_)));// A memory's elements are bytes, so the element count is already// the byte length.@@ -4098,7 +4164,7 @@ impl FuncEnvironment<'_> {// Tables/arrays are sometimes a memcpy, sometimes a per-element// loop. Delegate further to figure that out.- CheckedEntity::Table(_) | CheckedEntity::Array { .. } => self+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => self.emit_raw_array_or_table_copy(builder,dst_entity,@@ -4111,7 +4177,9 @@ impl FuncEnvironment<'_> {),// Cannot copy into a data or element segment in wasm.- CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(),+ CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {+ unreachable!()+ }}}@@ -4136,26 +4204,16 @@ impl FuncEnvironment<'_> {// Load the entity size, as `pointer_type`.let entity_size = match entity {CheckedEntity::Memory(i) => self.memory_size_in_bytes(&mut builder.cursor(), i),- CheckedEntity::Table(i) => {- let size = self.translate_table_size(builder.cursor(), i);+ CheckedEntity::Table { table, .. } => {+ let size = self.translate_table_size(builder.cursor(), table);self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), size, idx_type)}- CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment]+ CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment]{- Some(passive_index) => {- let vmctx = self.vmctx_val(&mut builder.cursor());- let offset =- i32::try_from(self.offsets.vmctx_passive_data_length(passive_index))- .unwrap();- let flags = ir::MemFlagsData::trusted();- match pointer_type {- I32 => builder.ins().load(I32, flags, vmctx, offset),- I64 => builder.ins().uload32(flags, vmctx, offset),- _ => unreachable!(),- }- }+ Some(i) => self.load_runtime_data_length_as_pointer(builder, i),None => builder.ins().iconst(pointer_type, 0),},+ CheckedEntity::RuntimeData(i) => self.load_runtime_data_length_as_pointer(builder, i),CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] {Some(passive_index) => {let vmctx = self.vmctx_val(&mut builder.cursor());@@ -4225,30 +4283,20 @@ impl FuncEnvironment<'_> {let heap = &self.heaps()[heap];builder.ins().global_value(pointer_type, heap.base)}- CheckedEntity::Table(i) => {- let table = self.get_or_create_table(builder.func, i);+ CheckedEntity::Table { table, .. } => {+ let table = self.get_or_create_table(builder.func, table);builder.ins().global_value(pointer_type, table.base_gv)}- CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment]+ CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment]{- Some(passive_index) => {- let vmctx = self.vmctx_val(&mut builder.cursor());- let offset =- i32::try_from(self.offsets.vmctx_passive_data_base(passive_index)).unwrap();- builder.ins().load(- self.pointer_type(),- ir::MemFlagsData::trusted(),- vmctx,- offset,- )- }-+ Some(runtime_index) => self.load_runtime_data_base(builder, runtime_index),// Any address should do for an active data segment, but pick// something non-null for now. Note that the length of an active// data segment is always 0, so we know that the memcpy, if any,// will be 0 elements, so the actual value here doesn't matter.None => builder.ins().iconst(pointer_type, 1),},+ CheckedEntity::RuntimeData(i) => self.load_runtime_data_base(builder, i),// Element segments are quite similar to data segments just above.CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] {Some(passive_index) => {@@ -4295,6 +4343,41 @@ impl FuncEnvironment<'_> {Ok(builder.ins().iadd(base, byte_offset))}+ fn load_runtime_data_length(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ runtime_index: RuntimeDataIndex,+ ) -> ir::Value {+ let vmctx = self.vmctx_val(&mut builder.cursor());+ let offset = i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap();+ let flags = ir::MemFlagsData::trusted();+ builder.ins().load(I32, flags, vmctx, offset)+ }++ fn load_runtime_data_length_as_pointer(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ runtime_index: RuntimeDataIndex,+ ) -> ir::Value {+ let length = self.load_runtime_data_length(builder, runtime_index);+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), length, IndexType::I32)+ }++ fn load_runtime_data_base(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ runtime_index: RuntimeDataIndex,+ ) -> ir::Value {+ let vmctx = self.vmctx_val(&mut builder.cursor());+ let offset = i32::try_from(self.offsets.vmctx_runtime_data_base(runtime_index)).unwrap();+ builder.ins().load(+ self.pointer_type(),+ ir::MemFlagsData::trusted(),+ vmctx,+ offset,+ )+ }+pub fn translate_table_copy(&mut self,builder: &mut FunctionBuilder<'_>,@@ -4304,7 +4387,20 @@ impl FuncEnvironment<'_> {src: ir::Value,len: ir::Value,) -> WasmResult<()> {- self.translate_entity_copy(builder, dst_table_index, src_table_index, dst, src, len)+ self.translate_entity_copy(+ builder,+ CheckedEntity::Table {+ table: dst_table_index,+ initialized: true,+ },+ CheckedEntity::Table {+ table: src_table_index,+ initialized: true,+ },+ dst,+ src,+ len,+ )}/// Emits a copy between two WebAssembly table or array entities.@@ -4377,7 +4473,7 @@ impl FuncEnvironment<'_> {// would mean that memcpy isn't suitable. If lazy init is// disabled though then funcrefs are just pointers so a// memcpy can be used.- CheckedEntity::Table(_) => self.tunables.table_lazy_init,+ CheckedEntity::Table { .. } => self.tunables.table_lazy_init,// The GC heap has integers representing funcrefs, so memcpy// is fine.CheckedEntity::Array { .. } => false,@@ -4385,7 +4481,9 @@ impl FuncEnvironment<'_> {// can't work.CheckedEntity::Elem(_) => true,// Not possible- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(),+ CheckedEntity::Memory(_)+ | CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_) => unreachable!(),},},};@@ -4439,7 +4537,10 @@ impl FuncEnvironment<'_> {// it's easier right now to share the internals of// `translate_table_get` which are a bit tricky with// funcrefs.- CheckedEntity::Table(i) => this.translate_table_get(builder, i, src_index)?,+ CheckedEntity::Table { table, initialized } => {+ assert!(initialized);+ this.translate_table_get(builder, table, src_index)?+ }CheckedEntity::Array { initialized, .. } => {assert!(initialized);let read_ty = src_entity.storage_type(this);@@ -4468,11 +4569,20 @@ impl FuncEnvironment<'_> {}}}- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(),+ CheckedEntity::Memory(_)+ | CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_) => unreachable!(),};match dst_entity {- CheckedEntity::Table(i) => {- this.emit_table_set(builder, i, dst, ir::MemFlagsData::trusted(), val)?;+ CheckedEntity::Table { table, initialized } => {+ this.emit_table_set(+ builder,+ table,+ dst,+ ir::MemFlagsData::trusted(),+ val,+ initialized,+ )?;}CheckedEntity::Array { initialized, .. } => {if initialized {@@ -4483,6 +4593,7 @@ impl FuncEnvironment<'_> {}CheckedEntity::Memory(_)| CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_)| CheckedEntity::Elem(_) => unreachable!(),}Ok(())@@ -4842,7 +4953,10 @@ impl FuncEnvironment<'_> {) -> WasmResult<()> {self.translate_entity_copy(builder,- table_index,+ CheckedEntity::Table {+ table: table_index,+ initialized: true,+ },CheckedEntity::Elem(ElemIndex::from_u32(seg_index)),dst,src,@@ -5645,6 +5759,386 @@ impl FuncEnvironment<'_> {pub fn is_reachable(&self) -> bool {self.stacks.reachable()}++ /// Generates the body of a `FuncKey::ModuleStartup(..)` function.+ ///+ /// This will perform all initialization of a module before any code in the+ /// module itself starts executing. This will, for example, execute all+ /// constant expressions for global initializers.+ pub fn translate_module_startup(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {+ for (i, expr) in self.translation.global_initializers.iter() {+ self.module_initialize_global(builder, *i, expr)?;+ }+ for (i, exprs) in self.translation.passive_elements.iter() {+ self.module_initialize_passive_element(builder, i, exprs)?;+ }+ for (i, init) in self.translation.table_initialization.initial_values.iter() {+ match init {+ TableInitialValue::Null => {}+ TableInitialValue::Expr(expr) => {+ self.module_initialize_table_with_fill(builder, i, expr)?;+ }+ }+ }+ for segment in self.translation.table_initialization.segments.iter() {+ self.module_initialize_table_with_segment(builder, segment)?;+ }+ match &self.translation.memory_init {+ MemoryInit::Unprocessed(_) => unreachable!(),+ MemoryInit::Processed(segments) => {+ for (memory, offset, data) in segments.iter() {+ self.module_initialize_memory_segment(builder, *memory, offset, *data)?;+ }+ }+ }+ if let Some(i) = self.translation.start_func {+ self.module_start(builder, i)?;+ }+ Ok(())+ }++ /// Initializes all "complicated" globals in a module.+ ///+ /// This will translate the given constant expression and then initialize+ /// the global. Note that this translation is done with the global not+ /// initialized, meaning that GC barriers are slightly tweaked.+ fn module_initialize_global(+ &mut self,+ builder: &mut FunctionBuilder,+ global: DefinedGlobalIndex,+ expr: &ConstExpr,+ ) -> WasmResult<()> {+ let index = self.module.global_index(global);+ let val = self.translate_const_expr(builder, expr)?;+ self.emit_global_set(builder, index, val, false)+ }++ /// Initializes all passive element segments for a module.+ ///+ /// This will execute all constant expressions for all passive element+ /// segments and initialize all `ValRaw` storage on the host. This+ /// implements the semantics of how passive element segments are evaluated+ /// once in a wasm instance.+ fn module_initialize_passive_element(+ &mut self,+ builder: &mut FunctionBuilder,+ elem: PassiveElemIndex,+ exprs: &TableSegmentElements,+ ) -> WasmResult<()> {+ let vmctx = self.vmctx_val(&mut builder.cursor());+ let libcall = self+ .builtin_functions+ .passive_elem_segment_base(builder.func);+ let idx = builder.ins().iconst(I32, i64::from(elem.as_u32()));+ let call = builder.ins().call(libcall, &[vmctx, idx]);+ let base = builder.func.dfg.first_result(call);++ // Values in `ValRaw` are always stored in little-endian.+ let flags = ir::MemFlagsData::trusted().with_endianness(Endianness::Little);++ match exprs {+ TableSegmentElements::Functions(indices) => {+ for (i, func) in indices.iter().enumerate() {+ let func = self.translate_ref_func(builder.cursor(), *func)?;+ builder.ins().store(+ flags,+ func,+ base,+ i32::try_from(i.checked_mul(16).unwrap()).unwrap(),+ );+ }+ }+ TableSegmentElements::Expressions { exprs, ty } => {+ for (i, expr) in exprs.iter().enumerate() {+ let val = self.translate_const_expr(builder, expr)?;++ let dst = builder+ .ins()+ .iadd_imm(base, i64::try_from(i.checked_mul(16).unwrap()).unwrap());+ match ty.heap_type.top() {+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {+ let ty = WasmStorageType::Val(WasmValType::Ref(*ty));+ gc::init_field_at_addr(self, builder, ty, dst, val)?;+ }+ WasmHeapTopType::Func | WasmHeapTopType::Cont => {+ builder.ins().store(+ flags,+ val,+ base,+ i32::try_from(i.checked_mul(16).unwrap()).unwrap(),+ );+ }+ }+ }+ }+ }+ Ok(())+ }++ /// Initializes all tables with non-null initializers.+ ///+ /// This function will morally execute a `table.fill` for the specified+ /// defined table in this module for the result of the constant expression+ /// provided. Note that the fill operation is done on an initialized table+ /// which slightly alters the barriers emitted for GC types.+ fn module_initialize_table_with_fill(+ &mut self,+ builder: &mut FunctionBuilder,+ table: DefinedTableIndex,+ init: &ConstExpr,+ ) -> WasmResult<()> {+ let table = self.module.table_index(table);+ let ty = self.module.tables[table];+ let val = self.translate_const_expr(builder, init)?;+ let dst = builder.ins().iconst(index_type_to_ir_type(ty.idx_type), 0);+ let len = builder.ins().iconst(+ index_type_to_ir_type(ty.idx_type),+ ty.limits.min.cast_signed(),+ );+ self.translate_entity_fill(+ builder,+ CheckedEntity::Table {+ table,+ initialized: false,+ },+ dst,+ val,+ len,+ )+ }++ /// Executes initialization for an active element segment in a module.+ ///+ /// Note that this isn't done for "precomputed" tables which won't have+ /// initializers present. Additionally note that this doesn't use+ /// `table.init` because the element segment won't actually be present+ /// anywhere at runtime.+ fn module_initialize_table_with_segment(+ &mut self,+ builder: &mut FunctionBuilder,+ segment: &TableSegment,+ ) -> WasmResult<()> {+ let offset = self.translate_const_expr(builder, &segment.offset)?;+ let segment_len = builder.ins().iconst(+ index_type_to_ir_type(self.module.tables[segment.table_index].idx_type),+ i64::try_from(segment.elements.len()).unwrap(),+ );++ // Check the bounds first before mutating anything.+ self.translate_entity_bounds_check(+ builder,+ CheckedEntity::Table {+ table: segment.table_index,+ initialized: true,+ },+ offset,+ segment_len,+ )?;++ // Re-use the `table.set` translation for making this a simple function+ // to define. That re-executes the bounds check which is a bit+ // inefficient, but can be optimized in the future.+ match &segment.elements {+ TableSegmentElements::Functions(indices) => {+ for (i, func) in indices.iter().enumerate() {+ let func = self.translate_ref_func(builder.cursor(), *func)?;+ let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap());+ self.translate_table_set(builder, segment.table_index, func, index)?;+ }+ }+ TableSegmentElements::Expressions { exprs, ty: _ } => {+ for (i, expr) in exprs.iter().enumerate() {+ let val = self.translate_const_expr(builder, expr)?;+ let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap());+ self.translate_table_set(builder, segment.table_index, val, index)?;+ }+ }+ }+ Ok(())+ }++ /// Peform initialization of an active data segment in a module.+ fn module_initialize_memory_segment(+ &mut self,+ builder: &mut FunctionBuilder,+ memory: MemoryIndex,+ offset: &MemorySegmentOffset,+ data: RuntimeDataIndex,+ ) -> WasmResult<()> {+ let mut end = None;+ let offset = match offset {+ // This is a bit subtle, but the goal here is to make a dynamic+ // deduction to see if this initialization is actually necessary+ // based on state at runtime. With a static initializer the goal is+ // to enable CoW initialization at runtime, but not all hosts have+ // that configured or supported. If it's supported then we shouldn't+ // do anything here, but if it's unsupported then this must actually+ // execute initialization.+ //+ // The host already put the addresses of all data segments into the+ // `VMContext` during `VMContext::initialize_vmctx`, and this reads+ // out the base pointer. If it's null then the host doesn't actually+ // need initialization, so this is skipped, and otherwise it's the+ // actual base pointer that's going to be used.+ MemorySegmentOffset::Static(n) => {+ let current_block = builder.current_block().unwrap();+ let init_block = builder.create_block();+ let end_block = builder.create_block();+ end = Some(end_block);++ builder.ensure_inserted_block();+ builder.insert_block_after(init_block, current_block);+ builder.insert_block_after(end_block, init_block);++ let base = self.load_runtime_data_base(builder, data);+ let null = builder.ins().iconst(self.pointer_type(), 0);+ let is_null = builder.ins().icmp(IntCC::Equal, base, null);+ builder.ins().brif(is_null, end_block, &[], init_block, &[]);+ builder.switch_to_block(init_block);+ builder.seal_block(init_block);+ let ty = index_type_to_ir_type(self.module.memories[memory].idx_type);+ builder.ins().iconst(ty, n.cast_signed())+ }+ MemorySegmentOffset::Expr(expr) => self.translate_const_expr(builder, expr)?,+ };++ // Model the initialization here as a `memory.init`.+ let len = self.load_runtime_data_length(builder, data);+ let start = builder.ins().iconst(I32, 0);+ self.translate_entity_copy(builder, memory, data, offset, start, len)?;++ // Finalize control-flow for the `MemorySegmentOffset::Static` case+ // above.+ if let Some(end) = end {+ builder.ins().jump(end, &[]);+ builder.switch_to_block(end);+ builder.seal_block(end);+ }+ Ok(())+ }++ /// Translates the final step of module initialization, executing the+ /// `start` function.+ fn module_start(&mut self, builder: &mut FunctionBuilder, func: FuncIndex) -> WasmResult<()> {+ // Manuall manage fuel around the call as the `Call` opcode does for+ // normal wasm to ensure that it's correctly accounted for.+ if self.tunables.consume_fuel {+ self.fuel_consumed += 1;+ self.fuel_increment_var(builder);+ self.fuel_save_from_var(builder);+ }+ let ty = self.module.functions[func]+ .signature+ .unwrap_module_type_index();+ let sig_ref = self.get_or_create_interned_sig_ref(builder.func, ty);+ self.translate_call(builder, Default::default(), func, sig_ref, &[])?;+ if self.tunables.consume_fuel {+ self.fuel_load_into_var(builder);+ }+ Ok(())+ }++ /// Helper function to evaluate `expr` into a value.+ ///+ /// This primarily reuses all the `translate_*` functions elsewhere on this+ /// builder toe deduplicate implementation details.+ fn translate_const_expr(+ &mut self,+ builder: &mut FunctionBuilder,+ expr: &ConstExpr,+ ) -> WasmResult<ir::Value> {+ let mut stack = Vec::new();+ for op in expr.ops() {+ if self.tunables.consume_fuel {+ self.fuel_consumed += 1;+ }+ match op {+ ConstOp::I32Const(i) => {+ stack.push(builder.ins().iconst(I32, i64::from(*i)));+ }+ ConstOp::I64Const(i) => {+ stack.push(builder.ins().iconst(I64, *i));+ }+ ConstOp::F32Const(i) => {+ stack.push(builder.ins().f32const(Ieee32::with_bits(*i)));+ }+ ConstOp::F64Const(i) => {+ stack.push(builder.ins().f64const(Ieee64::with_bits(*i)));+ }+ ConstOp::V128Const(i) => {+ let data = i.to_le_bytes().to_vec().into();+ let handle = builder.func.dfg.constants.insert(data);+ stack.push(builder.ins().vconst(ir::types::I8X16, handle));+ }+ ConstOp::GlobalGet(i) => {+ stack.push(self.translate_global_get(builder, *i)?);+ }+ ConstOp::RefI31 => {+ let val = stack.pop().unwrap();+ stack.push(self.translate_ref_i31(builder.cursor(), val)?);+ }+ ConstOp::RefNull(ty) => {+ stack.push(self.translate_ref_null(builder.cursor(), *ty)?);+ }+ ConstOp::RefFunc(i) => {+ stack.push(self.translate_ref_func(builder.cursor(), *i)?);+ }+ ConstOp::I32Add | ConstOp::I64Add => {+ let b = stack.pop().unwrap();+ let a = stack.pop().unwrap();+ stack.push(builder.ins().iadd(a, b));+ }+ ConstOp::I32Sub | ConstOp::I64Sub => {+ let b = stack.pop().unwrap();+ let a = stack.pop().unwrap();+ stack.push(builder.ins().isub(a, b));+ }+ ConstOp::I32Mul | ConstOp::I64Mul => {+ let b = stack.pop().unwrap();+ let a = stack.pop().unwrap();+ stack.push(builder.ins().imul(a, b));+ }+ ConstOp::StructNew { struct_type_index } => {+ let arity = self.struct_fields_len(*struct_type_index)?;+ let fields = stack.drain(stack.len() - arity..).collect::<_>();+ stack.push(self.translate_struct_new(builder, *struct_type_index, fields)?);+ }+ ConstOp::StructNewDefault { struct_type_index } => {+ stack.push(self.translate_struct_new_default(builder, *struct_type_index)?);+ }+ ConstOp::ArrayNew { array_type_index } => {+ let len = stack.pop().unwrap();+ let elem = stack.pop().unwrap();+ stack.push(self.translate_array_new(builder, *array_type_index, elem, len)?);+ }+ ConstOp::ArrayNewDefault { array_type_index } => {+ let len = stack.pop().unwrap();+ stack.push(self.translate_array_new_default(+ builder,+ *array_type_index,+ len,+ )?);+ }+ ConstOp::ArrayNewFixed {+ array_type_index,+ array_size,+ } => {+ let array_size = usize::try_from(*array_size).unwrap();+ let elems = stack.drain(stack.len() - array_size..).collect::<Vec<_>>();+ stack.push(self.translate_array_new_fixed(+ builder,+ *array_type_index,+ &elems,+ )?);+ }+ ConstOp::ExternConvertAny => {}+ ConstOp::AnyConvertExtern => {}+ }+ }+ let ret = stack.pop().unwrap();+ assert!(stack.is_empty());+ Ok(ret)+ }}// Helper function to convert an `IndexType` to an `ir::Type`.@@ -5698,7 +6192,12 @@ enum CheckedEntity {/// A WebAssembly linear memory.Memory(MemoryIndex),/// A WebAssembly table.- Table(TableIndex),+ Table {+ /// The index of the table that's being accessed.+ table: TableIndex,+ /// Whether or not this table has been initialized yet.+ initialized: bool,+ },/// A WebAssembly data segment loaded in chunks of `element_size` bytes.////// For `memory.init` this will have `element_size = 1`, but for@@ -5711,6 +6210,11 @@ enum CheckedEntity {/// or possibly more for array initialization.element_size: u32,},+ /// A refinement of `Data` above which uses a `RuntimeDataIndex`.+ ///+ /// This isn't reachable from core wasm but is used during module+ /// startup.+ RuntimeData(RuntimeDataIndex),/// A WebAssembly passive element segment.Elem(ElemIndex),/// An `arrayref` with the specified type.@@ -5731,9 +6235,9 @@ impl From<MemoryIndex> for CheckedEntity {}}-impl From<TableIndex> for CheckedEntity {- fn from(table_index: TableIndex) -> Self {- CheckedEntity::Table(table_index)+impl From<RuntimeDataIndex> for CheckedEntity {+ fn from(runtime_data_index: RuntimeDataIndex) -> Self {+ CheckedEntity::RuntimeData(runtime_data_index)}}@@ -5742,10 +6246,11 @@ impl CheckedEntity {fn index_type(&self, env: &FuncEnvironment) -> IndexType {match *self {CheckedEntity::Memory(i) => env.memory(i).idx_type,- CheckedEntity::Table(i) => env.table(i).idx_type,- CheckedEntity::Data { .. } | CheckedEntity::Array { .. } | CheckedEntity::Elem(_) => {- IndexType::I32- }+ CheckedEntity::Table { table, .. } => env.table(table).idx_type,+ CheckedEntity::Data { .. }+ | CheckedEntity::Array { .. }+ | CheckedEntity::Elem(_)+ | CheckedEntity::RuntimeData(_) => IndexType::I32,}}@@ -5756,9 +6261,9 @@ impl CheckedEntity {func: &mut ir::Function,) -> WasmResult<u32> {Ok(match *self {- CheckedEntity::Memory(_) => 1,+ CheckedEntity::Memory(_) | CheckedEntity::RuntimeData(_) => 1,CheckedEntity::Data { element_size, .. } => element_size,- CheckedEntity::Table(table) => env.get_or_create_table(func, table).element_size,+ CheckedEntity::Table { table, .. } => env.get_or_create_table(func, table).element_size,CheckedEntity::Array { ty, .. } => env.array_layout(ty)?.elem_size,CheckedEntity::Elem(_) => 16,})@@ -5770,8 +6275,9 @@ impl CheckedEntity {CheckedEntity::Memory(_)| CheckedEntity::Data {element_size: 1, ..- } => WasmStorageType::I8,- CheckedEntity::Table(table) => {+ }+ | CheckedEntity::RuntimeData(_) => WasmStorageType::I8,+ CheckedEntity::Table { table, .. } => {WasmStorageType::Val(WasmValType::Ref(env.table(table).ref_type))}CheckedEntity::Array { ty, .. } => {@@ -5786,10 +6292,10 @@ impl CheckedEntity {/// Returns the trap code to use when an index into this entity is out-of-bounds.fn oob_trap_code(&self) -> ir::TrapCode {match self {- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => {- ir::TrapCode::HEAP_OUT_OF_BOUNDS- }- CheckedEntity::Table(_) | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS,+ CheckedEntity::Memory(_)+ | CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_) => ir::TrapCode::HEAP_OUT_OF_BOUNDS,+ CheckedEntity::Table { .. } | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS,CheckedEntity::Array { .. } => TRAP_ARRAY_OUT_OF_BOUNDS,}}@@ -5798,13 +6304,22 @@ impl CheckedEntity {/// bit patterns.fn allows_memset(&self, env: &FuncEnvironment) -> bool {match self {- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } | CheckedEntity::Array { .. } => {- true+ CheckedEntity::Memory(_)+ | CheckedEntity::Data { .. }+ | CheckedEntity::RuntimeData(_) => true,+ CheckedEntity::Array { ty, .. } => {+ let array_ty = env.types.unwrap_array(*ty).unwrap();+ // Most GC references need barriers, funcrefs need intern-ing,+ // etc. Disallow memset on all reference types.+ !matches!(+ array_ty.0.element_type,+ WasmStorageType::Val(WasmValType::Ref(_))+ )}CheckedEntity::Elem(_) => false,// Tables that are lazily initialized can't be memset because the// initialized bit needs to be set when storing values.- CheckedEntity::Table(_) => !env.tunables.table_lazy_init,+ CheckedEntity::Table { .. } => !env.tunables.table_lazy_init,}}}diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex 9857146f03..7dd7b5905f 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -162,6 +162,20 @@ pub trait GcCompiler {flags: ir::MemFlagsData,) -> WasmResult<()>;+ /// Same as [`Self::translate_write_gc_reference`] except that the+ /// destination address has not been previously initialized.+ ///+ /// This will, for example, skip barriers on the destination address.+ fn translate_init_gc_reference(+ &mut self,+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ty: WasmRefType,+ dst: ir::Value,+ new_val: ir::Value,+ flags: ir::MemFlagsData,+ ) -> WasmResult<()>;+/// Stores `val` into `addr` with the `ty` specified.////// This initializes a previously-uninitialized field, so barriers on thediff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rsindex 5c0e89a244..79cbe1841e 100644--- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs+++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs@@ -400,6 +400,19 @@ impl GcCompiler for CopyingCompiler {unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)}+ fn translate_init_gc_reference(+ &mut self,+ _func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ty: WasmRefType,+ dst: ir::Value,+ new_val: ir::Value,+ flags: ir::MemFlagsData,+ ) -> WasmResult<()> {+ // No write barrier needed for the copying collector.+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)+ }+fn init_field(&mut self,func_env: &mut FuncEnvironment<'_>,diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rsindex 7938257a4b..d59c3dc6b2 100644--- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs+++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs@@ -330,109 +330,6 @@ impl DrcCompiler {);builder.ins().store(GC_MEMFLAGS, new_reserved, ptr, 0);}-- /// Write to an uninitialized GC reference field, initializing it.- ///- /// ```text- /// *dst = new_val- /// ```- ///- /// Doesn't need to do a full write barrier: we don't have an old reference- /// that is being overwritten and needs its refcount decremented, just a new- /// reference whose count should be incremented.- fn translate_init_gc_reference(- &mut self,- func_env: &mut FuncEnvironment<'_>,- builder: &mut FunctionBuilder,- ty: WasmRefType,- dst: ir::Value,- new_val: ir::Value,- flags: ir::MemFlagsData,- ) -> WasmResult<()> {- let (ref_ty, _) = func_env.reference_type(ty.heap_type);-- // Special case for references to uninhabited bottom types: see- // `translate_write_gc_reference` for details.- if let WasmHeapType::None = ty.heap_type {- if ty.nullable {- let null = builder.ins().iconst(ref_ty, 0);- builder.ins().store(flags, null, dst, 0);- } else {- let zero = builder.ins().iconst(ir::types::I32, 0);- builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);- }- return Ok(());- };-- // Special case for `i31ref`s: no need for any barriers.- if let WasmHeapType::I31 = ty.heap_type {- return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags);- }-- // Our initialization barrier for GC references being copied out of the- // stack and initializing a table/global/struct field/etc... is roughly- // equivalent to the following pseudo-CLIF:- //- // ```- // current_block:- // ...- // let new_val_is_null_or_i31 = ...- // brif new_val_is_null_or_i31, continue_block, inc_ref_block- //- // inc_ref_block:- // let ref_count = load new_val.ref_count- // let new_ref_count = iadd_imm ref_count, 1- // store new_val.ref_count, new_ref_count- // jump check_old_val_block- //- // continue_block:- // store dst, new_val- // ...- // ```- //- // This write barrier is responsible for ensuring that the new value's- // ref count is incremented now that the table/global/struct/etc... is- // holding onto it.-- let current_block = builder.current_block().unwrap();- let inc_ref_block = builder.create_block();- let continue_block = builder.create_block();-- builder.ensure_inserted_block();- builder.insert_block_after(inc_ref_block, current_block);- builder.insert_block_after(continue_block, inc_ref_block);-- // Current block: check whether the new value is non-null and- // non-i31. If so, branch to the `inc_ref_block`.- log::trace!("DRC initialization barrier: check if the value is null or i31");- let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val);- builder.ins().brif(- new_val_is_null_or_i31,- continue_block,- &[],- inc_ref_block,- &[],- );-- // Block to increment the ref count of the new value when it is non-null- // and non-i31.- builder.switch_to_block(inc_ref_block);- builder.seal_block(inc_ref_block);- log::trace!("DRC initialization barrier: increment the ref count of the initial value");- self.mutate_ref_count(func_env, builder, new_val, 1);- builder.ins().jump(continue_block, &[]);-- // Join point after we're done with the GC barrier: do the actual store- // to initialize the field.- builder.switch_to_block(continue_block);- builder.seal_block(continue_block);- log::trace!(- "DRC initialization barrier: finally, store into {dst:?} to initialize the field"- );- unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?;-- Ok(())- }}impl GcCompiler for DrcCompiler {@@ -959,6 +856,109 @@ impl GcCompiler for DrcCompiler {Ok(())}+ /// Write to an uninitialized GC reference field, initializing it.+ ///+ /// ```text+ /// *dst = new_val+ /// ```+ ///+ /// Doesn't need to do a full write barrier: we don't have an old reference+ /// that is being overwritten and needs its refcount decremented, just a new+ /// reference whose count should be incremented.+ fn translate_init_gc_reference(+ &mut self,+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ty: WasmRefType,+ dst: ir::Value,+ new_val: ir::Value,+ flags: ir::MemFlagsData,+ ) -> WasmResult<()> {+ let (ref_ty, _) = func_env.reference_type(ty.heap_type);++ // Special case for references to uninhabited bottom types: see+ // `translate_write_gc_reference` for details.+ if let WasmHeapType::None = ty.heap_type {+ if ty.nullable {+ let null = builder.ins().iconst(ref_ty, 0);+ builder.ins().store(flags, null, dst, 0);+ } else {+ let zero = builder.ins().iconst(ir::types::I32, 0);+ builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);+ }+ return Ok(());+ };++ // Special case for `i31ref`s: no need for any barriers.+ if let WasmHeapType::I31 = ty.heap_type {+ return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags);+ }++ // Our initialization barrier for GC references being copied out of the+ // stack and initializing a table/global/struct field/etc... is roughly+ // equivalent to the following pseudo-CLIF:+ //+ // ```+ // current_block:+ // ...+ // let new_val_is_null_or_i31 = ...+ // brif new_val_is_null_or_i31, continue_block, inc_ref_block+ //+ // inc_ref_block:+ // let ref_count = load new_val.ref_count+ // let new_ref_count = iadd_imm ref_count, 1+ // store new_val.ref_count, new_ref_count+ // jump check_old_val_block+ //+ // continue_block:+ // store dst, new_val+ // ...+ // ```+ //+ // This write barrier is responsible for ensuring that the new value's+ // ref count is incremented now that the table/global/struct/etc... is+ // holding onto it.++ let current_block = builder.current_block().unwrap();+ let inc_ref_block = builder.create_block();+ let continue_block = builder.create_block();++ builder.ensure_inserted_block();+ builder.insert_block_after(inc_ref_block, current_block);+ builder.insert_block_after(continue_block, inc_ref_block);++ // Current block: check whether the new value is non-null and+ // non-i31. If so, branch to the `inc_ref_block`.+ log::trace!("DRC initialization barrier: check if the value is null or i31");+ let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val);+ builder.ins().brif(+ new_val_is_null_or_i31,+ continue_block,+ &[],+ inc_ref_block,+ &[],+ );++ // Block to increment the ref count of the new value when it is non-null+ // and non-i31.+ builder.switch_to_block(inc_ref_block);+ builder.seal_block(inc_ref_block);+ log::trace!("DRC initialization barrier: increment the ref count of the initial value");+ self.mutate_ref_count(func_env, builder, new_val, 1);+ builder.ins().jump(continue_block, &[]);++ // Join point after we're done with the GC barrier: do the actual store+ // to initialize the field.+ builder.switch_to_block(continue_block);+ builder.seal_block(continue_block);+ log::trace!(+ "DRC initialization barrier: finally, store into {dst:?} to initialize the field"+ );+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?;++ Ok(())+ }+/// Write to an uninitialized field or element inside a GC object.fn init_field(&mut self,diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/enabled/null.rsindex 70818ab69a..789bdfb8ea 100644--- a/crates/cranelift/src/func_environ/gc/enabled/null.rs+++ b/crates/cranelift/src/func_environ/gc/enabled/null.rs@@ -371,6 +371,18 @@ impl GcCompiler for NullCompiler {unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)}+ fn translate_init_gc_reference(+ &mut self,+ _func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ty: WasmRefType,+ dst: ir::Value,+ new_val: ir::Value,+ flags: ir::MemFlagsData,+ ) -> WasmResult<()> {+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)+ }+fn init_field(&mut self,func_env: &mut FuncEnvironment<'_>,diff --git a/crates/cranelift/src/translate/func_translator.rs b/crates/cranelift/src/translate/func_translator.rsindex b687574550..80323def5c 100644--- a/crates/cranelift/src/translate/func_translator.rs+++ b/crates/cranelift/src/translate/func_translator.rs@@ -96,6 +96,26 @@ impl FuncTranslator {log::trace!("translated Wasm to CLIF:\n{}", func.display());Ok(())}++ /// Translate the `FuncKey::ModuleStartup` function.+ pub fn translate_module_startup(+ &mut self,+ func: &mut ir::Function,+ environ: &mut FuncEnvironment<'_>,+ ) -> WasmResult<()> {+ let mut builder = FunctionBuilder::new(func, &mut self.func_ctx);+ let entry_block = builder.create_block();+ builder.append_block_params_for_function_params(entry_block);+ builder.switch_to_block(entry_block);+ builder.seal_block(entry_block);+ builder.ensure_inserted_block();+ environ.before_translate_function(&mut builder)?;+ environ.translate_module_startup(&mut builder)?;+ environ.after_translate_function(&mut builder)?;+ builder.ins().return_(&[]);+ builder.finalize();+ Ok(())+ }}/// Declare local variables for the signature parameters that correspond to WebAssembly locals.diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rsindex d3cd69a872..4dc2fa6d01 100644--- a/crates/environ/src/compile/mod.rs+++ b/crates/environ/src/compile/mod.rs@@ -5,7 +5,7 @@ use crate::error::Result;use crate::prelude::*;use crate::{DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex,- TripleExt, Tunables, WasmError, WasmFuncType, obj,+ TripleExt, Tunables, WasmError, obj,};use object::write::{Object, SymbolId};use object::{Architecture, BinaryFormat, FileFlags};@@ -261,38 +261,11 @@ pub trait Compiler: Send + Sync {////// The trampoline should save the necessary state to record the/// host-to-Wasm transition (e.g. registers used for fast stack walking).- fn compile_array_to_wasm_trampoline(- &self,- translation: &ModuleTranslation<'_>,- types: &ModuleTypesBuilder,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError>;-- /// Compile a trampoline for a Wasm caller calling a array callee with the- /// given signature.- ///- /// The trampoline should save the necessary state to record the- /// Wasm-to-host transition (e.g. registers used for fast stack walking).- fn compile_wasm_to_array_trampoline(- &self,- wasm_func_ty: &WasmFuncType,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError>;-- /// Creates a trampoline that can be used to call Wasmtime's implementation- /// of the builtin function specified by `index`.- ///- /// The trampoline created can technically have any ABI but currently has- /// the native ABI. This will then perform all the necessary duties of an- /// exit trampoline from wasm and then perform the actual dispatch to the- /// builtin function. Builtin functions in Wasmtime are stored in an array- /// in all `VMContext` pointers, so the call to the host is an indirect- /// call.- fn compile_wasm_to_builtin(+ fn compile_trampoline(&self,+ translation: Option<&ModuleTranslation<'_>>,key: FuncKey,+ types: &ModuleTypesBuilder,symbol: &str,) -> Result<CompiledFunctionBody, CompileError>;diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rsindex ca409dae97..c9e1f302ad 100644--- a/crates/environ/src/compile/module_artifacts.rs+++ b/crates/environ/src/compile/module_artifacts.rs@@ -5,8 +5,7 @@ use crate::WasmChecksum;use crate::error::{Result, bail};use crate::prelude::*;use crate::{- CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata,- ModuleTranslation, Tunables, obj,+ CompiledModuleInfo, DebugInfoData, FunctionName, Metadata, ModuleTranslation, Tunables, obj,};use object::SectionKind;use object::write::{Object, SectionId, StandardSegment, WritableBuffer};@@ -118,9 +117,8 @@ impl<'a> ObjectBuilder<'a> {mut module,debuginfo,has_unparsed_debuginfo,- data,data_align,- passive_data,+ runtime_data,wasm,..} = translation;@@ -128,20 +126,15 @@ impl<'a> ObjectBuilder<'a> {// Place all data from the wasm module into a section which will the// source of the data later at runtime. This additionally keeps track of// the offset of- let mut total_data_len = 0;let data_offset = self.obj.append_section_data(self.data, &[], data_align.unwrap_or(1));- for (i, data) in data.iter().enumerate() {- // The first data segment has its alignment specified as the alignment- // for the entire section, but everything afterwards is adjacent so it- // has alignment of 1.+ for (i, (_, data)) in runtime_data.iter().enumerate() {+ // The first data segment has its alignment specified as the+ // alignment for the entire section, but everything afterwards is+ // adjacent so it has alignment of 1.let align = if i == 0 { data_align.unwrap_or(1) } else { 1 };self.obj.append_section_data(self.data, data, align);- total_data_len += data.len();- }- for data in passive_data.iter() {- self.obj.append_section_data(self.data, data, 1);}// If any names are present in the module then the `ELF_NAME_DATA` section@@ -172,34 +165,12 @@ impl<'a> ObjectBuilder<'a> {}}- // Data offsets in `MemoryInitialization` are offsets within the- // `translation.data` list concatenated which is now present in the data- // segment that's appended to the object. Increase the offsets by- // `self.data_size` to account for any previously added module.- let data_offset = u32::try_from(data_offset).unwrap();- match &mut module.memory_initialization {- MemoryInitialization::Segmented(list) => {- for segment in list {- segment.data.start = segment.data.start.checked_add(data_offset).unwrap();- segment.data.end = segment.data.end.checked_add(data_offset).unwrap();- }- }- MemoryInitialization::Static { map } => {- for (_, segment) in map {- if let Some(segment) = segment {- segment.data.start = segment.data.start.checked_add(data_offset).unwrap();- segment.data.end = segment.data.end.checked_add(data_offset).unwrap();- }- }- }- }-// Data offsets for passive data are relative to the start of- // `translation.passive_data` which was appended to the data segment+ // `translation.runtime_data` which was appended to the data segment// of this object, after active data in `translation.data`. Update the// offsets to account prior modules added in addition to active data.- let data_offset = data_offset + u32::try_from(total_data_len).unwrap();- for (_, range) in module.passive_data.iter_mut() {+ let data_offset = u32::try_from(data_offset).unwrap();+ for (_, range) in module.runtime_data.iter_mut() {range.start = range.start.checked_add(data_offset).unwrap();range.end = range.end.checked_add(data_offset).unwrap();}diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rsindex be2fc09c0d..b5b02b9e6f 100644--- a/crates/environ/src/compile/module_environ.rs+++ b/crates/environ/src/compile/module_environ.rs@@ -1,20 +1,19 @@use crate::error::{OutOfMemory, Result, bail};use crate::module::{- FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment,- TableSegmentElements,+ FuncRefIndex, Initializer, MemoryInitialization, Module, TableSegment, TableSegmentElements,};use crate::prelude::*;use crate::{- ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex,- EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex,- ModuleInternedTypeIndex, ModuleTypesBuilder, PanicOnOom as _, PassiveDataIndex,- PassiveElemIndex, PrimaryMap, SizeOverflow, StaticMemoryInitializer, StaticModuleIndex,- TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError,+ ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, DefinedGlobalIndex, ElemIndex,+ EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType,+ MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex, ModuleStartup, ModuleTypesBuilder,+ PanicOnOom as _, PassiveElemIndex, PrimaryMap, RuntimeDataIndex, StaticModuleIndex, TableIndex,+ TableInitialValue, TableInitialization, Tag, TagIndex, Tunables, TypeConvert, TypeIndex,WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,};+use alloc::borrow::Cow;use cranelift_entity::SecondaryMap;use cranelift_entity::packed_option::ReservedValue;-use std::borrow::Cow;use std::collections::HashMap;use std::mem;use std::path::PathBuf;@@ -84,13 +83,6 @@ pub struct ModuleTranslation<'data> {/// configuration.pub has_unparsed_debuginfo: bool,- /// List of data segments found in this module which should be concatenated- /// together for the final compiled artifact.- ///- /// These data segments, when concatenated, are indexed by the- /// `MemoryInitializer` type.- pub data: Vec<Cow<'data, [u8]>>,-/// The desired alignment of `data` in the final data section of the object/// file that we'll emit.///@@ -100,20 +92,21 @@ pub struct ModuleTranslation<'data> {pub data_align: Option<u64>,/// Map from a data segment to whether it's a passive data segment or not.- pub passive_data_map: SecondaryMap<DataIndex, Option<PassiveDataIndex>>,+ pub runtime_data_map: SecondaryMap<DataIndex, Option<RuntimeDataIndex>>,/// Map from an elem segment to whether it's a passive elem segment or not.pub passive_elem_map: SecondaryMap<ElemIndex, Option<PassiveElemIndex>>,- /// Total size of all data pushed onto `data` so far.- total_data: u32,-/// List of passive element segments found in this module which will get/// concatenated for the final artifact.- pub passive_data: Vec<&'data [u8]>,+ pub runtime_data: PrimaryMap<RuntimeDataIndex, Cow<'data, [u8]>>,- /// Total size of all passive data pushed into `passive_data` so far.- total_passive_data: u32,+ /// Record of all passive data segments that this module contains.+ ///+ /// These are processed during [`ModuleTranslation::finalize_memory_init`]+ /// and eventually moved over into the `runtime_data` list above. Until+ /// then, however, their `RuntimeDataIndex` is not yet assigned.+ passive_data: Vec<(DataIndex, &'data [u8])>,/// When we're parsing the code section this will be incremented so we know/// which function is currently being defined.@@ -122,6 +115,62 @@ pub struct ModuleTranslation<'data> {/// The type information of the current module made available at the end of the/// validation process.types: Option<Types>,++ /// The WebAssembly `start` function, if defined.+ pub start_func: Option<FuncIndex>,++ /// Initializers for `global` values which aren't considered "simple".+ ///+ /// These initializers are later compiled into a "module startup" function.+ pub global_initializers: Vec<(DefinedGlobalIndex, ConstExpr)>,++ /// Definitions of all passive elements found within a module.+ ///+ /// This maps passive element segments to their definition, either functions+ /// or expressions-basd.+ pub passive_elements: PrimaryMap<PassiveElemIndex, TableSegmentElements>,++ /// WebAssembly table initialization data, per table.+ ///+ /// This keeps track of all per-table initialization (e.g. initial value for+ /// non-null tables) as well as active element segments. This is processed+ /// and refined by [`ModuleTranslation::finalize_table_init`] after+ /// translation.+ pub table_initialization: TableInitialization,++ /// WebAssembly memory initialization.+ ///+ /// This is held here in an `Unprocessed` form during translation, and then+ /// this is later finished with [`ModuleTranslation::finalize_memory_init`].+ pub memory_init: MemoryInit<'data>,+}++/// Different forms of memory initialization that happens for a module.+pub enum MemoryInit<'a> {+ /// Raw active data segments that are being applied for an instance.+ ///+ /// This list contains the raw data which hasn't yet been processed into+ /// `RuntimeDataIndex`, for example. This is later processed during+ /// [`ModuleTranslation::finalize_memory_init`] to optionally shuffle things+ /// around.+ Unprocessed(Vec<MemoryInitializer<'a>>),++ /// Finalized memory initialization to be executed after+ /// [`ModuleTranslation::finalize_memory_init`] has run. This represents+ /// active data segments which may have been merged from the `Unprocessed`+ /// list above, and may or may not have statically know offsets.+ Processed(Vec<(MemoryIndex, MemorySegmentOffset, RuntimeDataIndex)>),+}++/// Offset within [`MemoryInit::Processed`] which indicates the initial offset+/// a data segment is applied at.+pub enum MemorySegmentOffset {+ /// A "complicated" constant expression deferred to get evaluated at runtime+ /// with compiled code.+ Expr(ConstExpr),++ /// A statically known, in-bounds, constant value.+ Static(u64),}impl<'data> ModuleTranslation<'data> {@@ -136,15 +185,18 @@ impl<'data> ModuleTranslation<'data> {exported_signatures: Vec::default(),debuginfo: DebugInfoData::default(),has_unparsed_debuginfo: false,- data: Vec::default(),data_align: None,- total_data: 0,- passive_data: Vec::default(),- total_passive_data: 0,+ runtime_data: Default::default(),code_index: 0,types: None,- passive_data_map: Default::default(),+ runtime_data_map: Default::default(),passive_elem_map: Default::default(),+ start_func: None,+ global_initializers: Vec::new(),+ passive_elements: Default::default(),+ table_initialization: Default::default(),+ memory_init: MemoryInit::Unprocessed(Vec::new()),+ passive_data: Default::default(),}}@@ -417,9 +469,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {self.result.module.needs_gc_heap |= table.ref_type.is_vmgcref_type();self.result.module.tables.push(table)?;let init = match init {- wasmparser::TableInit::RefNull => TableInitialValue::Null {- precomputed: TryVec::new(),- },+ wasmparser::TableInit::RefNull => TableInitialValue::Null,wasmparser::TableInit::Expr(expr) => {let (init, escaped) = ConstExpr::from_wasmparser(self, expr)?;for f in escaped {@@ -428,11 +478,11 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {TableInitialValue::Expr(init)}};+ self.result.table_initialization.initial_values.push(init)?;self.result.module.table_initialization- .initial_values- .push(init)?;+ .push(Default::default())?;}}@@ -475,8 +525,24 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {self.flag_func_escaped(f);}let ty = self.convert_global_type(&ty)?;- self.result.module.globals.push(ty)?;- self.result.module.global_initializers.push(initializer)?;+ let index = self.result.module.globals.push(ty)?;+ let defined_index = self.result.module.defined_global_index(index).unwrap();+ match initializer.const_eval() {+ Some(val) => {+ self.result+ .module+ .global_initializers+ .push((defined_index, val))?;+ }+ None => {+ // "Complicated" global initializers are deferred+ // to get evaluated in the startup function.+ self.require_startup_func();+ self.result+ .global_initializers+ .push((defined_index, initializer));+ }+ }}}@@ -508,9 +574,12 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {self.validator.start_section(func, &range)?;let func_index = FuncIndex::from_u32(func);- self.flag_func_escaped(func_index);- debug_assert!(self.result.module.start_func.is_none());- self.result.module.start_func = Some(func_index);+ debug_assert!(self.result.start_func.is_none());+ self.result.start_func = Some(func_index);++ // To make startup a bit easier, invoking the `start` function+ // is a responsibility deferred to the startup function.+ self.require_startup_func();}Payload::ElementSection(elements) => {@@ -566,19 +635,27 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;debug_assert!(escaped.is_empty());- self.result.module.table_initialization.segments.push(- TableSegment {+ self.result+ .table_initialization+ .segments+ .push(TableSegment {table_index,offset,elements,- },- )?;+ })?;None}ElementKind::Passive => {- let passive_index =- self.result.module.passive_elements.push(elements)?;+ let passive_index = self+ .result+ .module+ .passive_elements+ .push((elements.ty(), elements.len()))?;+ self.result.passive_elements.push(elements);+ // One-time initialization of passive element+ // segments is deferred to the startup function.+ self.require_startup_func();Some(passive_index)}@@ -639,14 +716,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {Payload::DataSection(data) => {self.validator.data_section(&data)?;- let initializers = match &mut self.result.module.memory_initialization {- MemoryInitialization::Segmented(i) => i,- _ => unreachable!(),- };-- let cnt = usize::try_from(data.count()).unwrap();- initializers.reserve_exact(cnt)?;- self.result.data.reserve_exact(cnt);+ assert!(self.result.module.memory_initialization.is_segmented());for (index, entry) in data.into_iter().enumerate() {let wasmparser::Data {@@ -655,53 +725,28 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {range: _,} = entry?;let data_index = DataIndex::from_u32(index.try_into().unwrap());- let mk_range = |total: &mut u32| -> Result<_, WasmError> {- let range = u32::try_from(data.len())- .ok()- .and_then(|size| {- let start = *total;- let end = start.checked_add(size)?;- Some(start..end)- })- .ok_or_else(|| {- WasmError::Unsupported(format!(- "more than 4 gigabytes of data in wasm module",- ))- })?;- *total += range.end - range.start;- Ok(range)- };- let passive_index = match kind {+ match kind {DataKind::Active {memory_index,offset_expr,} => {- let range = mk_range(&mut self.result.total_data)?;let memory_index = MemoryIndex::from_u32(memory_index);let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;debug_assert!(escaped.is_empty());- let initializers = match &mut self.result.module.memory_initialization {- MemoryInitialization::Segmented(i) => i,- _ => unreachable!(),+ let MemoryInit::Unprocessed(list) = &mut self.result.memory_init else {+ panic!("memory initializers should be unprocessed at this point");};- initializers.push(MemoryInitializer {+ list.push(MemoryInitializer {memory_index,offset,- data: range,- })?;- self.result.data.push(data.into());- None+ data,+ });}DataKind::Passive => {- let range = mk_range(&mut self.result.total_passive_data)?;- self.result.passive_data.push(data);- Some(self.result.module.passive_data.push(range)?)+ self.result.passive_data.push((data_index, data));}- };- self.result- .passive_data_map- .insert(data_index, passive_index);+ }}}@@ -952,6 +997,10 @@ and for re-adding support for interface types you can see this issue:}Ok(())}++ fn require_startup_func(&mut self) {+ self.result.require_startup_func(self.types);+ }}impl TypeConvert for ModuleEnvironment<'_, '_> {@@ -971,6 +1020,75 @@ impl TypeConvert for ModuleEnvironment<'_, '_> {}impl ModuleTranslation<'_> {+ /// Called after translation is complete this will finalize the memory+ /// initialization strategy for this module.+ ///+ /// This will notably use `Self::try_static_init` to attempt to massage+ /// data segments to being CoW-init-friendly. Afterwards the+ /// `self.memory_init` field is transitioned from `Unprocessed` to+ /// `Processed`.+ pub fn finalize_memory_init(+ &mut self,+ tunables: &Tunables,+ page_size: u64,+ max_image_size_always_allowed: u64,+ types: &mut ModuleTypesBuilder,+ ) {+ if tunables.memory_init_cow {+ self.try_static_init(page_size, max_image_size_always_allowed);+ }++ // If any memory is statically initialized, and if that memory has an+ // initial data segment, then a startup function is at least+ // conditionally needed if the memory needs initialization. Flag as such+ // here.+ if let MemoryInitialization::Static { map } = &self.module.memory_initialization {+ if map.iter().any(|(_, v)| v.is_some()) {+ self.require_startup_func_if_memories_need_init(types);+ }+ }++ // If, after `try_static_init`, initializers are still `Unprocessed`+ // then this is the catch-all fallback path for initialization. All+ // segments are promoted into `self.runtime_data` and then the+ // initialization is rewritten to `Processed`.+ if let MemoryInit::Unprocessed(list) = &mut self.memory_init {+ let segments = mem::take(list);+ let mut new_initializers = Vec::new();+ for segment in segments {+ new_initializers.push((+ segment.memory_index,+ MemorySegmentOffset::Expr(segment.offset),+ self.runtime_data.push(segment.data.into()),+ ));+ }+ if !new_initializers.is_empty() {+ self.require_startup_func(types);+ }+ self.memory_init = MemoryInit::Processed(new_initializers);+ }++ // At this point append all passive data to the `runtime_data` list.+ // This notably occurs after `try_static_init` above to ensure that the+ // page-aligned data for static initialization, if applicable, comes+ // first.+ for (data_index, segment) in self.passive_data.iter() {+ let runtime_index = self.runtime_data.push((*segment).into());+ self.runtime_data_map+ .insert(*data_index, Some(runtime_index));+ }++ // And, finally, record all chunks from `self.runtime_data` within+ // `self.module.runtime_data` as well.+ let mut cur = 0;+ for (idx, data) in self.runtime_data.iter() {+ let len = u32::try_from(data.len()).unwrap();+ let i = self.module.runtime_data.push(cur..cur + len).panic_on_oom();+ cur += len;+ assert_eq!(idx, i);+ }+ }+/// Attempts to convert segmented memory initialization into static/// initialization for the module that this translation represents.///@@ -1001,24 +1119,20 @@ impl ModuleTranslation<'_> {/// modules that have some dynamically-placed data segments. But,/// for now, this is sufficient to allow a system that "knows what/// it's doing" to always get static init.- pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) {- // This method only attempts to transform a `Segmented` memory init- // into a `Static` one, no other state.- if !self.module.memory_initialization.is_segmented() {- return;- }+ fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) {+ let segments = match &mut self.memory_init {+ MemoryInit::Unprocessed(list) => list,+ _ => return,+ };// First a dry run of memory initialization is performed. This// collects information about the extent of memory initialized for each// memory as well as the size of all data segments being copied in.- struct Memory {+ struct Memory<'a> {data_size: u64,min_addr: u64,max_addr: u64,- // The `usize` here is a pointer into `self.data` which is the list- // of data segments corresponding to what was found in the original- // wasm module.- segments: Vec<(usize, StaticMemoryInitializer)>,+ segments: Vec<(u64, &'a [u8])>,}let mut info = PrimaryMap::with_capacity(self.module.memories.len());for _ in 0..self.module.memories.len() {@@ -1030,59 +1144,64 @@ impl ModuleTranslation<'_> {});}- struct InitMemoryAtCompileTime<'a> {- module: &'a Module,- info: &'a mut PrimaryMap<MemoryIndex, Memory>,- idx: usize,- }- impl InitMemory for InitMemoryAtCompileTime<'_> {- fn memory_size_in_bytes(- &mut self,- memory_index: MemoryIndex,- ) -> Result<u64, SizeOverflow> {- self.module.memories[memory_index].minimum_byte_size()+ for initializer in segments.iter() {+ let &MemoryInitializer {+ memory_index,+ ref offset,+ ref data,+ } = initializer;++ // Currently `Static` only applies to locally-defined memories,+ // so if a data segment references an imported memory then+ // transitioning to a `Static` memory initializer is not+ // possible.+ if self.module.defined_memory_index(memory_index).is_none() {+ return;}- fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option<u64> {- match (expr.ops(), self.module.memories[memory_index].idx_type) {- (&[ConstOp::I32Const(offset)], IndexType::I32) => {- Some(offset.cast_unsigned().into())+ // First up determine the start/end range and verify that they're+ // in-bounds for the initial size of the memory at `memory_index`.+ // Note that this can bail if we don't have access to globals yet+ // (e.g. this is a task happening before instantiation at+ // compile-time).+ let start = match (offset.ops(), self.module.memories[memory_index].idx_type) {+ (&[ConstOp::I32Const(offset)], IndexType::I32) => offset.cast_unsigned().into(),+ (&[ConstOp::I64Const(offset)], IndexType::I64) => offset.cast_unsigned(),+ _ => return,+ };+ let len = u64::try_from(data.len()).unwrap();+ let end = match start.checked_add(len) {+ Some(end) => end,+ None => return,+ };++ match self.module.memories[memory_index].minimum_byte_size() {+ Ok(max) => {+ if end > max {+ return;}- (&[ConstOp::I64Const(offset)], IndexType::I64) => Some(offset.cast_unsigned()),- _ => None,}++ // Note that computing the minimum can overflow if the page+ // size is the default 64KiB and the memory's minimum size in+ // pages is `1 << 48`, the maximum number of minimum pages for+ // 64-bit memories. We don't return `false` to signal an error+ // here and instead defer the error to runtime, when it will be+ // impossible to allocate that much memory anyways.+ Err(_) => return,}- fn write(&mut self, memory: MemoryIndex, init: &StaticMemoryInitializer) -> bool {- // Currently `Static` only applies to locally-defined memories,- // so if a data segment references an imported memory then- // transitioning to a `Static` memory initializer is not- // possible.- if self.module.defined_memory_index(memory).is_none() {- return false;- };- let info = &mut self.info[memory];- let data_len = u64::from(init.data.end - init.data.start);- if data_len > 0 {- info.data_size += data_len;- info.min_addr = info.min_addr.min(init.offset);- info.max_addr = info.max_addr.max(init.offset + data_len);- info.segments.push((self.idx, init.clone()));- }- self.idx += 1;- true+ // Skip empty in-bounds data segments.+ if data.is_empty() {+ continue;}- }- let ok = self- .module- .memory_initialization- .init_memory(&mut InitMemoryAtCompileTime {- idx: 0,- module: &self.module,- info: &mut info,- });- if !ok {- return;++ let info = &mut info[memory_index];+ let len64 = u64::try_from(data.len()).unwrap();+ info.data_size += len64;+ info.min_addr = info.min_addr.min(start);+ info.max_addr = info.max_addr.max(start + len64);+ info.segments.push((start, data));}// Validate that the memory information collected is indeed valid for@@ -1127,9 +1246,8 @@ impl ModuleTranslation<'_> {// Here's where we've now committed to changing to static memory. The// memory initialization image is built here from the page data and then// it's converted to a single initializer.- let data = mem::replace(&mut self.data, Vec::new());let mut map = TryPrimaryMap::with_capacity(info.len()).panic_on_oom();- let mut module_data_size = 0u32;+ let mut new_initializers = Vec::new();for (memory, info) in info.iter() {// Create the in-memory `image` which is the initialized contents of// this linear memory.@@ -1139,10 +1257,8 @@ impl ModuleTranslation<'_> {0};let mut image = Vec::with_capacity(extent);- for (idx, init) in info.segments.iter() {- let data = &data[*idx];- assert_eq!(data.len(), init.data.len());- let offset = usize::try_from(init.offset - info.min_addr).unwrap();+ for (offset, data) in info.segments.iter() {+ let offset = usize::try_from(*offset - info.min_addr).unwrap();if image.len() < offset {image.resize(offset, 0u8);image.extend_from_slice(data);@@ -1181,50 +1297,72 @@ impl ModuleTranslation<'_> {// the front and back with extra zeros as necessaryif offset % page_size != 0 {let zero_padding = offset % page_size;- self.data.push(vec![0; zero_padding as usize].into());+ image.splice(0..0, std::iter::repeat(0).take(zero_padding as usize));offset -= zero_padding;len += zero_padding;}- self.data.push(image.into());if len % page_size != 0 {let zero_padding = page_size - (len % page_size);- self.data.push(vec![0; zero_padding as usize].into());+ image.extend(std::iter::repeat(0).take(zero_padding as usize));len += zero_padding;}+ let runtime_index = if image.is_empty() {+ None+ } else {+ Some(self.runtime_data.push(image.into()))+ };// Offset/length should now always be page-aligned.assert!(offset % page_size == 0);assert!(len % page_size == 0);- // Create the `StaticMemoryInitializer` which describes this image,+ // Record the static memory initializer which describes this image,// only needed if the image is actually present and has a nonzero// length. The `offset` has been calculates above, originally// sourced from `info.min_addr`. The `data` field is the extent// within the final data segment we'll emit to an ELF image, which// is the concatenation of `self.data`, so here it's the size of// the section-so-far plus the current segment we're appending.- let len = u32::try_from(len).unwrap();- let init = if len > 0 {- Some(StaticMemoryInitializer {- offset,- data: module_data_size..module_data_size + len,- })- } else {- None- };- let idx = map.push(init).panic_on_oom();+ let idx = map.push(runtime_index.map(|i| (offset, i))).panic_on_oom();assert_eq!(idx, memory);- module_data_size += len;+ if let Some(runtime_index) = runtime_index {+ new_initializers.push((idx, MemorySegmentOffset::Static(offset), runtime_index));+ }}self.data_align = Some(page_size);self.module.memory_initialization = MemoryInitialization::Static { map };+ self.memory_init = MemoryInit::Processed(new_initializers);+ }++ /// Finalizes the initialization of tables.+ ///+ /// This is invoked after translation and notably uses+ /// `Self::try_func_table_init` to attempt to optimize initialization of+ /// tables into static precomputed images.+ pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) {+ if tunables.table_lazy_init {+ self.try_func_table_init();+ }++ // If any table has a non-null initializers, or if there's any active+ // data segments, then a startup function is unconditionally required to+ // configure the table.+ if self+ .table_initialization+ .initial_values+ .iter()+ .any(|(_, v)| !matches!(v, TableInitialValue::Null))+ || !self.table_initialization.segments.is_empty()+ {+ self.require_startup_func(types);+ }}/// Attempts to convert the module's table initializers to/// FuncTable form where possible. This enables lazy table/// initialization later by providing a one-to-one map of initial/// table values, without having to parse all segments.- pub fn try_func_table_init(&mut self) {+ fn try_func_table_init(&mut self) {// This should be large enough to support very large Wasm// modules with huge funcref tables, but small enough to avoid// OOMs or DoS on truly sparse tables.@@ -1232,32 +1370,27 @@ impl ModuleTranslation<'_> {// First convert any element-initialized tables to images of just that// single function if the minimum size of the table allows doing so.- for ((_, init), (_, table)) in self- .module- .table_initialization- .initial_values- .iter_mut()- .zip(- self.module- .tables- .iter()- .skip(self.module.num_imported_tables),- )- {+ for ((i, init), (_, table)) in self.table_initialization.initial_values.iter_mut().zip(+ self.module+ .tables+ .iter()+ .skip(self.module.num_imported_tables),+ ) {let table_size = table.limits.min;if table_size > MAX_FUNC_TABLE_SIZE {continue;}if let TableInitialValue::Expr(expr) = init {if let [ConstOp::RefFunc(f)] = expr.ops() {- *init = TableInitialValue::Null {- precomputed: try_vec![*f; table_size as usize].panic_on_oom(),- };+ assert!(self.module.table_initialization[i].is_empty());+ self.module.table_initialization[i] =+ try_vec![*f; table_size as usize].panic_on_oom();+ *init = TableInitialValue::Null;}}}- let mut segments = mem::take(&mut self.module.table_initialization.segments)+ let mut segments = mem::take(&mut self.table_initialization.segments).into_iter().peekable();@@ -1326,18 +1459,18 @@ impl ModuleTranslation<'_> {TableSegmentElements::Expressions { .. } => break,};- let precomputed =- match &mut self.module.table_initialization.initial_values[defined_index] {- TableInitialValue::Null { precomputed } => precomputed,+ match &self.table_initialization.initial_values[defined_index] {+ TableInitialValue::Null => {}- // If this table is still listed as an initial value here- // then that means the initial size of the table doesn't- // support a precomputed function list, so skip this.- // Technically this won't trap so it's possible to process- // further initializers, but that's left as a future- // optimization.- TableInitialValue::Expr(_) => break,- };+ // If this table is still listed as an initial value here+ // then that means the initial size of the table doesn't+ // support a precomputed function list, so skip this.+ // Technically this won't trap so it's possible to process+ // further initializers, but that's left as a future+ // optimization.+ TableInitialValue::Expr(_) => break,+ }+ let precomputed = &mut self.module.table_initialization[defined_index];// At this point we're committing to pre-initializing the table// with the `segment` that's being iterated over. This segment is@@ -1355,6 +1488,27 @@ impl ModuleTranslation<'_> {// advance the iterator to see the next segmentlet _ = segments.next();}- self.module.table_initialization.segments = segments.try_collect().panic_on_oom();+ self.table_initialization.segments = segments.try_collect().panic_on_oom();+ }++ /// Helper function to ratchet the `startup` function for this module as+ /// `Always`.+ fn require_startup_func(&mut self, types: &mut ModuleTypesBuilder) {+ let ty = match self.module.startup {+ ModuleStartup::None => types.startup_func_type().into(),+ ModuleStartup::Always(_) => return,+ ModuleStartup::IfMemoriesNeedInit(ty) => ty,+ };+ self.module.startup = ModuleStartup::Always(ty);+ }++ /// Helper function to ratchet the `startup` function for this module as+ /// `IfMemoriesNeedInit`.+ fn require_startup_func_if_memories_need_init(&mut self, types: &mut ModuleTypesBuilder) {+ let ty = match self.module.startup {+ ModuleStartup::None => types.startup_func_type().into(),+ ModuleStartup::Always(_) | ModuleStartup::IfMemoriesNeedInit(_) => return,+ };+ self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty);}}diff --git a/crates/environ/src/compile/module_types.rs b/crates/environ/src/compile/module_types.rsindex a82843dc9a..082b3cab23 100644--- a/crates/environ/src/compile/module_types.rs+++ b/crates/environ/src/compile/module_types.rs@@ -2,7 +2,7 @@ use crate::{EngineOrModuleTypeIndex, EntityRef, ModuleInternedRecGroupIndex, ModuleInternedTypeIndex,ModuleTypes, PanicOnOom as _, TypeConvert, TypeIndex, WasmArrayType, WasmCompositeInnerType,WasmCompositeType, WasmExnType, WasmFieldType, WasmFuncType, WasmHeapType, WasmResult,- WasmStorageType, WasmStructType, WasmSubType,+ WasmStorageType, WasmStructType, WasmSubType, WasmValType,collections::{TryClone as _, TryCow},wasm_unsupported,};@@ -59,6 +59,9 @@ pub struct ModuleTypesBuilder {/// If we are in the middle of defining a recursion group, this is the/// metadata about the recursion group we started defining.defining_rec_group: Option<RecGroupStart>,++ /// Cache of the return value of [`Self::startup_func_type`].+ startup_func_type: Option<ModuleInternedTypeIndex>,}impl ModuleTypesBuilder {@@ -72,6 +75,7 @@ impl ModuleTypesBuilder {wasmparser_to_wasmtime: HashMap::default(),already_seen: HashMap::default(),defining_rec_group: None,+ startup_func_type: None,}}@@ -462,6 +466,45 @@ impl ModuleTypesBuilder {}Ok(composite_type.inner.unwrap_func())}++ /// Gets a type for the "module startup" function, or `[] -> []` in the wasm+ /// type system.+ pub fn startup_func_type(&mut self) -> ModuleInternedTypeIndex {+ *self.startup_func_type.get_or_insert_with(|| {+ let idx = self.types.push(WasmSubType {+ is_final: true,+ supertype: None,+ composite_type: WasmCompositeType {+ inner: WasmCompositeInnerType::Func(WasmFuncType::new([], []).panic_on_oom()),+ shared: false,+ },+ });+ let next = self.types.next_ty();+ self.types.push_rec_group(idx..next);+ idx+ })+ }++ /// Smaller helper method to find a `ModuleInternedTypeIndex` which+ /// corresponds to the `resource.drop` intrinsic in components, namely a+ /// core wasm function type which takes one `i32` argument and has no+ /// results.+ ///+ /// This is a bit of a hack right now as ideally this find operation+ /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself+ /// would be threaded through appropriately, but that's left for a future+ /// refactoring. Try not to lean too hard on this method though.+ pub fn find_resource_drop_signature(&self) -> Option<ModuleInternedTypeIndex> {+ self.wasm_types()+ .find(|(_, ty)| {+ ty.as_func().map_or(false, |sig| {+ sig.params().len() == 1+ && sig.results().len() == 0+ && sig.params()[0] == WasmValType::I32+ })+ })+ .map(|(i, _)| i)+ }}// Forward the indexing impl to the internal `ModuleTypes`diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rsindex 9e98fcac64..2bfc28a361 100644--- a/crates/environ/src/component/compiler.rs+++ b/crates/environ/src/component/compiler.rs@@ -10,7 +10,7 @@ pub trait ComponentCompiler: Send + Sync {/// Each trampoline is a member of the `Trampoline` enumeration and has a/// unique purpose and is translated differently. See the implementation of/// this trait for Cranelift for more information.- fn compile_trampoline(+ fn compile_component_trampoline(&self,component: &ComponentTranslation,types: &ComponentTypesBuilder,diff --git a/crates/environ/src/component/types_builder.rs b/crates/environ/src/component/types_builder.rsindex 3f8f7d9495..d6cd8ec85b 100644--- a/crates/environ/src/component/types_builder.rs+++ b/crates/environ/src/component/types_builder.rs@@ -2,8 +2,8 @@ use crate::component::*;use crate::error::{Result, bail};use crate::prelude::*;use crate::{- EngineOrModuleTypeIndex, EntityType, ModuleInternedTypeIndex, ModuleTypes, ModuleTypesBuilder,- PrimaryMap, TypeConvert, WasmHeapType, WasmValType,+ EngineOrModuleTypeIndex, EntityType, ModuleTypes, ModuleTypesBuilder, PrimaryMap, TypeConvert,+ WasmHeapType,};use cranelift_entity::EntityRef;use std::collections::HashMap;@@ -163,28 +163,6 @@ impl ComponentTypesBuilder {(self.component_types, ty)}- /// Smaller helper method to find a `ModuleInternedTypeIndex` which- /// corresponds to the `resource.drop` intrinsic in components, namely a- /// core wasm function type which takes one `i32` argument and has no- /// results.- ///- /// This is a bit of a hack right now as ideally this find operation- /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself- /// would be threaded through appropriately, but that's left for a future- /// refactoring. Try not to lean too hard on this method though.- pub fn find_resource_drop_signature(&self) -> Option<ModuleInternedTypeIndex> {- self.module_types- .wasm_types()- .find(|(_, ty)| {- ty.as_func().map_or(false, |sig| {- sig.params().len() == 1- && sig.results().len() == 0- && sig.params()[0] == WasmValType::I32- })- })- .map(|(i, _)| i)- }-/// Returns the underlying builder used to build up core wasm module types.////// Note that this is shared across all modules found within a component todiff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rsindex 8f0fd0e794..66bc7c189a 100644--- a/crates/environ/src/key.rs+++ b/crates/environ/src/key.rs@@ -46,6 +46,10 @@ pub enum FuncKeyKind {/// A Wasmtime unsafe intrinsic function.#[cfg(feature = "component-model")]UnsafeIntrinsic = FuncKey::new_kind(0b1000),++ /// Initialization function for a module, such as initializing "complicated"+ /// globals and passive element segments.+ ModuleStartup = FuncKey::new_kind(0b1001),}impl From<FuncKeyKind> for u32 {@@ -73,6 +77,7 @@ impl FuncKeyKind {Self::PatchableToBuiltinTrampoline}x if x == Self::PulleyHostCall.into() => Self::PulleyHostCall,+ x if x == Self::ModuleStartup.into() => Self::ModuleStartup,#[cfg(feature = "component-model")]x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline,@@ -125,7 +130,9 @@ impl FuncKeyNamespace {/// Panics when given invalid raw representations.pub fn from_raw(raw: u32) -> Self {match FuncKeyKind::from_raw(raw & FuncKey::KIND_MASK) {- FuncKeyKind::DefinedWasmFunction | FuncKeyKind::ArrayToWasmTrampoline => Self(raw),+ FuncKeyKind::DefinedWasmFunction+ | FuncKeyKind::ArrayToWasmTrampoline+ | FuncKeyKind::ModuleStartup => Self(raw),FuncKeyKind::WasmToArrayTrampoline| FuncKeyKind::WasmToBuiltinTrampoline| FuncKeyKind::PatchableToBuiltinTrampoline@@ -205,7 +212,6 @@ pub enum Abi {Patchable = 2,}-#[cfg(feature = "component-model")]impl Abi {fn from_raw(raw: u32) -> Self {match raw {@@ -254,6 +260,13 @@ pub enum FuncKey {/// A Wasmtime intrinsic function.#[cfg(feature = "component-model")]UnsafeIntrinsic(Abi, component::UnsafeIntrinsic),++ /// Initialization function for a module, such as initializing "complicated"+ /// globals and passive element segments.+ ///+ /// This function has the `Abi` specified and will initialize the module+ /// specified.+ ModuleStartup(Abi, StaticModuleIndex),}impl Ord for FuncKey {@@ -342,6 +355,13 @@ impl FuncKey {let index = intrinsic.index();(namespace, index)}++ FuncKey::ModuleStartup(abi, module) => {+ assert_eq!(module.as_u32() & Self::KIND_MASK, 0);+ let namespace = FuncKeyKind::ModuleStartup.into_raw() | module.as_u32();+ let index = abi.into_raw();+ (namespace, index)+ }};(FuncKeyNamespace(namespace), FuncKeyIndex(index))}@@ -376,6 +396,7 @@ impl FuncKey {FuncKey::ResourceDropTrampoline => Abi::Wasm,#[cfg(feature = "component-model")]FuncKey::UnsafeIntrinsic(abi, _) => abi,+ FuncKey::ModuleStartup(abi, _) => abi,}}@@ -462,6 +483,12 @@ impl FuncKey {let intrinsic = component::UnsafeIntrinsic::from_u32(b);Self::UnsafeIntrinsic(abi, intrinsic)}++ FuncKeyKind::ModuleStartup => {+ let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK);+ let abi = Abi::from_raw(b);+ Self::ModuleStartup(abi, module)+ }}}@@ -556,7 +583,9 @@ impl FuncKey {/// looking up the StoreCode (or having access to the Store).pub fn is_store_invariant(&self) -> bool {match self {- Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false,+ Self::DefinedWasmFunction(..)+ | Self::ArrayToWasmTrampoline(..)+ | Self::ModuleStartup(..) => false,Self::WasmToArrayTrampoline(..)| Self::WasmToBuiltinTrampoline(..)| Self::PatchableToBuiltinTrampoline(..)diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rsindex 1983b9729b..45e228449b 100644--- a/crates/environ/src/module.rs+++ b/crates/environ/src/module.rs@@ -7,8 +7,8 @@ use cranelift_entity::{EntityRef, packed_option::ReservedValue};use serde_derive::{Deserialize, Serialize};/// A WebAssembly linear memory initializer.-#[derive(Clone, Debug, Serialize, Deserialize)]-pub struct MemoryInitializer {+#[derive(Clone, Debug)]+pub struct MemoryInitializer<'a> {/// The index of a linear memory to initialize.pub memory_index: MemoryIndex,/// The base offset to start this segment at.@@ -17,19 +17,7 @@ pub struct MemoryInitializer {////// This range indexes into a separately stored data section which will be/// provided with the compiled module's code as well.- pub data: Range<u32>,-}--/// Similar to the above `MemoryInitializer` but only used when memory-/// initializers are statically known to be valid.-#[derive(Clone, Debug, Serialize, Deserialize)]-pub struct StaticMemoryInitializer {- /// The 64-bit offset, in bytes, of where this initializer starts.- pub offset: u64,-- /// The range of data to write at `offset`, where these indices are indexes- /// into the compiled wasm module's data section.- pub data: Range<u32>,+ pub data: &'a [u8],}/// The type of WebAssembly linear memory initialization to use for a module.@@ -47,7 +35,7 @@ pub enum MemoryInitialization {/// data segments when the module is instantiated.////// This is the default memory initialization type.- Segmented(TryVec<MemoryInitializer>),+ Segmented,/// Memory initialization is statically known and involves a single `memcpy`/// or otherwise simply making the defined data visible.@@ -81,13 +69,13 @@ pub enum MemoryInitialization {////// The offset, range base, and range end are all guaranteed to be page/// aligned to the page size passed in to `try_static_init`.- map: TryPrimaryMap<MemoryIndex, Option<StaticMemoryInitializer>>,+ map: TryPrimaryMap<MemoryIndex, Option<(u64, RuntimeDataIndex)>>,},}impl Default for MemoryInitialization {fn default() -> Self {- Self::Segmented(TryVec::new())+ Self::Segmented}}@@ -96,121 +84,10 @@ impl MemoryInitialization {/// `MemoryInitialization::Segmented`.pub fn is_segmented(&self) -> bool {match self {- MemoryInitialization::Segmented(_) => true,+ MemoryInitialization::Segmented => true,_ => false,}}-- /// Performs the memory initialization steps for this set of initializers.- ///- /// This will perform wasm initialization in compliance with the wasm spec- /// and how data segments are processed. This doesn't need to necessarily- /// only be called as part of initialization, however, as it's structured to- /// allow learning about memory ahead-of-time at compile time possibly.- ///- /// This function will return true if all memory initializers are processed- /// successfully. If any initializer hits an error or, for example, a- /// global value is needed but `None` is returned, then false will be- /// returned. At compile-time this typically means that the "error" in- /// question needs to be deferred to runtime, and at runtime this means- /// that an invalid initializer has been found and a trap should be- /// generated.- pub fn init_memory(&self, state: &mut dyn InitMemory) -> bool {- let initializers = match self {- // Fall through below to the segmented memory one-by-one- // initialization.- MemoryInitialization::Segmented(list) => list,-- // If previously switched to static initialization then pass through- // all those parameters here to the `write` callback.- //- // Note that existence of `Static` already guarantees that all- // indices are in-bounds.- MemoryInitialization::Static { map } => {- for (index, init) in map {- if let Some(init) = init {- let result = state.write(index, init);- if !result {- return result;- }- }- }- return true;- }- };-- for initializer in initializers {- let &MemoryInitializer {- memory_index,- ref offset,- ref data,- } = initializer;-- // First up determine the start/end range and verify that they're- // in-bounds for the initial size of the memory at `memory_index`.- // Note that this can bail if we don't have access to globals yet- // (e.g. this is a task happening before instantiation at- // compile-time).- let start = match state.eval_offset(memory_index, offset) {- Some(start) => start,- None => return false,- };- let len = u64::try_from(data.len()).unwrap();- let end = match start.checked_add(len) {- Some(end) => end,- None => return false,- };-- match state.memory_size_in_bytes(memory_index) {- Ok(max) => {- if end > max {- return false;- }- }-- // Note that computing the minimum can overflow if the page size- // is the default 64KiB and the memory's minimum size in pages- // is `1 << 48`, the maximum number of minimum pages for 64-bit- // memories. We don't return `false` to signal an error here and- // instead defer the error to runtime, when it will be- // impossible to allocate that much memory anyways.- Err(_) => {}- }-- // The limits of the data segment have been validated at this point- // so the `write` callback is called with the range of data being- // written. Any erroneous result is propagated upwards.- let init = StaticMemoryInitializer {- offset: start,- data: data.clone(),- };- let result = state.write(memory_index, &init);- if !result {- return result;- }- }-- return true;- }-}--/// The various callbacks provided here are used to drive the smaller bits of-/// memory initialization.-pub trait InitMemory {- /// Returns the size, in bytes, of the memory specified. For compile-time- /// purposes this would be the memory type's minimum size.- fn memory_size_in_bytes(&mut self, memory_index: MemoryIndex) -> Result<u64, SizeOverflow>;-- /// Returns the value of the constant expression, as a `u64`. Note that- /// this may involve zero-extending a 32-bit global to a 64-bit number. May- /// return `None` to indicate that the expression involves a value which is- /// not available yet.- fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option<u64>;-- /// A callback used to actually write data. This indicates that the- /// specified memory must receive the specified range of data at the- /// specified offset. This can return false on failure.- fn write(&mut self, memory_index: MemoryIndex, init: &StaticMemoryInitializer) -> bool;}/// Table initialization data for all tables in the module.@@ -240,16 +117,7 @@ pub struct TableInitialization {pub enum TableInitialValue {/// Initialize each table element to null, optionally setting some elements/// to non-null given the precomputed image.- Null {- /// A precomputed image of table initializers for this table.- ///- /// This image is constructed during `try_func_table_init` and- /// null-initialized elements are represented with- /// `FuncIndex::reserved_value()`. Note that this image is empty by- /// default and may not encompass the entire span of the table in which- /// case the elements are initialized to null.- precomputed: TryVec<FuncIndex>,- },+ Null,/// An arbitrary const expression.Expr(ConstExpr),}@@ -285,6 +153,14 @@ pub enum TableSegmentElements {}impl TableSegmentElements {+ /// Returns the type of this segment.+ pub fn ty(&self) -> WasmRefType {+ match self {+ Self::Functions(_) => WasmRefType::FUNCREF,+ Self::Expressions { ty, .. } => *ty,+ }+ }+/// Returns the number of elements in this segment.pub fn len(&self) -> u64 {match self {@@ -313,20 +189,38 @@ pub struct Module {/// Exported entities.pub exports: TryIndexMap<Atom, EntityIndex>,- /// The module "start" function, if present.- pub start_func: Option<FuncIndex>,+ /// Whether or not this module has a start function,+ pub startup: ModuleStartup,- /// WebAssembly table initialization data, per table.- pub table_initialization: TableInitialization,+ /// Precompute per-table static images, if applicable.+ ///+ /// This map tracks, for each defined table in this module, the initial+ /// precomputed contents of the table. This is only applicable for funcref+ /// tables and the `TryVec` here uses `FuncIndex::reserved_value()` for null+ /// entries. This structure is filled in if table initialization is detected+ /// to be infallible as part of [`ModuleTranslation::finalize_table_init`].+ pub table_initialization: TryPrimaryMap<DefinedTableIndex, TryVec<FuncIndex>>,/// WebAssembly linear memory initializer.+ ///+ /// This will track how memory is initialized, either exclusively via+ /// segments or if some memories can be initialized with static images. This+ /// is computed during [`ModuleTranslation::finalize_memory_init`].pub memory_initialization: MemoryInitialization,/// WebAssembly passive elements.- pub passive_elements: TryPrimaryMap<PassiveElemIndex, TableSegmentElements>,+ ///+ /// This is a map of all passive element segments to their type and the+ /// initial size of the segment. Note that the contents of the segment are+ /// initialized by compiled code.+ pub passive_elements: TryPrimaryMap<PassiveElemIndex, (WasmRefType, u64)>,- /// Where passive data segments are located in the module's image.- pub passive_data: TryPrimaryMap<PassiveDataIndex, Range<u32>>,+ /// Where runtime data segments are located in the module's image.+ ///+ /// Note that this does not directly correspond to either active or passive+ /// data segments. Those are massaged during+ /// [`ModuleTranslation::finalize_memory_init`] into the form used here.+ pub runtime_data: TryPrimaryMap<RuntimeDataIndex, Range<u32>>,/// Types declared in the wasm module.pub types: TryPrimaryMap<TypeIndex, EngineOrModuleTypeIndex>,@@ -368,8 +262,18 @@ pub struct Module {/// WebAssembly global variables.pub globals: TryPrimaryMap<GlobalIndex, Global>,- /// WebAssembly global initializers for locally-defined globals.- pub global_initializers: TryPrimaryMap<DefinedGlobalIndex, ConstExpr>,+ /// "Simple" WebAssembly global initializers for locally-defined globals.+ ///+ /// This map does not track initialization of all globals in this module,+ /// but only those considered "simple" which can be easily evaluated at+ /// compile-time. For example an initialization expression of `i32.const N`+ /// is considered simple. These globals are manually initialized in the+ /// host.+ ///+ /// This is all in contrast to [`ModuleTranslation::global_initializers`]+ /// which is processed in compiled code and initialized after the instance+ /// has been created.+ pub global_initializers: TryVec<(DefinedGlobalIndex, GlobalConstValue)>,/// WebAssembly exception and control tags.pub tags: TryPrimaryMap<TagIndex, Tag>,@@ -400,11 +304,11 @@ impl Module {name: Default::default(),initializers: Default::default(),exports: Default::default(),- start_func: Default::default(),+ startup: ModuleStartup::None,table_initialization: Default::default(),memory_initialization: Default::default(),passive_elements: Default::default(),- passive_data: Default::default(),+ runtime_data: Default::default(),types: Default::default(),num_imported_funcs: Default::default(),num_imported_tables: Default::default(),@@ -690,11 +594,11 @@ impl TypeTrace for Module {name: _,initializers: _,exports: _,- start_func: _,+ startup,table_initialization: _,memory_initialization: _,passive_elements: _,- passive_data: _,+ runtime_data: _,types,num_imported_funcs: _,num_imported_tables: _,@@ -726,6 +630,7 @@ impl TypeTrace for Module {for t in tags.values() {t.trace(func)?;}+ startup.trace(func)?;Ok(())}@@ -741,11 +646,11 @@ impl TypeTrace for Module {name: _,initializers: _,exports: _,- start_func: _,+ startup,table_initialization: _,memory_initialization: _,passive_elements: _,- passive_data: _,+ runtime_data: _,types,num_imported_funcs: _,num_imported_tables: _,@@ -777,10 +682,33 @@ impl TypeTrace for Module {for t in tags.values_mut() {t.trace_mut(func)?;}+ startup.trace_mut(func)?;Ok(())}}+impl TypeTrace for ModuleStartup {+ fn trace<F, E>(&self, func: &mut F) -> Result<(), E>+ where+ F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>,+ {+ match self {+ ModuleStartup::None => Ok(()),+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(*t),+ }+ }++ fn trace_mut<F, E>(&mut self, func: &mut F) -> Result<(), E>+ where+ F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>,+ {+ match self {+ ModuleStartup::None => Ok(()),+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(t),+ }+ }+}+/// Type information about functions in a wasm module.#[derive(Debug, Serialize, Deserialize)]pub struct FunctionType {@@ -821,3 +749,34 @@ impl FunctionType {#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]pub struct FuncRefIndex(u32);cranelift_entity::entity_impl!(FuncRefIndex);++/// Different means of startup for a wasm module.+#[derive(Debug, Serialize, Deserialize)]+pub enum ModuleStartup {+ /// No startup is necessary.+ None,++ /// Startup is always required, for example to apply active table segments.+ ///+ /// The type of the startup function, of wasm signature `[] -> []`, is+ /// provided here.+ Always(EngineOrModuleTypeIndex),++ /// Startup is only required if some linear memory within this module, at+ /// runtime, says `needs_init() == true`.+ ///+ /// This special mode of startup indicates that the startup function has no+ /// purpose other than to initialize the initial contents of+ /// `MemoryInitialization::Static` linear memories. In this situation if all+ /// memories say `needs_init() == false` then the startup function won't+ /// actually do anything meaning that it can be optimized slightly by+ /// skipping it entirely.+ IfMemoriesNeedInit(EngineOrModuleTypeIndex),+}++impl ModuleStartup {+ /// Returns if this is `ModuleStartup::None`.+ pub fn is_none(&self) -> bool {+ matches!(self, Self::None)+ }+}diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rsindex 0f78e765ca..c3173b3819 100644--- a/crates/environ/src/module_artifacts.rs+++ b/crates/environ/src/module_artifacts.rs@@ -552,7 +552,8 @@ impl CompiledFunctionsTable {FuncKeyKind::ArrayToWasmTrampoline| FuncKeyKind::WasmToBuiltinTrampoline- | FuncKeyKind::PatchableToBuiltinTrampoline => false,+ | FuncKeyKind::PatchableToBuiltinTrampoline+ | FuncKeyKind::ModuleStartup => false,#[cfg(feature = "component-model")]FuncKeyKind::ComponentTrampoline@@ -569,7 +570,8 @@ impl CompiledFunctionsTable {| FuncKeyKind::WasmToArrayTrampoline| FuncKeyKind::WasmToBuiltinTrampoline| FuncKeyKind::PatchableToBuiltinTrampoline- | FuncKeyKind::PulleyHostCall => false,+ | FuncKeyKind::PulleyHostCall+ | FuncKeyKind::ModuleStartup => false,#[cfg(feature = "component-model")]FuncKeyKind::ComponentTrampoline| FuncKeyKind::ResourceDropTrampolinediff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rsindex 0ee06f9378..9841f45ebc 100644--- a/crates/environ/src/types.rs+++ b/crates/environ/src/types.rs@@ -1699,13 +1699,19 @@ impl Default for VMSharedTypeIndex {pub struct DataIndex(u32);entity_impl_with_try_clone!(DataIndex);-/// Index type of a passive data segment inside the WebAssembly module.+/// Index into data segments needed at runtime by a module.///-/// Not a spec-level concept, just used to get dense index spaces for passive-/// data segments inside of Wasmtime.+/// This does not directly correspond to either active or passive data segments+/// in the wasm spec. Instead this is a concept purely for Wasmtime and+/// organizing memory initialization within the+/// `ModuleTranslation::finalize_memory_init` function, for example.+///+/// Passive data segments at runtime all have a corresponding+/// `RuntimeDataIndex`, but active data segments maybe coalesced or mutated if+/// they're statically evaluated.#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]-pub struct PassiveDataIndex(u32);-entity_impl_with_try_clone!(PassiveDataIndex);+pub struct RuntimeDataIndex(u32);+entity_impl_with_try_clone!(RuntimeDataIndex);/// Index type of an element segment inside the WebAssembly module.#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]@@ -2007,7 +2013,7 @@ impl ConstExpr {/// A global's constant value, known at compile time.#[expect(missing_docs, reason = "self-describing variants")]-#[derive(Clone, Copy)]+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]pub enum GlobalConstValue {I32(i32),I64(i64),@@ -2027,7 +2033,7 @@ pub enum ConstOp {V128Const(u128),GlobalGet(GlobalIndex),RefI31,- RefNull(WasmHeapTopType),+ RefNull(WasmHeapType),RefFunc(FuncIndex),I32Add,I32Sub,@@ -2069,7 +2075,7 @@ impl ConstOp {O::F32Const { value } => Self::F32Const(value.bits()),O::F64Const { value } => Self::F64Const(value.bits()),O::V128Const { value } => Self::V128Const(u128::from_le_bytes(*value.bytes())),- O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?.top()),+ O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?),O::RefFunc { function_index } => Self::RefFunc(FuncIndex::from_u32(function_index)),O::GlobalGet { global_index } => Self::GlobalGet(GlobalIndex::from_u32(global_index)),O::RefI31 => Self::RefI31,diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 74c50dd201..3437c14674 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -31,13 +31,14 @@// globals: [VMGlobalDefinition; module.num_defined_globals],// tags: [VMTagDefinition; module.num_defined_tags],// func_refs: [VMFuncRef; module.num_escaped_funcs],-// passive_data_bases: [*const u8; module.num_passive_data],-// passive_data_lengths: [u32; module.num_passive_data],+// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0],+// runtime_data_bases: [*const u8; module.num_runtime_data],+// runtime_data_lengths: [u32; module.num_runtime_data],// }use crate::{DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,- FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, PassiveDataIndex, TableIndex,+ FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,TagIndex,};use cranelift_entity::packed_option::ReservedValue;@@ -90,8 +91,10 @@ pub struct VMOffsets<P> {/// The number of escaped functions in the module, the size of the func_refs/// array.pub num_escaped_funcs: u32,- /// The number of passive data segments in the module.- pub num_passive_data: u32,+ /// The number of runtime data segments in the module.+ pub num_runtime_data: u32,+ /// Whether or not the module has a start function.+ pub has_startup_func: bool,// precalculated offsets of various member fieldsimported_functions: u32,@@ -105,8 +108,9 @@ pub struct VMOffsets<P> {defined_globals: u32,defined_tags: u32,defined_func_refs: u32,- passive_data_bases: u32,- passive_data_lengths: u32,+ startup_func_ref: u32,+ runtime_data_bases: u32,+ runtime_data_lengths: u32,size: u32,}@@ -607,8 +611,10 @@ pub struct VMOffsetsFields<P> {/// The number of escaped functions in the module, the size of the function/// references array.pub num_escaped_funcs: u32,- /// The number of passive data segments in the module.- pub num_passive_data: u32,+ /// The number of runtime data segments in the module.+ pub num_runtime_data: u32,+ /// Whether or not the module has a start function.+ pub has_startup_func: bool,}impl<P: PtrSize> VMOffsets<P> {@@ -635,7 +641,8 @@ impl<P: PtrSize> VMOffsets<P> {num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals),num_defined_tags: cast_to_u32(module.tags.len() - module.num_imported_tags),num_escaped_funcs: cast_to_u32(module.num_escaped_funcs),- num_passive_data: cast_to_u32(module.passive_data.len()),+ num_runtime_data: cast_to_u32(module.runtime_data.len()),+ has_startup_func: !module.startup.is_none(),})}@@ -667,7 +674,8 @@ impl<P: PtrSize> VMOffsets<P> {num_defined_tags: _,num_owned_memories: _,num_escaped_funcs: _,- num_passive_data: _,+ num_runtime_data: _,+ has_startup_func: _,// used as the initial size belowsize,@@ -696,8 +704,9 @@ impl<P: PtrSize> VMOffsets<P> {}calculate_sizes! {- passive_data_lengths: "passive data lengths",- passive_data_bases: "passive data base pointers",+ runtime_data_lengths: "runtime data lengths",+ runtime_data_bases: "runtime data base pointers",+ startup_func_ref: "startup funcref",defined_func_refs: "module functions",defined_tags: "defined tags",defined_globals: "defined globals",@@ -728,7 +737,8 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {num_defined_globals: fields.num_defined_globals,num_defined_tags: fields.num_defined_tags,num_escaped_funcs: fields.num_escaped_funcs,- num_passive_data: fields.num_passive_data,+ num_runtime_data: fields.num_runtime_data,+ has_startup_func: fields.has_startup_func,imported_functions: 0,imported_tables: 0,imported_memories: 0,@@ -740,8 +750,9 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {defined_globals: 0,defined_tags: 0,defined_func_refs: 0,- passive_data_bases: 0,- passive_data_lengths: 0,+ startup_func_ref: 0,+ runtime_data_bases: 0,+ runtime_data_lengths: 0,size: 0,};@@ -800,8 +811,13 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {ret.num_escaped_funcs,ret.ptr.size_of_vm_func_ref(),),- size(passive_data_bases) = cmul(ret.num_passive_data, ret.ptr.size()),- size(passive_data_lengths) = cmul(ret.num_passive_data, 4),+ size(startup_func_ref) = if ret.has_startup_func {+ ret.ptr.size_of_vm_func_ref()+ } else {+ 0+ },+ size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()),+ size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4),}ret.size = next_field_offset;@@ -1050,16 +1066,16 @@ impl<P: PtrSize> VMOffsets<P> {self.defined_func_refs}- /// The offset of the `passive_data_bases` array.+ /// The offset of the `runtime_data_bases` array.#[inline]- pub fn vmctx_passive_data_bases_begin(&self) -> u32 {- self.passive_data_bases+ pub fn vmctx_runtime_data_bases_begin(&self) -> u32 {+ self.runtime_data_bases}- /// The offset of the `passive_data_lengths` array.+ /// The offset of the `runtime_data_lengths` array.#[inline]- pub fn vmctx_passive_data_lengths_begin(&self) -> u32 {- self.passive_data_lengths+ pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 {+ self.runtime_data_lengths}/// Return the size of the `VMContext` allocation.@@ -1154,20 +1170,29 @@ impl<P: PtrSize> VMOffsets<P> {self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref())}- /// Return the offset to the base of the passive data segment at `index`.+ /// Returns the offset to the `VMFuncRef` for the module startup function.+ ///+ /// Panics if this module does not have a startup function.+ #[inline]+ pub fn vmctx_startup_func_ref(&self) -> u32 {+ assert!(self.has_startup_func);+ self.startup_func_ref+ }++ /// Return the offset to the base of the runtime data segment at `index`.#[inline]- pub fn vmctx_passive_data_base(&self, index: PassiveDataIndex) -> u32 {+ pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 {assert!(!index.is_reserved_value());- assert!(index.as_u32() < self.num_passive_data);- self.vmctx_passive_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())+ assert!(index.as_u32() < self.num_runtime_data);+ self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())}- /// Return the offset to the length of the passive data segment at `index`.+ /// Return the offset to the length of the runtime data segment at `index`.#[inline]- pub fn vmctx_passive_data_length(&self, index: PassiveDataIndex) -> u32 {+ pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 {assert!(!index.is_reserved_value());- assert!(index.as_u32() < self.num_passive_data);- self.vmctx_passive_data_lengths_begin() + index.as_u32() * 4+ assert!(index.as_u32() < self.num_runtime_data);+ self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4}/// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`.diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 3696f325ab..2994644c5d 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -85,10 +85,11 @@ pub(crate) fn build_module_artifacts<T: FinishedObject>().translate(parser, wasm).context("failed to parse WebAssembly module")?;+ prepare_translation(engine, compiler, &mut translation, &mut types);let functions = mem::take(&mut translation.function_body_inputs);let compile_inputs = CompileInputs::for_module(&types, &translation, functions);- let unlinked_compile_outputs = compile_inputs.compile(engine)?;+ let unlinked_compile_outputs = compile_inputs.compile(engine, &types)?;let PreLinkOutput {needs_gc_heap,compiled_funcs,@@ -165,6 +166,15 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(.translate(binary).context("failed to parse WebAssembly module")?;+ for (_, translation) in module_translations.iter_mut() {+ prepare_translation(+ engine,+ compiler,+ translation,+ types.module_types_builder_mut(),+ );+ }+let compile_inputs = CompileInputs::for_component(engine,&types,@@ -174,7 +184,7 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>((i, &*translation, functions)}),);- let unlinked_compile_outputs = compile_inputs.compile(&engine)?;+ let unlinked_compile_outputs = compile_inputs.compile(&engine, types.module_types_builder())?;let PreLinkOutput {needs_gc_heap,@@ -230,6 +240,26 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(Ok((result, Some(artifacts)))}+fn prepare_translation(+ engine: &Engine,+ compiler: &dyn Compiler,+ translation: &mut ModuleTranslation<'_>,+ types: &mut ModuleTypesBuilder,+) {+ // If configured attempt to use static memory initialization+ // which can either at runtime be implemented as a single memcpy+ // to initialize memory or otherwise enabling+ // virtual-memory-tricks such as mmap'ing from a file to get+ // copy-on-write.+ let align = compiler.page_size_align();+ let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;+ translation.finalize_memory_init(engine.tunables(), align, max_always_allowed, types);++ // Attempt to convert table initializer segments to FuncTable+ // representation where possible, to enable table lazy init.+ translation.finalize_table_init(engine.tunables(), types);+}+type CompileInput<'a> = Box<dyn FnOnce(&dyn Compiler) -> Result<CompileOutput<'a>> + Send + 'a>;struct CompileOutput<'a> {@@ -357,7 +387,7 @@ impl<'a> CompileInputs<'a> {);let function = compiler.component_compiler()- .compile_trampoline(component, types, key, abi, tunables, &symbol)+ .compile_component_trampoline(component, types, key, abi, tunables, &symbol).with_context(|| format!("failed to compile {symbol}"))?;Ok(CompileOutput {key,@@ -378,12 +408,16 @@ impl<'a> CompileInputs<'a> {// resources from the embedder and otherwise won't be explicitly// requested through initializers above or such.if component.component.num_resources > 0 {- if let Some(sig) = types.find_resource_drop_signature() {+ if types+ .module_types_builder()+ .find_resource_drop_signature()+ .is_some()+ {ret.push_input(move |compiler| {let key = FuncKey::ResourceDropTrampoline;let symbol = "resource_drop_trampoline".to_string();let function = compiler- .compile_wasm_to_array_trampoline(types[sig].unwrap_func(), key, &symbol)+ .compile_trampoline(None, key, types.module_types_builder(), &symbol).with_context(|| format!("failed to compile `{symbol}`"))?;Ok(CompileOutput {key,@@ -493,7 +527,7 @@ impl<'a> CompileInputs<'a> {func_index.as_u32());let function = compiler- .compile_array_to_wasm_trampoline(translation, types, key, &symbol)+ .compile_trampoline(Some(translation), key, types, &symbol).with_context(|| format!("failed to compile: {symbol}"))?;Ok(CompileOutput {key,@@ -506,6 +540,26 @@ impl<'a> CompileInputs<'a> {});}}++ if !translation.module.startup.is_none() {+ for abi in [Abi::Wasm, Abi::Array] {+ self.push_input(move |compiler| {+ let key = FuncKey::ModuleStartup(abi, module);+ let symbol = format!("module_start[{}]::{abi:?}", module.as_u32());+ let function = compiler+ .compile_trampoline(Some(translation), key, types, &symbol)+ .with_context(|| format!("failed to compile: {symbol}"))?;+ Ok(CompileOutput {+ key,+ function,+ symbol,+ start_srcloc: FilePos::default(),+ translation: None,+ func_body: None,+ })+ });+ }+ }}let mut trampoline_types_seen = HashSet::new();@@ -514,7 +568,6 @@ impl<'a> CompileInputs<'a> {if !is_new {continue;}- let trampoline_func_ty = types[trampoline_type_index].unwrap_func();self.push_input(move |compiler| {let key = FuncKey::WasmToArrayTrampoline(trampoline_type_index);let symbol = format!(@@ -522,7 +575,7 @@ impl<'a> CompileInputs<'a> {trampoline_type_index.as_u32());let function = compiler- .compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol)+ .compile_trampoline(None, key, types, &symbol).with_context(|| format!("failed to compile: {symbol}"))?;Ok(CompileOutput {key,@@ -538,7 +591,11 @@ impl<'a> CompileInputs<'a> {/// Compile these `CompileInput`s (maybe in parallel) and return the/// resulting `UnlinkedCompileOutput`s.- fn compile(self, engine: &Engine) -> Result<UnlinkedCompileOutputs<'a>> {+ fn compile(+ self,+ engine: &Engine,+ types: &'a ModuleTypesBuilder,+ ) -> Result<UnlinkedCompileOutputs<'a>> {let compiler = engine.try_compiler()?;if self.inputs.len() > 0 && cfg!(miri) {@@ -597,7 +654,7 @@ the use case.// wasmtime-builtin functions are necessary. If so those need to be// collected and then those trampolines additionally need to be// compiled.- compile_required_builtins(engine, &mut raw_outputs)?;+ compile_required_builtins(engine, types, &mut raw_outputs)?;// Bucket the outputs by kind.let mut outputs: BTreeMap<FuncKey, CompileOutput> = BTreeMap::new();@@ -910,7 +967,8 @@ fn is_inlining_function(key: FuncKey) -> bool {FuncKey::ArrayToWasmTrampoline(..)| FuncKey::WasmToArrayTrampoline(..)| FuncKey::WasmToBuiltinTrampoline(..)- | FuncKey::PatchableToBuiltinTrampoline(..) => false,+ | FuncKey::PatchableToBuiltinTrampoline(..)+ | FuncKey::ModuleStartup(..) => false,#[cfg(feature = "component-model")]FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,@@ -934,7 +992,11 @@ fn inlining_functions<'a>(})}-fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {+fn compile_required_builtins<'a>(+ engine: &Engine,+ types: &'a ModuleTypesBuilder,+ raw_outputs: &mut Vec<CompileOutput<'a>>,+) -> Result<()> {let compiler = engine.try_compiler()?;let mut builtins = HashSet::new();let mut new_inputs: Vec<CompileInput<'_>> = Vec::new();@@ -951,7 +1013,7 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutpu_ => unreachable!(),};let mut function = compiler- .compile_wasm_to_builtin(key, &symbol)+ .compile_trampoline(None, key, types, &symbol).with_context(|| format!("failed to compile `{symbol}`"))?;if let Some(compiler) = compiler.inlining_compiler() {compiler.finish_compiling(&mut function, None, &symbol)?;@@ -1116,26 +1178,7 @@ impl FunctionIndices {let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);let modules = translations.into_iter()- .map(|(_, mut translation)| {- // If configured attempt to use static memory initialization- // which can either at runtime be implemented as a single memcpy- // to initialize memory or otherwise enabling- // virtual-memory-tricks such as mmap'ing from a file to get- // copy-on-write.- if engine.tunables().memory_init_cow {- let align = compiler.page_size_align();- let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;- translation.try_static_init(align, max_always_allowed);- }-- // Attempt to convert table initializer segments to FuncTable- // representation where possible, to enable table lazy init.- if engine.tunables().table_lazy_init {- translation.try_func_table_init();- }-- obj.append(translation)- })+ .map(|(_, translation)| obj.append(translation)).collect::<Result<PrimaryMap<_, _>>>()?;let artifacts = Artifacts {diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rsindex 3d6af317c3..1faf0ac073 100644--- a/crates/wasmtime/src/runtime/code.rs+++ b/crates/wasmtime/src/runtime/code.rs@@ -9,11 +9,9 @@ use alloc::boxed::Box;use alloc::sync::Arc;use core::ops::{Add, Range, Sub};use wasmtime_core::error::OutOfMemory;-use wasmtime_environ::DefinedFuncIndex;-use wasmtime_environ::ModuleTypes;-use wasmtime_environ::StaticModuleIndex;#[cfg(feature = "component-model")]use wasmtime_environ::component::ComponentTypes;+use wasmtime_environ::{FuncKey, ModuleTypes, StaticModuleIndex};macro_rules! define_pc_kind {($ty:ident) => {@@ -431,30 +429,10 @@ impl<'a> ModuleWithCode<'a> {}/// Returns the slice in the text section of the function that- /// `index` points to.+ /// `key` points to.#[inline]- pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {- let range = self- .module- .compiled_module()- .finished_function_range(def_func_index);+ pub fn function(&self, key: FuncKey) -> &[u8] {+ let range = self.module.compiled_module().function_range(key);&self.store_code.text()[range]}-- /// Get the array-to-Wasm trampoline for the function `index`- /// points to, as a slice of raw code that can be converted to a- /// callable function pointer.- ///- /// If the function `index` points to does not escape, then `None` is- /// returned.- ///- /// These trampolines are used for array callers (e.g. `Func::new`)- /// calling Wasm callees.- pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {- let range = self- .module- .compiled_module()- .array_to_wasm_trampoline_range(def_func_index)?;- Some(&self.store_code.text()[range])- }}diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex 489caf0a8e..7b4f66cee6 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -967,6 +967,7 @@ mod tests {use wasmtime_environ::MemoryInitialization;#[test]+ #[cfg_attr(miri, ignore)]fn cow_on_by_default() {let mut config = Config::new();config.wasm_component_model(true);diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rsindex d03ba277f1..eb2f392270 100644--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs@@ -7,7 +7,7 @@ use core::convert::Infallible;use core::fmt::{self, Debug};use core::hash::{Hash, Hasher};use core::marker;-use core::ops::{Deref, DerefMut};+use core::ops::Deref;mod sealed {use super::*;@@ -209,27 +209,3 @@ impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {match self.inner {}}}--pub(crate) struct OpaqueRootScope<S> {- store: S,-}--impl<S> Deref for OpaqueRootScope<S> {- type Target = S;-- fn deref(&self) -> &Self::Target {- &self.store- }-}--impl<S> DerefMut for OpaqueRootScope<S> {- fn deref_mut(&mut self) -> &mut Self::Target {- &mut self.store- }-}--impl<S> OpaqueRootScope<S> {- pub(crate) fn new(store: S) -> Self {- OpaqueRootScope { store }- }-}diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rsindex f2133cc275..1b4a539710 100644--- a/crates/wasmtime/src/runtime/instance.rs+++ b/crates/wasmtime/src/runtime/instance.rs@@ -14,7 +14,6 @@ use crate::{};use alloc::sync::Arc;use core::ptr::NonNull;-use wasmparser::WasmFeatures;use wasmtime_environ::{EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TagIndex, TypeTrace,};@@ -255,21 +254,26 @@ impl Instance {imports: Imports<'_>,asyncness: Asyncness,) -> Result<Instance> {- let (instance, start) = {+ let instance = {let (mut limiter, store) = store.0.resource_limiter_and_store_opaque();// SAFETY: the safety contract of `new_raw` is the same as this// function.- unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports, asyncness).await? }+ unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports).await? }};- if let Some(start) = start {++ // If this instance requires startup, which is a dynamic decision made+ // at this point in conjunction with analysis at compile time, the+ // instance gets started. Note that this isn't just the wasm start+ // function itself, but it's finalization of initialization of this+ // instance, for example for complicated global initialization+ // expressions.+ if instance.id.get_mut(store.0).needs_startup() {if asyncness == Asyncness::No {- instance.start_raw(store, start)?;+ instance.start_raw(store)?;} else {#[cfg(feature = "async")]{- store- .on_fiber(|store| instance.start_raw(store, start))- .await??;+ store.on_fiber(|store| instance.start_raw(store)).await??;}#[cfg(not(feature = "async"))]unreachable!();@@ -298,8 +302,7 @@ impl Instance {mut limiter: Option<&mut StoreResourceLimiter<'_>>,module: &Module,imports: Imports<'_>,- asyncness: Asyncness,- ) -> Result<(Instance, Option<FuncIndex>)> {+ ) -> Result<Instance> {if !Engine::same(store.engine(), module.engine()) {bail!("cross-`Engine` instantiation is not currently supported");}@@ -317,8 +320,6 @@ impl Instance {}}- let compiled_module = module.compiled_module();-// Register the module just before instantiation to ensure we keep the module// properly referenced while in use by the store.let (modules, engine, breakpoints) = store.modules_and_engine_and_breakpoints_mut();@@ -341,46 +342,12 @@ impl Instance {.await?};- // Additionally, before we start doing fallible instantiation, we- // do one more step which is to insert an `InstanceData`- // corresponding to this instance. This `InstanceData` can be used- // via `Caller::get_export` if our instance's state "leaks" into- // other instances, even if we don't return successfully from this- // function.- //- // We don't actually load all exports from the instance at this- // time, instead preferring to lazily load them as they're demanded.- // For module/instance exports, though, those aren't actually- // stored in the instance handle so we need to immediately handle- // those here.- let instance = Instance::from_wasmtime(id, store);-- // Now that we've recorded all information we need to about this- // instance within a `Store` we can start performing fallible- // initialization. Note that we still defer the `start` function to- // later since that may need to run asynchronously.- //- // If this returns an error (or if the start function traps) then- // any other initialization which may have succeeded which placed- // items from this instance into other instances should be ok when- // those items are loaded and run we'll have all the metadata to- // look at them.- let bulk_memory = store- .engine()- .features()- .contains(WasmFeatures::BULK_MEMORY);-- vm::initialize_instance(- store,- limiter,- id,- compiled_module.module(),- bulk_memory,- asyncness,- )- .await?;-- Ok((instance, compiled_module.module().start_func))+ // At this point the instance is created and stored within the store,+ // but it's also not quite usable just yet. Initialization hasn't+ // completed (e.g. active data/element segments) and the `start`+ // function additionally has not yet been invoked. That's the+ // responsibility of the caller to handle, however.+ Ok(Instance::from_wasmtime(id, store))}pub(crate) fn from_wasmtime(id: InstanceId, store: &mut StoreOpaque) -> Instance {@@ -389,7 +356,7 @@ impl Instance {}}- fn start_raw<T>(&self, store: &mut StoreContextMut<'_, T>, start: FuncIndex) -> Result<()> {+ fn start_raw<T>(&self, store: &mut StoreContextMut<'_, T>) -> Result<()> {// If a start function is present, invoke it. Make sure we use all the// trap-handling configuration in `store` as well.let store_id = store.0.id();@@ -399,7 +366,8 @@ impl Instance {let f = unsafe {instance.as_mut()- .get_exported_func(registry, store_id, start)+ .get_startup_func(registry, store_id)+ .expect("should have a startup function")};let caller_vmctx = instance.vmctx();unsafe {diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rsindex 188d005e1f..8640cdf43a 100644--- a/crates/wasmtime/src/runtime/instantiate.rs+++ b/crates/wasmtime/src/runtime/instantiate.rs@@ -11,9 +11,8 @@ use alloc::sync::Arc;use core::ops::Range;use core::str;use wasmtime_environ::{- CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,- FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex,- StaticModuleIndex,+ CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FuncKey,+ FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, StaticModuleIndex,};/// A compiled wasm module, ready to be instantiated.@@ -116,55 +115,32 @@ impl CompiledModule {pub fn finished_function_ranges(&self,) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, Range<usize>)> + '_ {- self.module- .defined_func_indices()- .map(|i| (i, self.finished_function_range(i)))+ self.module.defined_func_indices().map(|i| {+ let key = FuncKey::DefinedWasmFunction(self.module_index(), i);+ (i, self.function_range(key))+ })}- /// Returns the offset in the text section of the function that `index` points to.+ /// Returns the offset in the text section of the function that `key`+ /// points to.#[inline]- pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range<usize> {- let loc = self.func_loc(def_func_index);+ pub fn function_range(&self, key: FuncKey) -> Range<usize> {+ let loc = self.func_loc(key);let start = usize::try_from(loc.start).unwrap();let end = usize::try_from(loc.start + loc.length).unwrap();start..end}- /// Get the array-to-Wasm trampoline for the function `index`- /// points to, as a range in the text segment.- ///- /// If the function `index` points to does not escape, then `None` is- /// returned.- ///- /// These trampolines are used for array callers (e.g. `Func::new`)- /// calling Wasm callees.- pub fn array_to_wasm_trampoline_range(- &self,- def_func_index: DefinedFuncIndex,- ) -> Option<Range<usize>> {- assert!(def_func_index.index() < self.module.num_defined_funcs());- let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);- let loc = self.index.func_loc(key)?;- let start = usize::try_from(loc.start).unwrap();- let end = usize::try_from(loc.start + loc.length).unwrap();- Some(start..end)- }-/// Get the Wasm-to-array trampoline for the given signature, as a/// range in the text segment.////// These trampolines are used for filling in/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs/// that don't have access to a compiler when created.- pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {+ pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] {let key = FuncKey::WasmToArrayTrampoline(signature);- let loc = self.index.func_loc(key)?;- let start = usize::try_from(loc.start).unwrap();- let end = usize::try_from(loc.start + loc.length).unwrap();- Some(- self.engine_code- .raw_wasm_to_array_trampoline_data(start..end),- )+ let range = self.function_range(key);+ self.engine_code.raw_wasm_to_array_trampoline_data(range)}/// Lookups a defined function by a program counter value.@@ -190,9 +166,7 @@ impl CompiledModule {}/// Gets the function location information for a given function index.- pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc {- assert!(def_func_index.index() < self.module.num_defined_funcs());- let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);+ pub fn func_loc(&self, key: FuncKey) -> &FunctionLoc {self.index.func_loc(key).expect("defined function should be present")@@ -200,9 +174,7 @@ impl CompiledModule {/// Returns the original binary offset in the file that `index` was defined/// at.- pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos {- assert!(def_func_index.index() < self.module.num_defined_funcs());- let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);+ pub fn func_start_srcloc(&self, key: FuncKey) -> FilePos {self.index.src_loc(key).expect("defined function should be present")diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex 9b4294f34f..3952255804 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -19,8 +19,8 @@ use core::ptr::NonNull;use std::{fs::File, path::Path};use wasmparser::{Parser, ValidPayload, Validator};use wasmtime_environ::{- CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind,- StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum,+ CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, FuncKey, HostPtr, ModuleTypes,+ ObjectKind, StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum,};mod registry;@@ -1092,7 +1092,8 @@ impl Module {let module = self.compiled_module();let module_index = self.env_module().module_index;self.env_module().defined_func_indices().map(move |idx| {- let loc = module.func_loc(idx);+ let key = FuncKey::DefinedWasmFunction(module_index, idx);+ let loc = module.func_loc(key);let idx = module.module().func_index(idx);ModuleFunction {module: module_index,@@ -1158,7 +1159,6 @@ impl Module {let ptr = self.compiled_module().wasm_to_array_trampoline(trampoline_module_ty)- .expect("always have a trampoline for the trampoline type").as_ptr().cast::<VMWasmCallFunction>().cast_mut();@@ -1275,6 +1275,7 @@ mod tests {use wasmtime_environ::MemoryInitialization;#[test]+ #[cfg_attr(miri, ignore)]fn cow_on_by_default() {let engine = Engine::default();let module = Module::new(diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rsindex 30234e32bc..336639c798 100644--- a/crates/wasmtime/src/runtime/profiling.rs+++ b/crates/wasmtime/src/runtime/profiling.rs@@ -13,7 +13,7 @@ use fxprof_processed_profile::{use std::ops::Range;use std::sync::Arc;use std::time::{Duration, Instant};-use wasmtime_environ::demangle_function_name_or_index;+use wasmtime_environ::{FuncKey, demangle_function_name_or_index};// TODO: collect more data// - On non-Windows, measure thread-local CPU usage between events with@@ -268,7 +268,9 @@ fn module_symbols(name: String, module: &Module) -> Option<LibraryInfo> {.env_module().defined_func_indices().map(|defined_idx| {- let loc = compiled.func_loc(defined_idx);+ let key =+ FuncKey::DefinedWasmFunction(module.env_module().module_index, defined_idx);+ let loc = compiled.func_loc(key);let func_idx = compiled.module().func_index(defined_idx);let mut name = String::new();demangle_function_name_or_index(diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rsindex cc49f4a7cf..e7d79cb4bf 100644--- a/crates/wasmtime/src/runtime/trap.rs+++ b/crates/wasmtime/src/runtime/trap.rs@@ -7,7 +7,7 @@ use core::fmt;use core::num::NonZeroUsize;use wasmtime_core::alloc::TryVec;use wasmtime_environ::{- CompiledTrap, FilePos, demangle_function_name, demangle_function_name_or_index,+ CompiledTrap, FilePos, FuncKey, demangle_function_name, demangle_function_name_or_index,};/// Representation of a WebAssembly trap and what caused it to occur.@@ -501,7 +501,8 @@ impl FrameInfo {pub(crate) fn new(module: Module, text_offset: usize) -> Option<FrameInfo> {let compiled_module = module.compiled_module();let index = compiled_module.func_by_text_offset(text_offset)?;- let func_start = compiled_module.func_start_srcloc(index);+ let key = FuncKey::DefinedWasmFunction(compiled_module.module().module_index, index);+ let func_start = compiled_module.func_start_srcloc(key);let instr =wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset);let index = compiled_module.module().func_index(index);diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rsindex 832652a694..dd2830084d 100644--- a/crates/wasmtime/src/runtime/values.rs+++ b/crates/wasmtime/src/runtime/values.rs@@ -4,7 +4,6 @@ use crate::{StructRef, V128, ValType, prelude::*,};use core::ptr;-use wasmtime_environ::WasmHeapTopType;pub use crate::runtime::vm::ValRaw;@@ -125,16 +124,6 @@ impl Val {Val::AnyRef(None)}- pub(crate) const fn null_top(top: WasmHeapTopType) -> Val {- match top {- WasmHeapTopType::Func => Val::FuncRef(None),- WasmHeapTopType::Extern => Val::ExternRef(None),- WasmHeapTopType::Any => Val::AnyRef(None),- WasmHeapTopType::Exn => Val::ExnRef(None),- WasmHeapTopType::Cont => Val::ContRef(None),- }- }-/// Returns the default value for the given type, if any exists.////// Returns `None` if there is no default value for the given type (fordiff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex 3a57c66547..a42ec6666b 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -53,7 +53,6 @@ use wasmtime_environ::ModuleInternedTypeIndex;mod always_mut;#[cfg(feature = "component-model")]pub mod component;-mod const_expr;mod export;mod gc;mod imports;@@ -100,7 +99,7 @@ pub use crate::runtime::vm::gc::*;pub use crate::runtime::vm::imports::Imports;pub use crate::runtime::vm::instance::{GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,- MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance,+ MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex,};#[cfg(feature = "pooling-allocator")]pub use crate::runtime::vm::instance::{diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rsdeleted file mode 100644index a3d836bacb..0000000000--- a/crates/wasmtime/src/runtime/vm/const_expr.rs+++ /dev/null@@ -1,499 +0,0 @@-//! Evaluating const expressions.--use crate::prelude::*;-use crate::runtime::vm;-use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter};-#[cfg(feature = "gc")]-use crate::{- AnyRef, ArrayRef, ArrayRefPre, ArrayType, ExternRef, I31, StructRef, StructRefPre, StructType,-};-use crate::{OpaqueRootScope, Val};-use wasmtime_environ::{ConstExpr, ConstOp, FuncIndex, GlobalConstValue, GlobalIndex};-#[cfg(feature = "gc")]-use wasmtime_environ::{VMSharedTypeIndex, WasmCompositeInnerType, WasmCompositeType, WasmSubType};--/// An interpreter for const expressions.-///-/// This can be reused across many const expression evaluations to reuse-/// allocated resources, if any.-pub struct ConstExprEvaluator {- stack: TryVec<Val>,- simple: Val,-}--impl Default for ConstExprEvaluator {- fn default() -> ConstExprEvaluator {- ConstExprEvaluator {- stack: TryVec::new(),- simple: Val::I32(0),- }- }-}--/// The context within which a particular const expression is evaluated.-pub struct ConstEvalContext {- pub(crate) instance: InstanceId,- #[cfg_attr(- not(feature = "gc"),- expect(dead_code, reason = "easier than conditionally compiling this field")- )]- pub(crate) asyncness: Asyncness,-}--impl ConstEvalContext {- /// Create a new context.- pub fn new(instance: InstanceId, asyncness: Asyncness) -> Self {- Self {- instance,- asyncness,- }- }-- fn global_get(&mut self, store: &mut StoreOpaque, index: GlobalIndex) -> Result<Val> {- let id = store.id();- Ok(store- .instance_mut(self.instance)- .get_exported_global(id, index)- ._get(&mut AutoAssertNoGc::new(store)))- }-- fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result<Val> {- let id = store.id();- let (instance, registry) = store.instance_and_module_registry_mut(self.instance);- // SAFETY: `id` is the correct store-owner of the function being looked- // up- let func = unsafe { instance.get_exported_func(registry, id, index) };- Ok(func.into())- }-- #[cfg(feature = "gc")]- fn struct_fields_len(&self, store: &mut StoreOpaque, shared_ty: VMSharedTypeIndex) -> usize {- let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty);- let fields = struct_ty.fields();- fields.len()- }-- #[cfg(feature = "gc")]- async fn struct_new(- &mut self,- store: &mut StoreOpaque,- limiter: Option<&mut StoreResourceLimiter<'_>>,- shared_ty: VMSharedTypeIndex,- fields: &[Val],- ) -> Result<Val> {- let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty);- let allocator = StructRefPre::_new(store, struct_ty);- let struct_ref =- StructRef::_new_async(store, limiter, &allocator, &fields, self.asyncness).await?;- Ok(Val::AnyRef(Some(struct_ref.into())))- }-- #[cfg(feature = "gc")]- async fn struct_new_default(- &mut self,- store: &mut StoreOpaque,- limiter: Option<&mut StoreResourceLimiter<'_>>,- shared_ty: VMSharedTypeIndex,- ) -> Result<Val> {- let module = store- .instance(self.instance)- .runtime_module()- .expect("should never be allocating a struct type defined in a dummy module");-- let borrowed = module- .engine()- .signatures()- .borrow(shared_ty)- .expect("should have a registered type for struct");- let WasmSubType {- composite_type:- WasmCompositeType {- shared: false,- inner: WasmCompositeInnerType::Struct(struct_ty),- },- ..- } = &*borrowed- else {- unreachable!("registered type should be a struct");- };-- let fields = struct_ty- .fields- .iter()- .map(|ty| match &ty.element_type {- wasmtime_environ::WasmStorageType::I8 | wasmtime_environ::WasmStorageType::I16 => {- Val::I32(0)- }- wasmtime_environ::WasmStorageType::Val(v) => match v {- wasmtime_environ::WasmValType::I32 => Val::I32(0),- wasmtime_environ::WasmValType::I64 => Val::I64(0),- wasmtime_environ::WasmValType::F32 => Val::F32(0.0f32.to_bits()),- wasmtime_environ::WasmValType::F64 => Val::F64(0.0f64.to_bits()),- wasmtime_environ::WasmValType::V128 => Val::V128(0u128.into()),- wasmtime_environ::WasmValType::Ref(r) => {- assert!(r.nullable);- Val::null_top(r.heap_type.top())- }- },- })- .collect::<smallvec::SmallVec<[_; 8]>>();-- self.struct_new(store, limiter, shared_ty, &fields).await- }-}--impl ConstExprEvaluator {- /// Same as [`Self::eval`] except that this is specifically intended for- /// integral constant expression.- ///- /// # Panics- ///- /// Panics if `ConstExpr` contains GC ops (e.g. it's not for an integral- /// type).- pub fn eval_int(- &mut self,- store: &mut StoreOpaque,- context: &mut ConstEvalContext,- expr: &ConstExpr,- ) -> Result<&Val> {- // Try to evaluate a simple expression first before doing the more- // complicated eval loop below.- if self.try_simple(expr).is_some() {- return Ok(&self.simple);- }-- // Note that `assert_ready` here should be valid as production of an- // integer cannot involve GC meaning that async operations aren't used.- let mut scope = OpaqueRootScope::new(store);- vm::assert_ready(self.eval_loop(&mut scope, None, context, expr))- }-- /// Attempts to peek into `expr` to see if it's trivial to evaluate, e.g.- /// for `i32.const N`.- #[inline]- pub fn try_simple(&mut self, expr: &ConstExpr) -> Option<&Val> {- match expr.const_eval()? {- GlobalConstValue::I32(i) => Some(self.return_one(Val::I32(i))),- GlobalConstValue::I64(i) => Some(self.return_one(Val::I64(i))),- GlobalConstValue::F32(f) => Some(self.return_one(Val::F32(f))),- GlobalConstValue::F64(f) => Some(self.return_one(Val::F64(f))),- GlobalConstValue::V128(i) => Some(self.return_one(Val::V128(i.into()))),- }- }-- /// Evaluate the given const expression in the given context.- ///- /// Note that the `store` argument is an `OpaqueRootScope` which is used to- /// require that a GC rooting scope external to evaluation of this constant- /// is required. Constant expression evaluation may perform GC allocations- /// and itself trigger a GC meaning that all references must be rooted,- /// hence the external requirement of a rooting scope.- ///- /// # Panics- ///- /// This function will panic if `expr` is an invalid constant expression.- pub async fn eval(- &mut self,- store: &mut OpaqueRootScope<&mut StoreOpaque>,- limiter: Option<&mut StoreResourceLimiter<'_>>,- context: &mut ConstEvalContext,- expr: &ConstExpr,- ) -> Result<&Val> {- // Same structure as `eval_int` above, except using `.await` and with a- // slightly different type signature here for this function.- if self.try_simple(expr).is_some() {- return Ok(&self.simple);- }- self.eval_loop(store, limiter, context, expr).await- }-- #[inline]- fn return_one(&mut self, val: Val) -> &Val {- self.simple = val;- &self.simple- }-- #[cold]- async fn eval_loop(- &mut self,- store: &mut OpaqueRootScope<&mut StoreOpaque>,- mut limiter: Option<&mut StoreResourceLimiter<'_>>,- context: &mut ConstEvalContext,- expr: &ConstExpr,- ) -> Result<&Val> {- log::trace!("evaluating const expr: {expr:?}");-- self.stack.clear();-- // On GC-less builds ensure that this is always considered used an- // needed-mutable.- let _ = &mut limiter;-- for op in expr.ops() {- log::trace!("const-evaluating op: {op:?}");- match op {- ConstOp::I32Const(i) => self.stack.push(Val::I32(*i))?,- ConstOp::I64Const(i) => self.stack.push(Val::I64(*i))?,- ConstOp::F32Const(f) => self.stack.push(Val::F32(*f))?,- ConstOp::F64Const(f) => self.stack.push(Val::F64(*f))?,- ConstOp::V128Const(v) => self.stack.push(Val::V128((*v).into()))?,- ConstOp::GlobalGet(g) => self.stack.push(context.global_get(store, *g)?)?,- ConstOp::RefNull(ty) => self.stack.push(Val::null_top(*ty))?,- ConstOp::RefFunc(f) => self.stack.push(context.ref_func(store, *f)?)?,- #[cfg(feature = "gc")]- ConstOp::RefI31 => {- let i = self.pop()?.unwrap_i32();- let i31 = I31::wrapping_i32(i);- let r = AnyRef::_from_i31(&mut AutoAssertNoGc::new(store), i31);- self.stack.push(Val::AnyRef(Some(r)))?;- }- #[cfg(not(feature = "gc"))]- ConstOp::RefI31 => panic!("should not have validated"),- ConstOp::I32Add => {- let b = self.pop()?.unwrap_i32();- let a = self.pop()?.unwrap_i32();- self.stack.push(Val::I32(a.wrapping_add(b)))?;- }- ConstOp::I32Sub => {- let b = self.pop()?.unwrap_i32();- let a = self.pop()?.unwrap_i32();- self.stack.push(Val::I32(a.wrapping_sub(b)))?;- }- ConstOp::I32Mul => {- let b = self.pop()?.unwrap_i32();- let a = self.pop()?.unwrap_i32();- self.stack.push(Val::I32(a.wrapping_mul(b)))?;- }- ConstOp::I64Add => {- let b = self.pop()?.unwrap_i64();- let a = self.pop()?.unwrap_i64();- self.stack.push(Val::I64(a.wrapping_add(b)))?;- }- ConstOp::I64Sub => {- let b = self.pop()?.unwrap_i64();- let a = self.pop()?.unwrap_i64();- self.stack.push(Val::I64(a.wrapping_sub(b)))?;- }- ConstOp::I64Mul => {- let b = self.pop()?.unwrap_i64();- let a = self.pop()?.unwrap_i64();- self.stack.push(Val::I64(a.wrapping_mul(b)))?;- }-- #[cfg(not(feature = "gc"))]- ConstOp::StructNew { .. }- | ConstOp::StructNewDefault { .. }- | ConstOp::ArrayNew { .. }- | ConstOp::ArrayNewDefault { .. }- | ConstOp::ArrayNewFixed { .. }- | ConstOp::ExternConvertAny- | ConstOp::AnyConvertExtern => {- bail!(- "const expr evaluation error: struct operations are not \- supported without the `gc` feature"- )- }-- #[cfg(feature = "gc")]- ConstOp::StructNew { struct_type_index } => {- let interned_type_index = store.instance(context.instance).env_module().types- [*struct_type_index]- .unwrap_engine_type_index();- let len = context.struct_fields_len(store, interned_type_index);-- if self.stack.len() < len {- bail!(- "const expr evaluation error: expected at least {len} values on the stack, found {}",- self.stack.len()- )- }-- let start = self.stack.len() - len;- let s = context- .struct_new(- store,- limiter.as_deref_mut(),- interned_type_index,- &self.stack[start..],- )- .await?;- self.stack.truncate(start);- self.stack.push(s)?;- }-- #[cfg(feature = "gc")]- ConstOp::StructNewDefault { struct_type_index } => {- let ty = store.instance(context.instance).env_module().types- [*struct_type_index]- .unwrap_engine_type_index();- self.stack.push(- context- .struct_new_default(store, limiter.as_deref_mut(), ty)- .await?,- )?;- }-- #[cfg(feature = "gc")]- ConstOp::ArrayNew { array_type_index } => {- let ty = store.instance(context.instance).env_module().types[*array_type_index]- .unwrap_engine_type_index();- let ty = ArrayType::from_shared_type_index(store.engine(), ty);-- let len = self.pop()?.unwrap_i32().cast_unsigned();-- let elem = self.pop()?;-- let pre = ArrayRefPre::_new(store, ty);- let array = ArrayRef::_new_async(- store,- limiter.as_deref_mut(),- &pre,- &elem,- len,- context.asyncness,- )- .await?;-- self.stack.push(Val::AnyRef(Some(array.into())))?;- }-- #[cfg(feature = "gc")]- ConstOp::ArrayNewDefault { array_type_index } => {- let ty = store.instance(context.instance).env_module().types[*array_type_index]- .unwrap_engine_type_index();- let ty = ArrayType::from_shared_type_index(store.engine(), ty);-- let len = self.pop()?.unwrap_i32().cast_unsigned();-- let elem = Val::default_for_ty(ty.element_type().unpack())- .expect("type should have a default value");-- let pre = ArrayRefPre::_new(store, ty);- let array = ArrayRef::_new_async(- store,- limiter.as_deref_mut(),- &pre,- &elem,- len,- context.asyncness,- )- .await?;-- self.stack.push(Val::AnyRef(Some(array.into())))?;- }-- #[cfg(feature = "gc")]- ConstOp::ArrayNewFixed {- array_type_index,- array_size,- } => {- let ty = store.instance(context.instance).env_module().types[*array_type_index]- .unwrap_engine_type_index();- let ty = ArrayType::from_shared_type_index(store.engine(), ty);-- let array_size = usize::try_from(*array_size).unwrap();- if self.stack.len() < array_size {- bail!(- "const expr evaluation error: expected at least {array_size} values on the stack, found {}",- self.stack.len()- )- }-- let start = self.stack.len() - array_size;-- let elems = self- .stack- .drain(start..)- .collect::<smallvec::SmallVec<[_; 8]>>();-- let pre = ArrayRefPre::_new(store, ty);- let array = ArrayRef::_new_fixed_async(- store,- limiter.as_deref_mut(),- &pre,- &elems,- context.asyncness,- )- .await?;-- self.stack.push(Val::AnyRef(Some(array.into())))?;- }-- #[cfg(feature = "gc")]- ConstOp::ExternConvertAny => {- let mut store = AutoAssertNoGc::new(store);- let result = match self.pop()?.unwrap_anyref() {- Some(anyref) => Some(ExternRef::_convert_any(&mut store, *anyref)?),- None => None,- };- self.stack.push(Val::ExternRef(result))?;- }-- #[cfg(feature = "gc")]- ConstOp::AnyConvertExtern => {- let mut store = AutoAssertNoGc::new(store);- let result = match self.pop()?.unwrap_externref() {- Some(externref) => Some(AnyRef::_convert_extern(&mut store, *externref)?),- None => None,- };- self.stack.push(result.into())?;- }- }- }-- if self.stack.len() == 1 {- log::trace!("const expr evaluated to {:?}", self.stack[0]);- Ok(&self.stack[0])- } else {- bail!(- "const expr evaluation error: expected 1 resulting value, found {}",- self.stack.len()- )- }- }-- fn pop(&mut self) -> Result<Val> {- self.stack.pop().ok_or_else(|| {- format_err!(- "const expr evaluation error: attempted to pop from an empty \- evaluation stack"- )- })- }-}--#[cfg(test)]-mod tests {- use super::{ConstExprEvaluator, Val};- use wasmtime_environ::{ConstExpr, ConstOp};-- #[test]- fn pop_empty_stack() {- let mut evaluator = ConstExprEvaluator::default();- let err = evaluator.pop().unwrap_err();- assert!(- format!("{err}").contains("attempted to pop from an empty"),- "unexpected error: {err}"- );- }-- #[test]- fn try_simple_scalar_types() {- let mut ev = ConstExprEvaluator::default();-- let expr = ConstExpr::new([ConstOp::I32Const(7)]);- assert!(matches!(ev.try_simple(&expr), Some(Val::I32(7))));-- let expr = ConstExpr::new([ConstOp::I64Const(99)]);- assert!(matches!(ev.try_simple(&expr), Some(Val::I64(99))));-- let expr = ConstExpr::new([ConstOp::F32Const(0x3f800000)]);- assert!(matches!(ev.try_simple(&expr), Some(Val::F32(_))));-- let expr = ConstExpr::new([ConstOp::F64Const(0x3ff0000000000000)]);- assert!(matches!(ev.try_simple(&expr), Some(Val::F64(_))));-- let expr = ConstExpr::new([ConstOp::V128Const(0x12345678)]);- assert!(matches!(ev.try_simple(&expr), Some(Val::V128(_))));- }-}diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rsindex 15dbbaae5d..6f8129f47c 100644--- a/crates/wasmtime/src/runtime/vm/cow.rs+++ b/crates/wasmtime/src/runtime/vm/cow.rs@@ -194,7 +194,7 @@ impl ModuleMemoryImages {// If there's no initialization for this memory known then we don't// need an image for the memory so push `None` and move on.- let init = match init {+ let (offset, runtime_index) = match init {Some(init) => init,None => {memories.push(None)?;@@ -202,11 +202,14 @@ impl ModuleMemoryImages {}};- let data_range = init.data.start as usize..init.data.end as usize;+ let data_range = &module.runtime_data[*runtime_index];+ let data_range = usize::try_from(data_range.start).unwrap()+ ..usize::try_from(data_range.end).unwrap();+if module.memories[memory_index].minimum_byte_size().map_or(false, |mem_initial_len| {- init.offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len+ *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len}){// The image is rounded up to multiples of the host OS page@@ -220,7 +223,7 @@ impl ModuleMemoryImages {return Ok(None);}- let offset_usize = match usize::try_from(init.offset) {+ let offset_usize = match usize::try_from(*offset) {Ok(offset) => offset,Err(_) => return Ok(None),};diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 38057577f1..61be8250cd 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -1370,7 +1370,8 @@ mod tests {num_defined_globals: 0,num_defined_tags: 0,num_escaped_funcs: 0,- num_passive_data: 0,+ num_runtime_data: 0,+ has_startup_func: false,});assert_eq!(diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex 9b097c43e7..600d3c1a88 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -2,7 +2,6 @@//! wasm module (except its callstack and register state). An//! `InstanceHandle` is a reference-counting handle for an `Instance`.-use crate::Val;use crate::code::ModuleWithCode;use crate::module::ModuleRegistry;use crate::prelude::*;@@ -18,9 +17,7 @@ use crate::runtime::vm::{GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGcRef, VMGlobalKind, VMStore,VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,};-use crate::store::{- AutoAssertNoGc, InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter,-};+use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter};use crate::vm::{VMWasmCallFunction, ValRaw};use alloc::sync::Arc;use core::alloc::Layout;@@ -35,10 +32,11 @@ use core::{mem, ptr};use wasmtime_environ::ModuleInternedTypeIndex;use wasmtime_environ::error::OutOfMemory;use wasmtime_environ::{- DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, EntityIndex,- EntityRef, FuncIndex, GlobalIndex, HostPtr, MemoryIndex, PassiveDataIndex, PassiveElemIndex,- PtrSize, TableIndex, TableInitialValue, TagIndex, VMCONTEXT_MAGIC, VMOffsets,- VMSharedTypeIndex, WasmRefType, packed_option::ReservedValue,+ Abi, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,+ DefinedTagIndex, EntityIndex, EntityRef, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,+ HostPtr, MemoryIndex, MemoryInitialization, ModuleStartup, PassiveElemIndex, PtrSize,+ RuntimeDataIndex, TableIndex, TagIndex, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex,+ WasmRefType, packed_option::ReservedValue,};#[cfg(feature = "wmemcheck")]use wasmtime_wmemcheck::Wmemcheck;@@ -170,7 +168,7 @@ impl Instance {) -> Result<InstanceHandle, OutOfMemory> {let module = req.runtime_info.env_module();let memory_tys = &module.memories;- let passive_elements = TryVec::with_capacity(module.passive_elements.len())?;+ let mut passive_elements = TryVec::with_capacity(module.passive_elements.len())?;#[cfg(feature = "wmemcheck")]let wmemcheck_state = if req.store.engine().config().wmemcheck {@@ -188,6 +186,11 @@ impl Instance {#[cfg(not(feature = "wmemcheck"))]let _ = memory_tys;+ for (_, (ty, len)) in req.runtime_info.env_module().passive_elements.iter() {+ let len = usize::try_from(*len).unwrap();+ passive_elements.push(PassiveElementSegment::new(*ty, len)?)?;+ }+let mut ret = OwnedInstance::new(Instance {id: req.id,runtime_info: req.runtime_info.clone(),@@ -205,6 +208,7 @@ impl Instance {unsafe {ret.get_mut().initialize_vmctx(req.store, req.imports);}+Ok(ret)}@@ -447,6 +451,7 @@ impl Instance {}/// Get a locally defined or imported memory.+ #[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {if let Some(defined_index) = self.env_module().defined_memory_index(index) {self.memory(defined_index)@@ -594,6 +599,26 @@ impl Instance {unsafe { crate::Func::from_vm_func_ref(store, func_ref) }}+ /// Returns a `Func` corresponding to the startup function for this+ /// instance, if generated at compile time.+ ///+ /// # Safety+ ///+ /// The `store` parameter must be the store that owns this instance and the+ /// functions that this instance can reference.+ pub unsafe fn get_startup_func(+ self: Pin<&mut Self>,+ registry: &ModuleRegistry,+ store: StoreId,+ ) -> Option<crate::Func> {+ let func_ref = self.get_start_func_ref(registry)?;++ // SAFETY: the validity of `func_ref` is guaranteed by the validity of+ // `self`, and the contract that `store` must own `func_ref` is a+ // contract of this function itself.+ Some(unsafe { crate::Func::from_vm_func_ref(store, func_ref) })+ }+/// Lookup a table by index.////// # Panics@@ -820,11 +845,86 @@ impl Instance {return None;}- let Some(def_index) = self.env_module().defined_func_index(index) else {- debug_assert!(self.env_module().is_imported_function(index));- return Some(self.imported_function(index).as_func_ref().into());+ match self.env_module().defined_func_index(index) {+ Some(index) => self.initialize_defined_funcref(registry, index),+ None => {+ debug_assert!(self.env_module().is_imported_function(index));+ Some(self.imported_function(index).as_func_ref().into())+ }+ }+ }++ /// Initializes a defined function's `VMFuncRef` in-place and then returns a+ /// pointer to that location.+ fn initialize_defined_funcref(+ self: Pin<&mut Self>,+ registry: &ModuleRegistry,+ def_index: DefinedFuncIndex,+ ) -> Option<NonNull<VMFuncRef>> {+ let module = self.env_module();+ let index = module.func_index(def_index);+ let func = &module.functions[index];+ let type_index = func.signature.unwrap_engine_type_index();+ let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref);+ let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index);+ let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index);+ // SAFETY: the type/offset/keys here are all valid for the defined+ // function at `def_index`.+ unsafe {+ self.initialize_and_return_funcref(+ registry,+ type_index,+ vmctx_offset,+ array_to_wasm_key,+ wasm_key,+ )+ }+ }++ fn get_start_func_ref(+ self: Pin<&mut Self>,+ registry: &ModuleRegistry,+ ) -> Option<NonNull<VMFuncRef>> {+ let module = self.env_module();+ let type_index = match module.startup {+ ModuleStartup::None => return None,+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => {+ t.unwrap_engine_type_index()+ }};+ let vmctx_offset = self.offsets().vmctx_startup_func_ref();+ let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index);+ let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index);+ // SAFETY: the type/offset/keys here are all valid for the module+ // startup function.+ unsafe {+ self.initialize_and_return_funcref(+ registry,+ type_index,+ vmctx_offset,+ array_to_wasm_key,+ wasm_key,+ )+ }+ }+ /// Common implementation of initializing a `VMFuncRef` stored within this+ /// instance's `VMContext`.+ ///+ /// # Safety+ ///+ /// This function requires that `type_index` accurately describes this+ /// function and `vmctx_offset` is indeed the correct offset for the+ /// functions here. Effectively all the arguments here must be "logically+ /// correct" for the `VMFuncRef` being initialized.+ unsafe fn initialize_and_return_funcref(+ self: Pin<&mut Self>,+ registry: &ModuleRegistry,+ type_index: VMSharedTypeIndex,+ vmctx_offset: u32,+ array_to_wasm_key: FuncKey,+ wasm_key: FuncKey,+ ) -> Option<NonNull<VMFuncRef>> {// For now, we eagerly initialize an funcref struct in-place whenever// asked for a reference to it. This is mostly fine, because in practice// each funcref is unlikely to be requested more than a few times:@@ -846,9 +946,6 @@ impl Instance {// it's better for instantiation performance if we don't have to track// "is-initialized" state at all!- let func = &self.env_module().functions[index];- let type_index = func.signature.unwrap_engine_type_index();-let module_with_code = ModuleWithCode::in_store(registry,self.runtime_module()@@ -856,19 +953,13 @@ impl Instance {).expect("module not in store");- let array_call = VmPtr::from(- NonNull::from(- module_with_code- .array_to_wasm_trampoline(def_index)- .expect("should have array-to-Wasm trampoline for escaping function"),- )- .cast(),- );+ let array_call =+ VmPtr::from(NonNull::from(module_with_code.function(array_to_wasm_key)).cast());let wasm_call = Some(VmPtr::from(NonNull::new(module_with_code- .finished_function(def_index)+ .function(wasm_key).as_ptr().cast::<VMWasmCallFunction>().cast_mut(),@@ -880,9 +971,7 @@ impl Instance {// SAFETY: the offset calculated here should be correct with// `self.offsets`- let func_ref_ptr = unsafe {- self.vmctx_plus_offset_raw::<VMFuncRef>(self.offsets().vmctx_func_ref(func.func_ref))- };+ let func_ref_ptr = unsafe { self.vmctx_plus_offset_raw::<VMFuncRef>(vmctx_offset) };// SAFETY: the `func_ref_ptr` should be valid as it's within our// `VMContext` area.@@ -899,15 +988,16 @@ impl Instance {}/// Get the passive elements segment at the given index.- pub(crate) fn passive_element_segment(&self, passive: PassiveElemIndex) -> &[ValRaw] {- self.passive_elements[passive.index()].elements()+ pub(crate) fn passive_element_segment(+ self: Pin<&mut Self>,+ passive: PassiveElemIndex,+ ) -> &mut [ValRaw] {+ self.passive_elements_mut()[passive.index()].elements_mut()}- pub(crate) fn passive_elements_mut(- self: Pin<&mut Self>,- ) -> Pin<&mut TryVec<PassiveElementSegment>> {+ pub(crate) fn passive_elements_mut(self: Pin<&mut Self>) -> &mut TryVec<PassiveElementSegment> {// SAFETY: Not moving data out of `self`.- Pin::new(&mut unsafe { self.get_unchecked_mut() }.passive_elements)+ &mut unsafe { self.get_unchecked_mut() }.passive_elements}/// Drop an element.@@ -946,7 +1036,7 @@ impl Instance {&self.runtime_info.wasm_data()[start..end]}- /// Returns the data for the passive segment identified by `index`+ /// Returns the data for the runtime segment identified by `index`////// Does not take into account the dynamic size of the data pointed to by/// `index`, always returns the raw data from the module itself.@@ -954,8 +1044,8 @@ impl Instance {/// # Panics////// Panics if `index` is out-of-bounds.- fn passive_data(&self, index: PassiveDataIndex) -> &[u8] {- let range = self.env_module().passive_data[index].clone();+ fn runtime_data(&self, index: RuntimeDataIndex) -> &[u8] {+ let range = self.env_module().runtime_data[index].clone();self.wasm_data(range)}@@ -1013,10 +1103,7 @@ impl Instance {// that `i` may be outside the limits of the static// initialization so it's a fallible `get` instead of an index.let module = self.env_module();- let precomputed = match &module.table_initialization.initial_values[idx] {- TableInitialValue::Null { precomputed } => precomputed,- TableInitialValue::Expr(_) => unreachable!(),- };+ let precomputed = &module.table_initialization[idx];// Panicking here helps catch bugs rather than silently truncating by accident.let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned();let func_ref = func_index@@ -1031,13 +1118,6 @@ impl Instance {self.get_defined_table(idx)}- /// Get a table by index regardless of whether it is locally-defined or an- /// imported, foreign table.- pub(crate) fn get_table(self: Pin<&mut Self>, table_index: TableIndex) -> &mut Table {- let (idx, instance) = self.defined_table_index_and_instance(table_index);- instance.get_defined_table(idx)- }-/// Get a locally-defined table.pub(crate) fn get_defined_table(self: Pin<&mut Self>, index: DefinedTableIndex) -> &mut Table {&mut self.tables_mut()[index].1@@ -1237,9 +1317,21 @@ impl Instance {// after this, but if it's read then it'd hopefully crash faster than// leaving this undefined.unsafe {- for (index, _init) in module.global_initializers.iter() {+ for i in 0..module.num_defined_globals() {+ let index = DefinedGlobalIndex::new(i);instance.global_ptr(index).write(VMGlobalDefinition::new());}+ for (index, val) in module.global_initializers.iter() {+ let mut def = VMGlobalDefinition::new();+ match val {+ GlobalConstValue::I32(i) => *def.as_i32_mut() = *i,+ GlobalConstValue::I64(i) => *def.as_i64_mut() = *i,+ GlobalConstValue::F32(i) => *def.as_f32_bits_mut() = *i,+ GlobalConstValue::F64(i) => *def.as_f64_bits_mut() = *i,+ GlobalConstValue::V128(i) => def.set_u128(*i),+ }+ instance.global_ptr(*index).write(def);+ }}// Initialize the defined tags@@ -1261,7 +1353,7 @@ impl Instance {}}- // Initialize the lengths of passive data segments.+ // Initialize the lengths of runtime data segments.//// SAFETY: it's safe to initialize these lengths during initialization// here and the various types of pointers and such here should all be@@ -1269,17 +1361,48 @@ impl Instance {unsafe {let offsets = instance.runtime_info.offsets();let mut lengths =- instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_lengths_begin());+ instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin());let mut bases =- instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_bases_begin());- for i in module.passive_data.keys() {- let data = instance.passive_data(i);+ instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin());+ for i in module.runtime_data.keys() {+ let data = instance.runtime_data(i);lengths.write(u32::try_from(data.len()).unwrap());lengths = lengths.add(1);bases.write(VmPtr::from(NonNull::from(data).cast::<u8>()));bases = bases.add(1);}}++ // This is the half of the strategy of implementing memory-init-cow data+ // segments. Notably the compiled startup function, if present, will+ // skip data segments that have a null pointer. Here each linear memory+ // is tested to see if it needs initialization. If it does, then the+ // data segment is left in-place (and the startup function will+ // initialize linear memory). Otherwise the data segment is null'd out.+ // If the startup function runs (e.g. something else in the module needs+ // it), then the corresponding data segment's initialization will be+ // skipped.+ if let MemoryInitialization::Static { map } = &module.memory_initialization {+ for (memory, init) in map {+ let Some(memory) = module.defined_memory_index(memory) else {+ continue;+ };+ if instance.memories[memory].1.needs_init() {+ continue;+ }+ if let Some((_offset, data)) = init {+ let offsets = instance.runtime_info.offsets();+ unsafe {+ instance+ .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data))+ .write(0u32);+ instance+ .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data))+ .write(0usize);+ }+ }+ }+ }}/// Attempts to convert from the host `addr` specified to a WebAssembly@@ -1397,6 +1520,17 @@ impl Instance {// SAFETY: see `store_mut` above.unsafe { &mut self.get_unchecked_mut().wmemcheck_state }}++ pub(crate) fn needs_startup(&self) -> bool {+ match self.env_module().startup {+ ModuleStartup::None => false,+ ModuleStartup::Always(_) => true,+ ModuleStartup::IfMemoriesNeedInit(_) => self+ .memories+ .iter()+ .any(|(_, (_, memory))| memory.needs_init()),+ }+ }}// SAFETY: `layout` should describe this accurately and `OwnedVMContext` is the@@ -1721,39 +1855,14 @@ pub(crate) struct PassiveElementSegment {impl PassiveElementSegment {/// Create a new passive element segment with the given capacity.pub(crate) fn new(ty: WasmRefType, capacity: usize) -> Result<Self, OutOfMemory> {+ let mut elements = TryVec::with_capacity(capacity)?;+ elements.resize_with(capacity, || ValRaw::null())?;Ok(Self {needs_gc_rooting: ty.is_vmgcref_type_and_not_i31(),- elements: TryVec::with_capacity(capacity)?,+ elements,})}- /// Push a value onto this passive element segment.- ///- /// NB: Does not type check the value, relies on callers to ensure the- /// value is of the correct type (generally, due to validation).- pub(crate) fn push(&mut self, store: &mut StoreOpaque, val: Val) -> Result<()> {- let mut val = {- let mut store = AutoAssertNoGc::new(store);- val.to_raw_(&mut store)?- };- if self.needs_gc_rooting {- // Note that `anyref` accessors and constructors are used here- // without actually checking the type of this segment or value. The- // representation and handling of all three is the same which means- // that this should work out.- let gc_ref = val.get_anyref();- debug_assert_eq!(gc_ref, val.get_exnref());- debug_assert_eq!(gc_ref, val.get_externref());- if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) {- if let Some(gc_store) = store.optional_gc_store_mut() {- val = ValRaw::anyref(gc_store.clone_gc_ref(&gc_ref).as_raw_u32());- }- }- }- self.elements.push(val)?;- Ok(())- }-/// Clear this segment's elements.pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) {let elements = mem::take(&mut self.elements);@@ -1776,12 +1885,6 @@ impl PassiveElementSegment {}/// The elements of this segment.- pub(crate) fn elements(&self) -> &[ValRaw] {- &self.elements- }-- /// The elements of this segment.- #[cfg(feature = "gc")]pub(crate) fn elements_mut(&mut self) -> &mut [ValRaw] {&mut self.elements}diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rsindex e2f6ae0824..47e301ede3 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs@@ -1,21 +1,16 @@use crate::prelude::*;-use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator};use crate::runtime::vm::imports::Imports;use crate::runtime::vm::instance::{Instance, InstanceHandle};use crate::runtime::vm::memory::Memory;use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::table::Table;use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo};-use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter};-use crate::vm::instance::PassiveElementSegment;-use crate::{OpaqueRootScope, Val};+use crate::store::{InstanceId, StoreOpaque, StoreResourceLimiter};use core::future::Future;+use core::mem;use core::pin::Pin;-use core::{mem, ptr};use wasmtime_environ::{- DefinedMemoryIndex, DefinedTableIndex, EntityRef, HostPtr, InitMemory, MemoryInitialization,- MemoryInitializer, MemoryKind, Module, SizeOverflow, TableInitialValue, TableSegmentElements,- Trap, VMOffsets, WasmRefType,+ DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryKind, Module, VMOffsets,};#[cfg(feature = "gc")]@@ -499,445 +494,6 @@ impl dyn InstanceAllocator + '_ {}}-fn check_table_init_bounds(- store: &mut StoreOpaque,- instance: InstanceId,- module: &Module,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,-) -> Result<()> {- let mut store = OpaqueRootScope::new(store);-- for segment in module.table_initialization.segments.iter() {- let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?;- let start = get_index(start, module.tables[segment.table_index].idx_type);- let end = start.checked_add(segment.elements.len());-- let table = store.instance_mut(instance).get_table(segment.table_index);- match end {- Some(end) if end <= u64::try_from(table.size())? => {- // Initializer is in bounds- }- _ => {- bail!(Trap::TableOutOfBounds);- }- }- }-- Ok(())-}--async fn initialize_tables(- store: &mut StoreOpaque,- mut limiter: Option<&mut StoreResourceLimiter<'_>>,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,- module: &Module,-) -> Result<()> {- let mut store = OpaqueRootScope::new(store);- for (table, init) in module.table_initialization.initial_values.iter() {- match init {- // Tables are always initially null-initialized at this time- TableInitialValue::Null { precomputed: _ } => {}-- TableInitialValue::Expr(expr) => {- let init = const_evaluator- .eval(&mut store, limiter.as_deref_mut(), context, expr)- .await?;- let idx = module.table_index(table);- let id = store.id();- let table = store- .instance_mut(context.instance)- .get_exported_table(id, idx);- let size = table.size_(&store);- table._fill(&mut store, 0, init.ref_().unwrap(), size)?;- }- }- }-- // Note: if the module's table initializer state is in- // FuncTable mode, we will lazily initialize tables based on- // any statically-precomputed image of FuncIndexes, but there- // may still be "leftover segments" that could not be- // incorporated. So we have a unified handler here that- // iterates over all segments (Segments mode) or leftover- // segments (FuncTable mode) to initialize.- for segment in module.table_initialization.segments.iter() {- let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?;- let start = get_index(start, module.tables[segment.table_index].idx_type);-- let end = start- .checked_add(segment.elements.len())- .ok_or_else(|| Trap::TableOutOfBounds)?;-- let store_id = store.id();- let table = {- let instance = store.instance(context.instance);- instance.get_exported_table(store_id, segment.table_index)- };-- if end > table.size_(&store) {- return Err(Trap::TableOutOfBounds.into());- }-- let positions = start..end;-- match &segment.elements {- TableSegmentElements::Functions(funcs) => {- for (i, func_idx) in positions.zip(funcs) {- let func = {- let (instance, registry) =- store.instance_and_module_registry_mut(context.instance);- // SAFETY: the `store_id` passed to `get_exported_func` is- // indeed the store that owns the function.- unsafe { instance.get_exported_func(registry, store_id, *func_idx) }- };- table.set_(&mut store, i, func.into())?;- }- }- TableSegmentElements::Expressions { exprs, ty: _ } => {- for (i, expr) in positions.zip(exprs) {- let val = const_evaluator- .eval(&mut store, limiter.as_deref_mut(), context, expr)- .await?;- table.set_(&mut store, i, val.ref_().unwrap())?;- }- }- }- }-- Ok(())-}--fn get_index(val: &Val, ty: wasmtime_environ::IndexType) -> u64 {- match ty {- wasmtime_environ::IndexType::I32 => val.unwrap_i32().cast_unsigned().into(),- wasmtime_environ::IndexType::I64 => val.unwrap_i64().cast_unsigned(),- }-}--fn get_memory_init_start(- store: &mut StoreOpaque,- init: &MemoryInitializer,- instance: InstanceId,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,-) -> Result<u64> {- let mut store = OpaqueRootScope::new(store);- const_evaluator- .eval_int(&mut store, context, &init.offset)- .map(|v| {- get_index(- v,- store.instance(instance).env_module().memories[init.memory_index].idx_type,- )- })-}--fn check_memory_init_bounds(- store: &mut StoreOpaque,- instance: InstanceId,- initializers: &[MemoryInitializer],- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,-) -> Result<()> {- for init in initializers {- let memory = store.instance_mut(instance).get_memory(init.memory_index);- let start = get_memory_init_start(store, init, instance, context, const_evaluator)?;- let end = usize::try_from(start)- .ok()- .and_then(|start| start.checked_add(init.data.len()));-- match end {- Some(end) if end <= memory.current_length() => {- // Initializer is in bounds- }- _ => {- bail!(Trap::MemoryOutOfBounds);- }- }- }-- Ok(())-}--fn initialize_memories(- store: &mut StoreOpaque,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,- module: &Module,-) -> Result<()> {- // Delegates to the `init_memory` method which is sort of a duplicate of- // `instance.memory_init_segment` but is used at compile-time in other- // contexts so is shared here to have only one method of memory- // initialization.- //- // This call to `init_memory` notably implements all the bells and whistles- // so errors only happen if an out-of-bounds segment is found, in which case- // a trap is returned.-- struct InitMemoryAtInstantiation<'a> {- module: &'a Module,- store: &'a mut StoreOpaque,- context: &'a mut ConstEvalContext,- const_evaluator: &'a mut ConstExprEvaluator,- }-- impl InitMemory for InitMemoryAtInstantiation<'_> {- fn memory_size_in_bytes(- &mut self,- memory: wasmtime_environ::MemoryIndex,- ) -> Result<u64, SizeOverflow> {- let len = self- .store- .instance(self.context.instance)- .get_memory(memory)- .current_length();- let len = u64::try_from(len).unwrap();- Ok(len)- }-- fn eval_offset(- &mut self,- memory: wasmtime_environ::MemoryIndex,- expr: &wasmtime_environ::ConstExpr,- ) -> Option<u64> {- let mut store = OpaqueRootScope::new(&mut *self.store);- let val = self- .const_evaluator- .eval_int(&mut store, self.context, expr)- .ok()?;- Some(get_index(- val,- store.instance(self.context.instance).env_module().memories[memory].idx_type,- ))- }-- fn write(- &mut self,- memory_index: wasmtime_environ::MemoryIndex,- init: &wasmtime_environ::StaticMemoryInitializer,- ) -> bool {- // If this initializer applies to a defined memory but that memory- // doesn't need initialization, due to something like copy-on-write- // pre-initializing it via mmap magic, then this initializer can be- // skipped entirely.- let instance = self.store.instance_mut(self.context.instance);- if let Some(memory_index) = self.module.defined_memory_index(memory_index) {- if !instance.memories[memory_index].1.needs_init() {- return true;- }- }- let memory = instance.get_memory(memory_index);-- unsafe {- let src = instance.wasm_data(init.data.clone());- let offset = usize::try_from(init.offset).unwrap();- let dst = memory.base.as_ptr().add(offset);-- assert!(offset + src.len() <= memory.current_length());-- // FIXME audit whether this is safe in the presence of shared- // memory- // (https://github.com/bytecodealliance/wasmtime/issues/4203).- ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len())- }- true- }- }-- let ok = module- .memory_initialization- .init_memory(&mut InitMemoryAtInstantiation {- module,- store,- context,- const_evaluator,- });- if !ok {- return Err(Trap::MemoryOutOfBounds.into());- }-- Ok(())-}--fn check_init_bounds(- store: &mut StoreOpaque,- instance: InstanceId,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,- module: &Module,-) -> Result<()> {- check_table_init_bounds(store, instance, module, context, const_evaluator)?;-- match &module.memory_initialization {- MemoryInitialization::Segmented(initializers) => {- check_memory_init_bounds(store, instance, initializers, context, const_evaluator)?;- }- // Statically validated already to have everything in-bounds.- MemoryInitialization::Static { .. } => {}- }-- Ok(())-}--async fn initialize_globals(- store: &mut StoreOpaque,- mut limiter: Option<&mut StoreResourceLimiter<'_>>,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,- module: &Module,-) -> Result<()> {- assert!(core::ptr::eq(- &**store.instance(context.instance).env_module(),- module- ));-- let mut store = OpaqueRootScope::new(store);-- for (index, init) in module.global_initializers.iter() {- // Attempt a simple, synchronous evaluation before hitting the- // general-purpose `.await` point below. This benchmarks ~15% faster in- // instantiation vs just falling through to `.await` below.- let val = if let Some(val) = const_evaluator.try_simple(init) {- val- } else {- const_evaluator- .eval(&mut store, limiter.as_deref_mut(), context, init)- .await?- };-- let id = store.id();- let index = module.global_index(index);- let mut instance = store.instance_mut(context.instance);-- #[cfg(feature = "wmemcheck")]- if index.as_u32() == 0- && module.globals[index].wasm_ty == wasmtime_environ::WasmValType::I32- {- if let Some(wmemcheck) = instance.as_mut().wmemcheck_state_mut() {- let size = usize::try_from(val.unwrap_i32()).unwrap();- wmemcheck.set_stack_size(size);- }- }-- let global = instance.as_mut().get_exported_global(id, index);-- // Note that mutability is bypassed here because this is, by definition,- // initialization of globals meaning that if it's an immutable global- // this is the one and only write.- //- // SAFETY: this is a valid module so `val` should have the correct type- // for this global, and it's safe to write to a global for the first- // time as-is happening here.- unsafe {- global.set_unchecked(&mut store, &val)?;- }- }- Ok(())-}--async fn initialize_passive_elements(- store: &mut StoreOpaque,- mut limiter: Option<&mut StoreResourceLimiter<'_>>,- context: &mut ConstEvalContext,- const_evaluator: &mut ConstExprEvaluator,- module: &Module,-) -> Result<()> {- let store_id = store.id();-- let instance = store.instance_mut(context.instance);- debug_assert!(instance.passive_elements.is_empty());- instance- .passive_elements_mut()- .reserve(module.passive_elements.len())?;-- for (idx, segment) in &module.passive_elements {- match segment {- TableSegmentElements::Functions(func_indices) => {- let mut segment =- PassiveElementSegment::new(WasmRefType::FUNCREF, func_indices.len())?;- for func_idx in func_indices {- let (instance, registry) =- store.instance_and_module_registry_mut(context.instance);- // SAFETY: `store_id` is for the store that owns this instance.- let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) };- segment.push(store, func.into())?;- }- let instance = store.instance_mut(context.instance);- debug_assert_eq!(instance.passive_elements.len(), idx.index());- instance.passive_elements_mut().push(segment)?;- }- TableSegmentElements::Expressions { ty, exprs } => {- let mut segment = PassiveElementSegment::new(*ty, exprs.len())?;- for expr in exprs {- let mut store = OpaqueRootScope::new(&mut *store);- let val = const_evaluator- .eval(&mut store, limiter.as_deref_mut(), context, expr)- .await?;- segment.push(&mut store, *val)?;- }- let instance = store.instance_mut(context.instance);- debug_assert_eq!(instance.passive_elements.len(), idx.index());- instance.passive_elements_mut().push(segment)?;- }- }- }-- debug_assert_eq!(- module.passive_elements.len(),- store.instance(context.instance).passive_elements.len()- );- Ok(())-}--pub async fn initialize_instance(- store: &mut StoreOpaque,- mut limiter: Option<&mut StoreResourceLimiter<'_>>,- instance: InstanceId,- module: &Module,- is_bulk_memory: bool,- asyncness: Asyncness,-) -> Result<()> {- let mut context = ConstEvalContext::new(instance, asyncness);- let mut const_evaluator = ConstExprEvaluator::default();-- // If bulk memory is not enabled, bounds check the data and element segments before- // making any changes. With bulk memory enabled, initializers are processed- // in-order and side effects are observed up to the point of an out-of-bounds- // initializer, so the early checking is not desired.- if !is_bulk_memory {- check_init_bounds(store, instance, &mut context, &mut const_evaluator, module)?;- }-- initialize_globals(- store,- limiter.as_deref_mut(),- &mut context,- &mut const_evaluator,- module,- )- .await?;-- initialize_tables(- store,- limiter.as_deref_mut(),- &mut context,- &mut const_evaluator,- module,- )- .await?;-- initialize_memories(store, &mut context, &mut const_evaluator, &module)?;-- if is_bulk_memory {- initialize_passive_elements(store, limiter, &mut context, &mut const_evaluator, &module)- .await?;- }-- Ok(())-}-#[cfg(test)]mod tests {use super::*;diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 241057fa71..f8641c87d6 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -341,8 +341,7 @@ fn passive_elem_segment_base(store.instance_mut(instance).passive_element_segment(elem_index)- .as_ptr()- .cast_mut()+ .as_mut_ptr().cast()}diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex ab097b978e..83844fb4ee 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -4,7 +4,6 @@mod vm_host_func_context;pub use self::vm_host_func_context::VMArrayCallHostFuncContext;-use crate::bail_bug;use crate::prelude::*;use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16};use crate::store::StoreOpaque;@@ -20,7 +19,6 @@ use core::sync::atomic::{AtomicUsize, Ordering};use wasmtime_environ::{BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,DefinedTagIndex, NUM_COMPONENT_CONTEXT_SLOTS, VMCONTEXT_MAGIC, VMSharedTypeIndex,- WasmHeapTopType, WasmValType,};/// A function pointer that exposes the array calling convention.@@ -600,86 +598,6 @@ impl VMGlobalDefinition {Self { storage: [0; 16] }}- /// Create a `VMGlobalDefinition` from a `ValRaw`.- ///- /// # Unsafety- ///- /// This raw value's type must match the given `WasmValType`.- pub unsafe fn from_val_raw(- store: &mut StoreOpaque,- wasm_ty: WasmValType,- raw: ValRaw,- ) -> Result<Self> {- let mut global = Self::new();- unsafe {- match wasm_ty {- WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(),- WasmValType::I64 => *global.as_i64_mut() = raw.get_i64(),- WasmValType::F32 => *global.as_f32_bits_mut() = raw.get_f32(),- WasmValType::F64 => *global.as_f64_bits_mut() = raw.get_f64(),- WasmValType::V128 => global.set_u128(raw.get_v128()),- WasmValType::Ref(r) => match r.heap_type.top() {- WasmHeapTopType::Extern => {- let r = VMGcRef::from_raw_u32(raw.get_externref());- global.init_gc_ref(store, r.as_ref())?- }- WasmHeapTopType::Any => {- let r = VMGcRef::from_raw_u32(raw.get_anyref());- global.init_gc_ref(store, r.as_ref())?- }- WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),- WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack.- WasmHeapTopType::Exn => {- let r = VMGcRef::from_raw_u32(raw.get_exnref());- global.init_gc_ref(store, r.as_ref())?- }- },- }- }- Ok(global)- }-- /// Get this global's value as a `ValRaw`.- ///- /// # Unsafety- ///- /// This global's value's type must match the given `WasmValType`.- pub unsafe fn to_val_raw(- &self,- store: &mut StoreOpaque,- wasm_ty: WasmValType,- ) -> Result<ValRaw> {- unsafe {- Ok(match wasm_ty {- WasmValType::I32 => ValRaw::i32(*self.as_i32()),- WasmValType::I64 => ValRaw::i64(*self.as_i64()),- WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()),- WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()),- WasmValType::V128 => ValRaw::v128(self.get_u128()),- WasmValType::Ref(r) => match r.heap_type.top() {- WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() {- Some(r) => store.clone_gc_ref(r).as_raw_u32(),- None => 0,- }),- WasmHeapTopType::Any => ValRaw::anyref({- match self.as_gc_ref() {- Some(r) => store.clone_gc_ref(r).as_raw_u32(),- None => 0,- }- }),- WasmHeapTopType::Exn => ValRaw::exnref({- match self.as_gc_ref() {- Some(r) => store.clone_gc_ref(r).as_raw_u32(),- None => 0,- }- }),- WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),- WasmHeapTopType::Cont => bail_bug!("unimplemented"), // FIXME: #10248 stack switching support.- },- })- }- }-/// Return a reference to the value as an i32.pub unsafe fn as_i32(&self) -> &i32 {unsafe { &*(self.storage.as_ref().as_ptr().cast::<i32>()) }diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rsindex 3939c26ff7..6a9a7f2982 100644--- a/crates/winch/src/compiler.rs+++ b/crates/winch/src/compiler.rs@@ -167,25 +167,15 @@ impl wasmtime_environ::Compiler for Compiler {})}- fn compile_array_to_wasm_trampoline(- &self,- translation: &ModuleTranslation<'_>,- types: &ModuleTypesBuilder,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- self.trampolines- .compile_array_to_wasm_trampoline(translation, types, key, symbol)- }-- fn compile_wasm_to_array_trampoline(+ fn compile_trampoline(&self,- wasm_func_ty: &wasmtime_environ::WasmFuncType,+ translation: Option<&ModuleTranslation<'_>>,key: FuncKey,+ types: &ModuleTypesBuilder,symbol: &str,) -> Result<CompiledFunctionBody, CompileError> {self.trampolines- .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)+ .compile_trampoline(translation, key, types, symbol)}fn append_code(@@ -236,14 +226,6 @@ impl wasmtime_environ::Compiler for Compiler {self.isa.create_systemv_cie()}- fn compile_wasm_to_builtin(- &self,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- self.trampolines.compile_wasm_to_builtin(key, symbol)- }-fn compiled_function_relocation_targets<'a>(&'a self,func: &'a dyn Any,@@ -280,45 +262,14 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {Ok(body)}- fn compile_array_to_wasm_trampoline(- &self,- translation: &ModuleTranslation<'_>,- types: &ModuleTypesBuilder,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- let mut body = self- .0- .compile_array_to_wasm_trampoline(translation, types, key, symbol)?;- if let Some(c) = self.0.inlining_compiler() {- c.finish_compiling(&mut body, None, symbol)- .map_err(|e| CompileError::Codegen(e.to_string()))?;- }- Ok(body)- }-- fn compile_wasm_to_array_trampoline(- &self,- wasm_func_ty: &wasmtime_environ::WasmFuncType,- key: FuncKey,- symbol: &str,- ) -> Result<CompiledFunctionBody, CompileError> {- let mut body = self- .0- .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)?;- if let Some(c) = self.0.inlining_compiler() {- c.finish_compiling(&mut body, None, symbol)- .map_err(|e| CompileError::Codegen(e.to_string()))?;- }- Ok(body)- }-- fn compile_wasm_to_builtin(+ fn compile_trampoline(&self,+ translation: Option<&ModuleTranslation<'_>>,key: FuncKey,+ types: &ModuleTypesBuilder,symbol: &str,) -> Result<CompiledFunctionBody, CompileError> {- let mut body = self.0.compile_wasm_to_builtin(key, symbol)?;+ let mut body = self.0.compile_trampoline(translation, key, types, symbol)?;if let Some(c) = self.0.inlining_compiler() {c.finish_compiling(&mut body, None, symbol).map_err(|e| CompileError::Codegen(e.to_string()))?;@@ -381,7 +332,7 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {#[cfg(feature = "component-model")]impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {- fn compile_trampoline(+ fn compile_component_trampoline(&self,component: &wasmtime_environ::component::ComponentTranslation,types: &wasmtime_environ::component::ComponentTypesBuilder,@@ -393,7 +344,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {let mut body = self.0.component_compiler()- .compile_trampoline(component, types, key, abi, tunables, symbol)?;+ .compile_component_trampoline(component, types, key, abi, tunables, symbol)?;if let Some(c) = self.0.inlining_compiler() {c.finish_compiling(&mut body, None, symbol).map_err(|e| CompileError::Codegen(e.to_string()))?;diff --git a/src/commands/objdump.rs b/src/commands/objdump.rsindex a81dc2a8be..c85607b634 100644--- a/src/commands/objdump.rs+++ b/src/commands/objdump.rs@@ -205,6 +205,7 @@ impl ObjdumpCommand {|| name.ends_with("_array_call")|| name.ends_with("_wasm_call")|| name.contains("unsafe-intrinsics-")+ || name.contains("module_start"){Func::Trampoline} else if name.contains("libcall") || name.starts_with("component") {diff --git a/tests/all/arrays.rs b/tests/all/arrays.rsindex 31f68b306e..b42a38855d 100644--- a/tests/all/arrays.rs+++ b/tests/all/arrays.rs@@ -1002,6 +1002,7 @@ fn issue_13034_array_layout_overflow() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn host_arrayref_has_trace_info_for_gc() -> Result<()> {for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {println!("Using GC collector: {collector:?}");diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rsindex 96d45dcebd..a7f34fb7fc 100644--- a/tests/all/cli_tests.rs+++ b/tests/all/cli_tests.rs@@ -3051,11 +3051,11 @@ fn profile_guest() -> Result<()> {None,)?;- assert!(output.status.success());let stdout = String::from_utf8_lossy(&output.stdout);let stderr = String::from_utf8_lossy(&output.stderr);println!("> stdout:\n{stdout}");println!("> stderr:\n{stderr}");+ assert!(output.status.success());assert!(!stderr.contains("Error"));let out_json = std::fs::read_to_string(format!("{dir}/out.json")).unwrap();println!("> out.json:\n{out_json}");diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rsindex e4f326877a..cde1dec742 100644--- a/tests/all/exceptions.rs+++ b/tests/all/exceptions.rs@@ -216,6 +216,7 @@ fn exception_across_no_wasm(config: &mut Config) -> Result<()> {}#[wasmtime_test(wasm_features(gc, exceptions))]+#[cfg_attr(miri, ignore)]fn gc_with_exnref_global(config: &mut Config) -> Result<()> {let engine = Engine::new(config)?;let mut store = Store::new(&engine, ());diff --git a/tests/all/exnrefs.rs b/tests/all/exnrefs.rsindex 6251d01ff6..de0b73efbb 100644--- a/tests/all/exnrefs.rs+++ b/tests/all/exnrefs.rs@@ -85,6 +85,7 @@ fn exn_objects() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn host_exnref_has_trace_info_for_gc() -> Result<()> {for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {println!("Using GC collector: {collector:?}");diff --git a/tests/all/fuel.wast b/tests/all/fuel.wastindex 7196df646a..f918eb94ac 100644--- a/tests/all/fuel.wast+++ b/tests/all/fuel.wast@@ -1,11 +1,11 @@(assert_fuel 0 (module))-(assert_fuel 1+(assert_fuel 3(module(func $f)(start $f)))-(assert_fuel 2+(assert_fuel 4(module(func $fi32.const 0@@ -13,7 +13,7 @@)(start $f)))-(assert_fuel 1+(assert_fuel 3(module(func $fblock@@ -21,14 +21,14 @@)(start $f)))-(assert_fuel 1+(assert_fuel 3(module(func $funreachable)(start $f)))-(assert_fuel 7+(assert_fuel 9(module(func $fi32.const 0@@ -41,7 +41,7 @@)(start $f)))-(assert_fuel 1+(assert_fuel 3(module(func $freturn@@ -55,7 +55,7 @@)(start $f)))-(assert_fuel 3+(assert_fuel 5(module(func $fi32.const 0@@ -65,7 +65,7 @@)(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fi32.const 1@@ -76,7 +76,7 @@)(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fi32.const 1@@ -89,7 +89,7 @@)(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fi32.const 0@@ -102,7 +102,7 @@)(start $f)))-(assert_fuel 3+(assert_fuel 5(module(func $fblock@@ -114,7 +114,7 @@)(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fblock@@ -127,7 +127,7 @@(start $f)));; count code before unreachable-(assert_fuel 2+(assert_fuel 4(module(func $fi32.const 0@@ -136,7 +136,7 @@(start $f)));; count code before return-(assert_fuel 2+(assert_fuel 4(module(func $fi32.const 0@@ -145,14 +145,14 @@(start $f)));; cross-function fuel works-(assert_fuel 3+(assert_fuel 5(module(func $fcall $other)(func $other)(start $f)))-(assert_fuel 5+(assert_fuel 7(module(func $fi32.const 0@@ -162,7 +162,7 @@)(func $other (param i32))(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fcall $other@@ -172,7 +172,7 @@i32.const 0)(start $f)))-(assert_fuel 4+(assert_fuel 6(module(func $fi32.const 0@@ -183,14 +183,14 @@(start $f)));; loops!-(assert_fuel 1+(assert_fuel 3(module(func $floopend)(start $f)))-(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func+(assert_fuel 55 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func(module(func $f(local i32)@@ -207,7 +207,7 @@)(start $f)))-(assert_fuel 105+(assert_fuel 107(module(memory 1)(func $f@@ -218,7 +218,7 @@)(start $f)))-(assert_fuel 105+(assert_fuel 107(module(memory 1)(func $f@@ -229,7 +229,7 @@)(start $f)))-(assert_fuel 25+(assert_fuel 27(module(memory 1)(func $f@@ -241,7 +241,7 @@(start $f)(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))-(assert_fuel 105+(assert_fuel 107(module(table 100 funcref)(func $f@@ -252,7 +252,7 @@)(start $f)))-(assert_fuel 105+(assert_fuel 107(module(table 100 funcref)(func $f@@ -263,7 +263,7 @@)(start $f)))-(assert_fuel 104+(assert_fuel 106(module(table 0 funcref)(func $f@@ -274,7 +274,7 @@)(start $f)))-(assert_fuel 25+(assert_fuel 27(module(table 20 funcref)(func $f@@ -286,7 +286,7 @@(start $f)(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))-(assert_fuel 107+(assert_fuel 211(module(type $a (array (mut i8)))(global $a (ref $a) (array.new_default $a (i32.const 100)))@@ -300,7 +300,7 @@)(start $f)))-(assert_fuel 106+(assert_fuel 210(module(type $a (array (mut i8)))(global $a (ref $a) (array.new_default $a (i32.const 100)))@@ -313,7 +313,7 @@)(start $f)))-(assert_fuel 24+(assert_fuel 26(module(type $a (array (mut i8)))(func $f@@ -325,7 +325,7 @@(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")(start $f)))-(assert_fuel 26+(assert_fuel 130(module(type $a (array (mut i8)))(global $a (ref $a) (array.new_default $a (i32.const 100)))@@ -339,7 +339,7 @@(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")(start $f)))-(assert_fuel 24+(assert_fuel 26(module(type $a (array (mut funcref)))(func $f@@ -351,7 +351,7 @@(start $f)(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))-(assert_fuel 26+(assert_fuel 130(module(type $a (array (mut funcref)))(global $a (ref $a) (array.new_default $a (i32.const 100)))@@ -365,7 +365,7 @@(start $f)(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))-(assert_fuel 403 ;; 400 bytes set + 1 func + 2 instructions (drop is 0)+(assert_fuel 105(module(type $a (array (mut funcref)))(func $f@@ -375,7 +375,7 @@)(start $f)))-(assert_fuel 404 ;; 400 bytes set + 1 func + 3 instructions (drop is 0)+(assert_fuel 106(module(type $a (array (mut funcref)))(func $fdiff --git a/tests/all/gc.rs b/tests/all/gc.rsindex 7301fe6bf2..b86a73f0b4 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -373,6 +373,7 @@ fn table_drops_externref() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn global_init_no_leak() -> Result<()> {let (mut store, module) = ref_types_module(false,@@ -1157,6 +1158,7 @@ fn issue_9669() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn drc_transitive_drop_cons_list() -> Result<()> {let _ = env_logger::try_init();@@ -3315,6 +3317,8 @@ fn typed_option_noextern() -> Result<()> {/// A test that performs a GC without actually compiling or running any Wasm/// functions so we can run this test under MIRI.#[test]+// FIXME: need to fold this into other miri testing which precompiles+#[cfg_attr(miri, ignore)]fn miri_gc_smoke_test() -> Result<()> {for collector in [Collector::Copying,diff --git a/tests/all/memory.rs b/tests/all/memory.rsindex 8897ad081d..f321f5d98c 100644--- a/tests/all/memory.rs+++ b/tests/all/memory.rs@@ -510,6 +510,7 @@ fn dynamic_extra_growth_unchanged_pointer(config: &mut Config) -> Result<()> {// determining this failure we shouldn't hit any overflows or anything like that// (checked via debug-mode tests).#[wasmtime_test]+#[cfg_attr(miri, ignore)]fn memory64_maximum_minimum(config: &mut Config) -> Result<()> {config.wasm_memory64(true);let engine = Engine::new(&config)?;diff --git a/tests/all/module.rs b/tests/all/module.rsindex da99f7442b..6a4daff1a7 100644--- a/tests/all/module.rs+++ b/tests/all/module.rs@@ -141,6 +141,7 @@ fn serialize_deterministic() {// into an initialization image doesn't unnecessarily create a massive module by// accident with a very large initialization image in it.#[test]+#[cfg_attr(miri, ignore)]fn serialize_not_overly_massive() -> Result<()> {let mut config = Config::new();config.memory_guaranteed_dense_image_size(1 << 20);diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rsindex c14a6dbbc5..ffb7482003 100644--- a/tests/all/pooling_allocator.rs+++ b/tests/all/pooling_allocator.rs@@ -98,6 +98,7 @@ fn memory_limit() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn memory_init() -> Result<()> {let mut pool = crate::small_pool_config();pool.max_memory_size(2 << 16).table_elements(0);@@ -441,6 +442,7 @@ fn total_core_instances_limit() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn preserve_data_segments() -> Result<()> {let mut pool = crate::small_pool_config();pool.total_memories(2);@@ -516,6 +518,7 @@ fn multi_memory_with_imported_memories() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn drop_externref_global_during_module_init() -> Result<()> {struct Limiter;@@ -1248,6 +1251,7 @@ fn decommit_batching() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn tricky_empty_table_with_empty_virtual_memory_alloc() -> Result<()> {// Configure the pooling allocator to have no access to virtual memory, e.g.// no table elements but a single table. This should technically support a@@ -1458,6 +1462,7 @@ fn pooling_reuse_resets() -> Result<()> {// asserts that the previous image is indeed not available any more as that// would otherwise mean data was leaked between modules.#[test]+#[cfg_attr(miri, ignore)]fn memory_reset_if_instantiation_fails() -> Result<()> {struct Limiter;diff --git a/tests/all/structs.rs b/tests/all/structs.rsindex 59e14ae087..652f028f31 100644--- a/tests/all/structs.rs+++ b/tests/all/structs.rs@@ -912,6 +912,7 @@ fn issue_9714(config: &mut Config) -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn host_structref_has_trace_info_for_gc() -> Result<()> {for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {println!("Using GC collector: {collector:?}");diff --git a/tests/all/table.rs b/tests/all/table.rsindex b4375c479a..6d8d9bd843 100644--- a/tests/all/table.rs+++ b/tests/all/table.rs@@ -368,6 +368,7 @@ fn host_table_keep_type_registration() -> Result<()> {}#[test]+#[cfg_attr(miri, ignore)]fn gc_store_with_table_initializers() -> Result<()> {let mut config = Config::new();config.wasm_gc(true);diff --git a/tests/disas/array-fill-anyref.wat b/tests/disas/array-fill-anyref.watindex 52bc524188..428cb320f9 100644--- a/tests/disas/array-fill-anyref.wat+++ b/tests/disas/array-fill-anyref.wat@@ -81,14 +81,12 @@;; gv4 = load.i64 notrap aligned readonly can_move gv3+8;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40-;; sig0 = (i64 vmctx, i64, i32, i64) tail-;; fn0 = colocated u805306368:2 sig0;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):;; @003e trapz v2, user16-;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8-;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32+;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8+;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32;; @003e v6 = uextend.i64 v2;; @003e v8 = iadd v7, v6;; @003e v9 = iconst.i64 16@@ -100,19 +98,33 @@;; @003e v12 = uextend.i64 v11;; @003e v17 = icmp ugt v16, v12;; @003e trapnz v17, user17-;; @003e v28 = load.i64 notrap aligned v44+40-;; v40 = iconst.i64 20-;; @003e v21 = iadd v8, v40 ; v40 = 20-;; v48 = iconst.i64 2-;; v49 = ishl v13, v48 ; v48 = 2-;; @003e v24 = iadd v21, v49-;; v51 = ishl v14, v48 ; v48 = 2-;; @003e v30 = uadd_overflow_trap v24, v51, user2+;; @003e v28 = load.i64 notrap aligned v49+40+;; v45 = iconst.i64 20+;; @003e v21 = iadd v8, v45 ; v45 = 20+;; v53 = iconst.i64 2+;; v54 = ishl v13, v53 ; v53 = 2+;; @003e v24 = iadd v21, v54+;; v56 = ishl v14, v53 ; v53 = 2+;; @003e v30 = uadd_overflow_trap v24, v56, user2;; @003e v29 = iadd v7, v28;; @003e v31 = icmp ugt v30, v29;; @003e trapnz v31, user2+;; v51 = iconst.i64 0+;; @003e v33 = icmp eq v14, v51 ; v51 = 0;; @003a v5 = iconst.i32 0-;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0+;; v44 = iconst.i64 4+;; @003e v32 = iadd v24, v56+;; @003e brif v33, block3, block2(v24)+;;+;; block2(v34: i64):+;; v58 = iconst.i32 0+;; @003e store user2 little v58, v34 ; v58 = 0+;; v59 = iconst.i64 4+;; v60 = iadd v34, v59 ; v59 = 4+;; @003e v36 = icmp eq v60, v32+;; @003e brif v36, block3, block2(v60)+;;+;; block3:;; @0041 jump block1;;;; block1:diff --git a/tests/disas/array-fill-externref.wat b/tests/disas/array-fill-externref.watindex a327322ede..542857bb45 100644--- a/tests/disas/array-fill-externref.wat+++ b/tests/disas/array-fill-externref.wat@@ -77,14 +77,12 @@;; gv4 = load.i64 notrap aligned readonly can_move gv3+8;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40-;; sig0 = (i64 vmctx, i64, i32, i64) tail-;; fn0 = colocated u805306368:2 sig0;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):;; @003d trapz v2, user16-;; @003d v44 = load.i64 notrap aligned readonly can_move v0+8-;; @003d v7 = load.i64 notrap aligned readonly can_move v44+32+;; @003d v49 = load.i64 notrap aligned readonly can_move v0+8+;; @003d v7 = load.i64 notrap aligned readonly can_move v49+32;; @003d v6 = uextend.i64 v2;; @003d v8 = iadd v7, v6;; @003d v9 = iconst.i64 16@@ -96,19 +94,33 @@;; @003d v12 = uextend.i64 v11;; @003d v17 = icmp ugt v16, v12;; @003d trapnz v17, user17-;; @003d v28 = load.i64 notrap aligned v44+40-;; v40 = iconst.i64 20-;; @003d v21 = iadd v8, v40 ; v40 = 20-;; v48 = iconst.i64 2-;; v49 = ishl v13, v48 ; v48 = 2-;; @003d v24 = iadd v21, v49-;; v51 = ishl v14, v48 ; v48 = 2-;; @003d v30 = uadd_overflow_trap v24, v51, user2+;; @003d v28 = load.i64 notrap aligned v49+40+;; v45 = iconst.i64 20+;; @003d v21 = iadd v8, v45 ; v45 = 20+;; v53 = iconst.i64 2+;; v54 = ishl v13, v53 ; v53 = 2+;; @003d v24 = iadd v21, v54+;; v56 = ishl v14, v53 ; v53 = 2+;; @003d v30 = uadd_overflow_trap v24, v56, user2;; @003d v29 = iadd v7, v28;; @003d v31 = icmp ugt v30, v29;; @003d trapnz v31, user2+;; v51 = iconst.i64 0+;; @003d v33 = icmp eq v14, v51 ; v51 = 0;; @0039 v5 = iconst.i32 0-;; @003d call fn0(v0, v24, v5, v51) ; v5 = 0+;; v44 = iconst.i64 4+;; @003d v32 = iadd v24, v56+;; @003d brif v33, block3, block2(v24)+;;+;; block2(v34: i64):+;; v58 = iconst.i32 0+;; @003d store user2 little v58, v34 ; v58 = 0+;; v59 = iconst.i64 4+;; v60 = iadd v34, v59 ; v59 = 4+;; @003d v36 = icmp eq v60, v32+;; @003d brif v36, block3, block2(v60)+;;+;; block3:;; @0040 jump block1;;;; block1:diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.watindex 4f594096bb..026b754942 100644--- a/tests/disas/array-fill-funcref.wat+++ b/tests/disas/array-fill-funcref.wat@@ -88,14 +88,14 @@;; gv4 = load.i64 notrap aligned readonly can_move gv3+8;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40-;; sig0 = (i64 vmctx, i64, i32, i64) tail-;; fn0 = colocated u805306368:2 sig0+;; sig0 = (i64 vmctx, i64) -> i64 tail+;; fn0 = colocated u805306368:25 sig0;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):;; @0049 trapz v2, user16-;; @0049 v44 = load.i64 notrap aligned readonly can_move v0+8-;; @0049 v7 = load.i64 notrap aligned readonly can_move v44+32+;; @0049 v52 = load.i64 notrap aligned readonly can_move v0+8+;; @0049 v7 = load.i64 notrap aligned readonly can_move v52+32;; @0049 v6 = uextend.i64 v2;; @0049 v8 = iadd v7, v6;; @0049 v9 = iconst.i64 16@@ -107,19 +107,34 @@;; @0049 v12 = uextend.i64 v11;; @0049 v17 = icmp ugt v16, v12;; @0049 trapnz v17, user17-;; @0049 v28 = load.i64 notrap aligned v44+40-;; v40 = iconst.i64 20-;; @0049 v21 = iadd v8, v40 ; v40 = 20-;; v47 = iconst.i64 2-;; v48 = ishl v13, v47 ; v47 = 2-;; @0049 v24 = iadd v21, v48-;; v50 = ishl v14, v47 ; v47 = 2-;; @0049 v30 = uadd_overflow_trap v24, v50, user2+;; @0049 v28 = load.i64 notrap aligned v52+40+;; v48 = iconst.i64 20+;; @0049 v21 = iadd v8, v48 ; v48 = 20+;; v55 = iconst.i64 2+;; v56 = ishl v13, v55 ; v55 = 2+;; @0049 v24 = iadd v21, v56+;; v58 = ishl v14, v55 ; v55 = 2+;; @0049 v30 = uadd_overflow_trap v24, v58, user2;; @0049 v29 = iadd v7, v28;; @0049 v31 = icmp ugt v30, v29;; @0049 trapnz v31, user2-;; @0049 v32 = iconst.i32 0-;; @0049 call fn0(v0, v24, v32, v50) ; v32 = 0+;; @0045 v5 = iconst.i64 0+;; @0049 v33 = icmp eq v14, v5 ; v5 = 0+;; v47 = iconst.i64 4+;; @0049 v32 = iadd v24, v58+;; @0049 brif v33, block3, block2(v24)+;;+;; block2(v34: i64):+;; v60 = iconst.i64 0+;; @0049 v36 = call fn0(v0, v60) ; v60 = 0+;; @0049 v37 = ireduce.i32 v36+;; @0049 store user2 little v37, v34+;; v61 = iconst.i64 4+;; v62 = iadd v34, v61 ; v61 = 4+;; @0049 v39 = icmp eq v62, v32+;; @0049 brif v39, block3, block2(v62)+;;+;; block3:;; @004c jump block1;;;; block1:diff --git a/tests/disas/array-fill-i31ref.wat b/tests/disas/array-fill-i31ref.watindex 6723ac376d..b094bdde3f 100644--- a/tests/disas/array-fill-i31ref.wat+++ b/tests/disas/array-fill-i31ref.wat@@ -81,14 +81,12 @@;; gv4 = load.i64 notrap aligned readonly can_move gv3+8;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40-;; sig0 = (i64 vmctx, i64, i32, i64) tail-;; fn0 = colocated u805306368:2 sig0;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):;; @003e trapz v2, user16-;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8-;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32+;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8+;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32;; @003e v6 = uextend.i64 v2;; @003e v8 = iadd v7, v6;; @003e v9 = iconst.i64 16@@ -100,19 +98,33 @@;; @003e v12 = uextend.i64 v11;; @003e v17 = icmp ugt v16, v12;; @003e trapnz v17, user17-;; @003e v28 = load.i64 notrap aligned v44+40-;; v40 = iconst.i64 20-;; @003e v21 = iadd v8, v40 ; v40 = 20-;; v48 = iconst.i64 2-;; v49 = ishl v13, v48 ; v48 = 2-;; @003e v24 = iadd v21, v49-;; v51 = ishl v14, v48 ; v48 = 2-;; @003e v30 = uadd_overflow_trap v24, v51, user2+;; @003e v28 = load.i64 notrap aligned v49+40+;; v45 = iconst.i64 20+;; @003e v21 = iadd v8, v45 ; v45 = 20+;; v53 = iconst.i64 2+;; v54 = ishl v13, v53 ; v53 = 2+;; @003e v24 = iadd v21, v54+;; v56 = ishl v14, v53 ; v53 = 2+;; @003e v30 = uadd_overflow_trap v24, v56, user2;; @003e v29 = iadd v7, v28;; @003e v31 = icmp ugt v30, v29;; @003e trapnz v31, user2+;; v51 = iconst.i64 0+;; @003e v33 = icmp eq v14, v51 ; v51 = 0;; @003a v5 = iconst.i32 0-;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0+;; v44 = iconst.i64 4+;; @003e v32 = iadd v24, v56+;; @003e brif v33, block3, block2(v24)+;;+;; block2(v34: i64):+;; v58 = iconst.i32 0+;; @003e store user2 little v58, v34 ; v58 = 0+;; v59 = iconst.i64 4+;; v60 = iadd v34, v59 ; v59 = 4+;; @003e v36 = icmp eq v60, v32+;; @003e brif v36, block3, block2(v60)+;;+;; block3:;; @0041 jump block1;;;; block1:diff --git a/tests/disas/gc/array-init-data.wat b/tests/disas/gc/array-init-data.watindex 39ff89c798..c44bdab873 100644--- a/tests/disas/gc/array-init-data.wat+++ b/tests/disas/gc/array-init-data.wat@@ -27,8 +27,8 @@;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):;; @002a trapz v2, user16-;; @002a v57 = load.i64 notrap aligned readonly can_move v0+8-;; @002a v7 = load.i64 notrap aligned readonly can_move v57+32+;; @002a v58 = load.i64 notrap aligned readonly can_move v0+8+;; @002a v7 = load.i64 notrap aligned readonly can_move v58+32;; @002a v6 = uextend.i64 v2;; @002a v8 = iadd v7, v6;; @002a v9 = iconst.i64 16@@ -40,22 +40,23 @@;; @002a v12 = uextend.i64 v11;; @002a v17 = icmp ugt v16, v12;; @002a trapnz v17, user17-;; @002a v26 = uload32 notrap aligned v0+56-;; @002a v27 = uextend.i64 v4-;; @002a v30 = iadd v27, v14-;; @002a v31 = icmp ugt v30, v26-;; @002a trapnz v31, heap_oob-;; @002a v33 = load.i64 notrap aligned v0+48-;; @002a v40 = load.i64 notrap aligned v57+40-;; v53 = iconst.i64 20-;; @002a v21 = iadd v8, v53 ; v53 = 20+;; @002a v26 = load.i32 notrap aligned v0+56+;; @002a v28 = uextend.i64 v4+;; @002a v31 = iadd v28, v14+;; @002a v27 = uextend.i64 v26+;; @002a v32 = icmp ugt v31, v27+;; @002a trapnz v32, heap_oob+;; @002a v34 = load.i64 notrap aligned v0+48+;; @002a v41 = load.i64 notrap aligned v58+40+;; v54 = iconst.i64 20+;; @002a v21 = iadd v8, v54 ; v54 = 20;; @002a v24 = iadd v21, v13-;; @002a v42 = uadd_overflow_trap v24, v14, user2-;; @002a v41 = iadd v7, v40-;; @002a v43 = icmp ugt v42, v41-;; @002a trapnz v43, user2-;; @002a v35 = iadd v33, v27-;; @002a call fn0(v0, v24, v35, v14)+;; @002a v43 = uadd_overflow_trap v24, v14, user2+;; @002a v42 = iadd v7, v41+;; @002a v44 = icmp ugt v43, v42+;; @002a trapnz v44, user2+;; @002a v36 = iadd v34, v28+;; @002a call fn0(v0, v24, v36, v14);; @002e jump block1;;;; block1:diff --git a/tests/disas/gc/array-new-data.wat b/tests/disas/gc/array-new-data.watindex 2ff85b8dfb..29e109b03e 100644--- a/tests/disas/gc/array-new-data.wat+++ b/tests/disas/gc/array-new-data.wat@@ -92,96 +92,98 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @0025 v6 = uload32 notrap aligned v0+56-;; @0025 v7 = uextend.i64 v2-;; @0025 v8 = uextend.i64 v3-;; @0025 v10 = iadd v7, v8-;; @0025 v11 = icmp ugt v10, v6-;; @0025 trapnz v11, heap_oob-;; @0025 v13 = load.i64 notrap aligned v0+48-;; v124 = iconst.i64 32-;; @0025 v19 = ushr v8, v124 ; v124 = 32-;; @0025 trapnz v19, user18-;; @0025 v16 = iconst.i32 20-;; @0025 v21 = uadd_overflow_trap v16, v3, user18 ; v16 = 20-;; @0025 v23 = load.i64 notrap aligned readonly can_move v0+32-;; @0025 v24 = load.i32 notrap aligned v23-;; @0025 v25 = load.i32 notrap aligned v23+4-;; @0025 v31 = uextend.i64 v24-;; @0025 v26 = uextend.i64 v21-;; @0025 v27 = iconst.i64 15-;; @0025 v29 = iadd v26, v27 ; v27 = 15-;; @0025 v28 = iconst.i64 -16-;; @0025 v30 = band v29, v28 ; v28 = -16-;; @0025 v32 = iadd v31, v30-;; @0025 v33 = uextend.i64 v25-;; @0025 v34 = icmp ule v32, v33-;; @0025 brif v34, block2, block3+;; @0025 v6 = load.i32 notrap aligned v0+56+;; @0025 v8 = uextend.i64 v2+;; @0025 v9 = uextend.i64 v3+;; @0025 v11 = iadd v8, v9+;; @0025 v7 = uextend.i64 v6+;; @0025 v12 = icmp ugt v11, v7+;; @0025 trapnz v12, heap_oob+;; @0025 v14 = load.i64 notrap aligned v0+48+;; v126 = iconst.i64 32+;; @0025 v20 = ushr v9, v126 ; v126 = 32+;; @0025 trapnz v20, user18+;; @0025 v17 = iconst.i32 20+;; @0025 v22 = uadd_overflow_trap v17, v3, user18 ; v17 = 20+;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+32+;; @0025 v25 = load.i32 notrap aligned v24+;; @0025 v26 = load.i32 notrap aligned v24+4+;; @0025 v32 = uextend.i64 v25+;; @0025 v27 = uextend.i64 v22+;; @0025 v28 = iconst.i64 15+;; @0025 v30 = iadd v27, v28 ; v28 = 15+;; @0025 v29 = iconst.i64 -16+;; @0025 v31 = band v30, v29 ; v29 = -16+;; @0025 v33 = iadd v32, v31+;; @0025 v34 = uextend.i64 v26+;; @0025 v35 = icmp ule v33, v34+;; @0025 brif v35, block2, block3;;;; block2:-;; v134 = iconst.i32 15-;; v135 = iadd.i32 v21, v134 ; v134 = 15-;; v138 = iconst.i32 -16-;; v139 = band v135, v138 ; v138 = -16-;; v141 = iadd.i32 v24, v139-;; @0025 store notrap aligned region0 v141, v23-;; v155 = iconst.i32 -1476395002-;; v156 = load.i64 notrap aligned readonly can_move v0+8-;; v157 = load.i64 notrap aligned readonly can_move v156+32-;; @0025 v48 = iadd v157, v31-;; @0025 store notrap aligned v155, v48 ; v155 = -1476395002-;; v158 = load.i64 notrap aligned readonly can_move v0+40-;; v159 = load.i32 notrap aligned readonly can_move v158-;; @0025 store notrap aligned v159, v48+4-;; v160 = band.i64 v29, v28 ; v28 = -16-;; @0025 istore32 notrap aligned v160, v48+8-;; @0025 jump block4(v24, v48)+;; v136 = iconst.i32 15+;; v137 = iadd.i32 v22, v136 ; v136 = 15+;; v140 = iconst.i32 -16+;; v141 = band v137, v140 ; v140 = -16+;; v143 = iadd.i32 v25, v141+;; @0025 store notrap aligned region0 v143, v24+;; v157 = iconst.i32 -1476395002+;; v158 = load.i64 notrap aligned readonly can_move v0+8+;; v159 = load.i64 notrap aligned readonly can_move v158+32+;; @0025 v49 = iadd v159, v32+;; @0025 store notrap aligned v157, v49 ; v157 = -1476395002+;; v160 = load.i64 notrap aligned readonly can_move v0+40+;; v161 = load.i32 notrap aligned readonly can_move v160+;; @0025 store notrap aligned v161, v49+4+;; v162 = band.i64 v30, v29 ; v29 = -16+;; @0025 istore32 notrap aligned v162, v49+8+;; @0025 jump block4(v25, v49);;;; block3 cold:-;; @0025 v36 = iconst.i32 -1476395002-;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+40-;; @0025 v39 = load.i32 notrap aligned readonly can_move v38-;; @0025 v40 = iconst.i32 16-;; @0025 v41 = call fn0(v0, v36, v39, v21, v40) ; v36 = -1476395002, v40 = 16-;; @0025 v120 = load.i64 notrap aligned readonly can_move v0+8-;; @0025 v42 = load.i64 notrap aligned readonly can_move v120+32-;; @0025 v43 = uextend.i64 v41-;; @0025 v44 = iadd v42, v43-;; @0025 jump block4(v41, v44)+;; @0025 v37 = iconst.i32 -1476395002+;; @0025 v39 = load.i64 notrap aligned readonly can_move v0+40+;; @0025 v40 = load.i32 notrap aligned readonly can_move v39+;; @0025 v41 = iconst.i32 16+;; @0025 v42 = call fn0(v0, v37, v40, v22, v41) ; v37 = -1476395002, v41 = 16+;; @0025 v122 = load.i64 notrap aligned readonly can_move v0+8+;; @0025 v43 = load.i64 notrap aligned readonly can_move v122+32+;; @0025 v44 = uextend.i64 v42+;; @0025 v45 = iadd v43, v44+;; @0025 jump block4(v42, v45);;-;; block4(v53: i32, v54: i64):-;; v119 = stack_addr.i64 ss0-;; store notrap v53, v119-;; v118 = iconst.i64 16-;; @0025 v55 = iadd v54, v118 ; v118 = 16-;; @0025 store.i32 user2 v3, v55-;; v99 = load.i32 notrap v119-;; @0025 trapz v99, user16-;; v161 = load.i64 notrap aligned readonly can_move v0+8-;; v162 = load.i64 notrap aligned readonly can_move v161+32-;; @0025 v57 = uextend.i64 v99-;; @0025 v59 = iadd v162, v57-;; @0025 v61 = iadd v59, v118 ; v118 = 16-;; @0025 v62 = load.i32 user2 readonly v61-;; @0025 v63 = uextend.i64 v62-;; @0025 v68 = icmp.i64 ugt v8, v63-;; @0025 trapnz v68, user17-;; @0025 v77 = uload32.i64 notrap aligned v0+56-;; @0025 v82 = icmp.i64 ugt v10, v77-;; @0025 trapnz v82, heap_oob-;; @0025 v84 = load.i64 notrap aligned v0+48-;; @0025 v91 = load.i64 notrap aligned v161+40-;; v109 = iconst.i64 20-;; @0025 v72 = iadd v59, v109 ; v109 = 20-;; @0025 v93 = uadd_overflow_trap v72, v8, user2-;; @0025 v92 = iadd v162, v91-;; @0025 v94 = icmp ugt v93, v92-;; @0025 trapnz v94, user2-;; @0025 v86 = iadd v84, v7-;; @0025 call fn1(v0, v72, v86, v8), stack_map=[i32 @ ss0+0]-;; v96 = load.i32 notrap v119+;; block4(v54: i32, v55: i64):+;; v121 = stack_addr.i64 ss0+;; store notrap v54, v121+;; v120 = iconst.i64 16+;; @0025 v56 = iadd v55, v120 ; v120 = 16+;; @0025 store.i32 user2 v3, v56+;; v101 = load.i32 notrap v121+;; @0025 trapz v101, user16+;; v163 = load.i64 notrap aligned readonly can_move v0+8+;; v164 = load.i64 notrap aligned readonly can_move v163+32+;; @0025 v58 = uextend.i64 v101+;; @0025 v60 = iadd v164, v58+;; @0025 v62 = iadd v60, v120 ; v120 = 16+;; @0025 v63 = load.i32 user2 readonly v62+;; @0025 v64 = uextend.i64 v63+;; @0025 v69 = icmp.i64 ugt v9, v64+;; @0025 trapnz v69, user17+;; @0025 v78 = load.i32 notrap aligned v0+56+;; @0025 v79 = uextend.i64 v78+;; @0025 v84 = icmp.i64 ugt v11, v79+;; @0025 trapnz v84, heap_oob+;; @0025 v86 = load.i64 notrap aligned v0+48+;; @0025 v93 = load.i64 notrap aligned v163+40+;; v111 = iconst.i64 20+;; @0025 v73 = iadd v60, v111 ; v111 = 20+;; @0025 v95 = uadd_overflow_trap v73, v9, user2+;; @0025 v94 = iadd v164, v93+;; @0025 v96 = icmp ugt v95, v94+;; @0025 trapnz v96, user2+;; @0025 v88 = iadd v86, v8+;; @0025 call fn1(v0, v73, v88, v9), stack_map=[i32 @ ss0+0]+;; v98 = load.i32 notrap v121;; @0029 jump block1;;;; block1:-;; @0029 return v96+;; @0029 return v98;; }diff --git a/tests/disas/gc/array-new-default-anyref.wat b/tests/disas/gc/array-new-default-anyref.watindex 82015ac769..c80b3f3925 100644--- a/tests/disas/gc/array-new-default-anyref.wat+++ b/tests/disas/gc/array-new-default-anyref.wat@@ -10,7 +10,6 @@));; function u0:0(i64 vmctx, i64, i32) -> i32 tail {-;; ss0 = explicit_slot 4, align = 4;; region0 = 2 "vmctx";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly gv0+8@@ -20,22 +19,20 @@;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail-;; sig1 = (i64 vmctx, i64, i32, i64) tail;; fn0 = colocated u805306368:24 sig0-;; fn1 = colocated u805306368:2 sig1;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):;; @001f v5 = uextend.i64 v2-;; v102 = iconst.i64 2-;; v103 = ishl v5, v102 ; v102 = 2-;; v100 = iconst.i64 32-;; @001f v7 = ushr v103, v100 ; v100 = 32+;; v98 = iconst.i64 2+;; v99 = ishl v5, v98 ; v98 = 2+;; v96 = iconst.i64 32+;; @001f v7 = ushr v99, v96 ; v96 = 32;; @001f trapnz v7, user18;; @001f v4 = iconst.i32 20-;; v109 = iconst.i32 2-;; v110 = ishl v2, v109 ; v109 = 2-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20+;; v105 = iconst.i32 2+;; v106 = ishl v2, v105 ; v105 = 2+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32;; @001f v12 = load.i32 notrap aligned v11;; @001f v13 = load.i32 notrap aligned v11+4@@ -51,22 +48,22 @@;; @001f brif v22, block2, block3;;;; block2:-;; v118 = iconst.i32 15-;; v119 = iadd.i32 v9, v118 ; v118 = 15-;; v122 = iconst.i32 -16-;; v123 = band v119, v122 ; v122 = -16-;; v125 = iadd.i32 v12, v123-;; @001f store notrap aligned region0 v125, v11-;; v141 = iconst.i32 -1476394994-;; v142 = load.i64 notrap aligned readonly can_move v0+8-;; v143 = load.i64 notrap aligned readonly can_move v142+32-;; @001f v36 = iadd v143, v19-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994-;; v144 = load.i64 notrap aligned readonly can_move v0+40-;; v145 = load.i32 notrap aligned readonly can_move v144-;; @001f store notrap aligned v145, v36+4-;; v146 = band.i64 v17, v16 ; v16 = -16-;; @001f istore32 notrap aligned v146, v36+8+;; v114 = iconst.i32 15+;; v115 = iadd.i32 v9, v114 ; v114 = 15+;; v118 = iconst.i32 -16+;; v119 = band v115, v118 ; v118 = -16+;; v121 = iadd.i32 v12, v119+;; @001f store notrap aligned region0 v121, v11+;; v137 = iconst.i32 -1476394994+;; v138 = load.i64 notrap aligned readonly can_move v0+8+;; v139 = load.i64 notrap aligned readonly can_move v138+32+;; @001f v36 = iadd v139, v19+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994+;; v140 = load.i64 notrap aligned readonly can_move v0+40+;; v141 = load.i32 notrap aligned readonly can_move v140+;; @001f store notrap aligned v141, v36+4+;; v142 = band.i64 v17, v16 ; v16 = -16+;; @001f istore32 notrap aligned v142, v36+8;; @001f jump block4(v12, v36);;;; block3 cold:@@ -75,41 +72,51 @@;; @001f v27 = load.i32 notrap aligned readonly can_move v26;; @001f v28 = iconst.i32 16;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32;; @001f v31 = uextend.i64 v29;; @001f v32 = iadd v30, v31;; @001f jump block4(v29, v32);;;; block4(v41: i32, v42: i64):-;; v95 = stack_addr.i64 ss0-;; store notrap v41, v95-;; v94 = iconst.i64 16-;; @001f v43 = iadd v42, v94 ; v94 = 16+;; v91 = iconst.i64 16+;; @001f v43 = iadd v42, v91 ; v91 = 16;; @001f store.i32 user2 v2, v43-;; v77 = load.i32 notrap v95-;; @001f trapz v77, user16-;; v147 = load.i64 notrap aligned readonly can_move v0+8-;; v148 = load.i64 notrap aligned readonly can_move v147+32-;; @001f v46 = uextend.i64 v77-;; @001f v48 = iadd v148, v46-;; @001f v50 = iadd v48, v94 ; v94 = 16+;; @001f trapz v41, user16+;; v143 = load.i64 notrap aligned readonly can_move v0+8+;; v144 = load.i64 notrap aligned readonly can_move v143+32+;; @001f v46 = uextend.i64 v41+;; @001f v48 = iadd v144, v46+;; @001f v50 = iadd v48, v91 ; v91 = 16;; @001f v51 = load.i32 user2 readonly v50;; @001f v52 = uextend.i64 v51;; @001f v57 = icmp.i64 ugt v5, v52;; @001f trapnz v57, user17-;; @001f v68 = load.i64 notrap aligned v147+40+;; @001f v68 = load.i64 notrap aligned v143+40;; v85 = iconst.i64 20;; @001f v61 = iadd v48, v85 ; v85 = 20-;; @001f v70 = uadd_overflow_trap v61, v103, user2-;; @001f v69 = iadd v148, v68+;; @001f v70 = uadd_overflow_trap v61, v99, user2+;; @001f v69 = iadd v144, v68;; @001f v71 = icmp ugt v70, v69;; @001f trapnz v71, user2+;; v123 = iconst.i64 0+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0;; @001f v44 = iconst.i32 0-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0-;; v74 = load.i32 notrap v95-;; @0022 jump block1+;; v97 = iconst.i64 4+;; @001f v72 = iadd v61, v99+;; @001f brif v73, block6, block5(v61);;-;; block1:-;; @0022 return v74+;; block5(v74: i64):+;; v145 = iconst.i32 0+;; @001f store user2 little v145, v74 ; v145 = 0+;; v146 = iconst.i64 4+;; v147 = iadd v74, v146 ; v146 = 4+;; @001f v76 = icmp eq v147, v72+;; @001f brif v76, block6, block5(v147)+;;+;; block6:+;; @0022 jump block1(v41)+;;+;; block1(v3: i32):+;; @0022 return v3;; }diff --git a/tests/disas/gc/array-new-default-exnref.wat b/tests/disas/gc/array-new-default-exnref.watindex 910d21579e..10e6a6d2a3 100644--- a/tests/disas/gc/array-new-default-exnref.wat+++ b/tests/disas/gc/array-new-default-exnref.wat@@ -10,7 +10,6 @@));; function u0:0(i64 vmctx, i64, i32) -> i32 tail {-;; ss0 = explicit_slot 4, align = 4;; region0 = 2 "vmctx";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly gv0+8@@ -20,22 +19,20 @@;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail-;; sig1 = (i64 vmctx, i64, i32, i64) tail;; fn0 = colocated u805306368:24 sig0-;; fn1 = colocated u805306368:2 sig1;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):;; @001f v5 = uextend.i64 v2-;; v102 = iconst.i64 2-;; v103 = ishl v5, v102 ; v102 = 2-;; v100 = iconst.i64 32-;; @001f v7 = ushr v103, v100 ; v100 = 32+;; v98 = iconst.i64 2+;; v99 = ishl v5, v98 ; v98 = 2+;; v96 = iconst.i64 32+;; @001f v7 = ushr v99, v96 ; v96 = 32;; @001f trapnz v7, user18;; @001f v4 = iconst.i32 20-;; v109 = iconst.i32 2-;; v110 = ishl v2, v109 ; v109 = 2-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20+;; v105 = iconst.i32 2+;; v106 = ishl v2, v105 ; v105 = 2+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32;; @001f v12 = load.i32 notrap aligned v11;; @001f v13 = load.i32 notrap aligned v11+4@@ -51,22 +48,22 @@;; @001f brif v22, block2, block3;;;; block2:-;; v118 = iconst.i32 15-;; v119 = iadd.i32 v9, v118 ; v118 = 15-;; v122 = iconst.i32 -16-;; v123 = band v119, v122 ; v122 = -16-;; v125 = iadd.i32 v12, v123-;; @001f store notrap aligned region0 v125, v11-;; v141 = iconst.i32 -1476394994-;; v142 = load.i64 notrap aligned readonly can_move v0+8-;; v143 = load.i64 notrap aligned readonly can_move v142+32-;; @001f v36 = iadd v143, v19-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994-;; v144 = load.i64 notrap aligned readonly can_move v0+40-;; v145 = load.i32 notrap aligned readonly can_move v144-;; @001f store notrap aligned v145, v36+4-;; v146 = band.i64 v17, v16 ; v16 = -16-;; @001f istore32 notrap aligned v146, v36+8+;; v114 = iconst.i32 15+;; v115 = iadd.i32 v9, v114 ; v114 = 15+;; v118 = iconst.i32 -16+;; v119 = band v115, v118 ; v118 = -16+;; v121 = iadd.i32 v12, v119+;; @001f store notrap aligned region0 v121, v11+;; v137 = iconst.i32 -1476394994+;; v138 = load.i64 notrap aligned readonly can_move v0+8+;; v139 = load.i64 notrap aligned readonly can_move v138+32+;; @001f v36 = iadd v139, v19+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994+;; v140 = load.i64 notrap aligned readonly can_move v0+40+;; v141 = load.i32 notrap aligned readonly can_move v140+;; @001f store notrap aligned v141, v36+4+;; v142 = band.i64 v17, v16 ; v16 = -16+;; @001f istore32 notrap aligned v142, v36+8;; @001f jump block4(v12, v36);;;; block3 cold:@@ -75,41 +72,51 @@;; @001f v27 = load.i32 notrap aligned readonly can_move v26;; @001f v28 = iconst.i32 16;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32;; @001f v31 = uextend.i64 v29;; @001f v32 = iadd v30, v31;; @001f jump block4(v29, v32);;;; block4(v41: i32, v42: i64):-;; v95 = stack_addr.i64 ss0-;; store notrap v41, v95-;; v94 = iconst.i64 16-;; @001f v43 = iadd v42, v94 ; v94 = 16+;; v91 = iconst.i64 16+;; @001f v43 = iadd v42, v91 ; v91 = 16;; @001f store.i32 user2 v2, v43-;; v77 = load.i32 notrap v95-;; @001f trapz v77, user16-;; v147 = load.i64 notrap aligned readonly can_move v0+8-;; v148 = load.i64 notrap aligned readonly can_move v147+32-;; @001f v46 = uextend.i64 v77-;; @001f v48 = iadd v148, v46-;; @001f v50 = iadd v48, v94 ; v94 = 16+;; @001f trapz v41, user16+;; v143 = load.i64 notrap aligned readonly can_move v0+8+;; v144 = load.i64 notrap aligned readonly can_move v143+32+;; @001f v46 = uextend.i64 v41+;; @001f v48 = iadd v144, v46+;; @001f v50 = iadd v48, v91 ; v91 = 16;; @001f v51 = load.i32 user2 readonly v50;; @001f v52 = uextend.i64 v51;; @001f v57 = icmp.i64 ugt v5, v52;; @001f trapnz v57, user17-;; @001f v68 = load.i64 notrap aligned v147+40+;; @001f v68 = load.i64 notrap aligned v143+40;; v85 = iconst.i64 20;; @001f v61 = iadd v48, v85 ; v85 = 20-;; @001f v70 = uadd_overflow_trap v61, v103, user2-;; @001f v69 = iadd v148, v68+;; @001f v70 = uadd_overflow_trap v61, v99, user2+;; @001f v69 = iadd v144, v68;; @001f v71 = icmp ugt v70, v69;; @001f trapnz v71, user2+;; v123 = iconst.i64 0+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0;; @001f v44 = iconst.i32 0-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0-;; v74 = load.i32 notrap v95-;; @0022 jump block1+;; v97 = iconst.i64 4+;; @001f v72 = iadd v61, v99+;; @001f brif v73, block6, block5(v61);;-;; block1:-;; @0022 return v74+;; block5(v74: i64):+;; v145 = iconst.i32 0+;; @001f store user2 little v145, v74 ; v145 = 0+;; v146 = iconst.i64 4+;; v147 = iadd v74, v146 ; v146 = 4+;; @001f v76 = icmp eq v147, v72+;; @001f brif v76, block6, block5(v147)+;;+;; block6:+;; @0022 jump block1(v41)+;;+;; block1(v3: i32):+;; @0022 return v3;; }diff --git a/tests/disas/gc/array-new-default-externref.wat b/tests/disas/gc/array-new-default-externref.watindex 4a0771b172..bb5b2a4e56 100644--- a/tests/disas/gc/array-new-default-externref.wat+++ b/tests/disas/gc/array-new-default-externref.wat@@ -10,7 +10,6 @@));; function u0:0(i64 vmctx, i64, i32) -> i32 tail {-;; ss0 = explicit_slot 4, align = 4;; region0 = 2 "vmctx";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly gv0+8@@ -20,22 +19,20 @@;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail-;; sig1 = (i64 vmctx, i64, i32, i64) tail;; fn0 = colocated u805306368:24 sig0-;; fn1 = colocated u805306368:2 sig1;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):;; @001f v5 = uextend.i64 v2-;; v102 = iconst.i64 2-;; v103 = ishl v5, v102 ; v102 = 2-;; v100 = iconst.i64 32-;; @001f v7 = ushr v103, v100 ; v100 = 32+;; v98 = iconst.i64 2+;; v99 = ishl v5, v98 ; v98 = 2+;; v96 = iconst.i64 32+;; @001f v7 = ushr v99, v96 ; v96 = 32;; @001f trapnz v7, user18;; @001f v4 = iconst.i32 20-;; v109 = iconst.i32 2-;; v110 = ishl v2, v109 ; v109 = 2-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20+;; v105 = iconst.i32 2+;; v106 = ishl v2, v105 ; v105 = 2+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32;; @001f v12 = load.i32 notrap aligned v11;; @001f v13 = load.i32 notrap aligned v11+4@@ -51,22 +48,22 @@;; @001f brif v22, block2, block3;;;; block2:-;; v118 = iconst.i32 15-;; v119 = iadd.i32 v9, v118 ; v118 = 15-;; v122 = iconst.i32 -16-;; v123 = band v119, v122 ; v122 = -16-;; v125 = iadd.i32 v12, v123-;; @001f store notrap aligned region0 v125, v11-;; v141 = iconst.i32 -1476394994-;; v142 = load.i64 notrap aligned readonly can_move v0+8-;; v143 = load.i64 notrap aligned readonly can_move v142+32-;; @001f v36 = iadd v143, v19-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994-;; v144 = load.i64 notrap aligned readonly can_move v0+40-;; v145 = load.i32 notrap aligned readonly can_move v144-;; @001f store notrap aligned v145, v36+4-;; v146 = band.i64 v17, v16 ; v16 = -16-;; @001f istore32 notrap aligned v146, v36+8+;; v114 = iconst.i32 15+;; v115 = iadd.i32 v9, v114 ; v114 = 15+;; v118 = iconst.i32 -16+;; v119 = band v115, v118 ; v118 = -16+;; v121 = iadd.i32 v12, v119+;; @001f store notrap aligned region0 v121, v11+;; v137 = iconst.i32 -1476394994+;; v138 = load.i64 notrap aligned readonly can_move v0+8+;; v139 = load.i64 notrap aligned readonly can_move v138+32+;; @001f v36 = iadd v139, v19+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994+;; v140 = load.i64 notrap aligned readonly can_move v0+40+;; v141 = load.i32 notrap aligned readonly can_move v140+;; @001f store notrap aligned v141, v36+4+;; v142 = band.i64 v17, v16 ; v16 = -16+;; @001f istore32 notrap aligned v142, v36+8;; @001f jump block4(v12, v36);;;; block3 cold:@@ -75,41 +72,51 @@;; @001f v27 = load.i32 notrap aligned readonly can_move v26;; @001f v28 = iconst.i32 16;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32;; @001f v31 = uextend.i64 v29;; @001f v32 = iadd v30, v31;; @001f jump block4(v29, v32);;;; block4(v41: i32, v42: i64):-;; v95 = stack_addr.i64 ss0-;; store notrap v41, v95-;; v94 = iconst.i64 16-;; @001f v43 = iadd v42, v94 ; v94 = 16+;; v91 = iconst.i64 16+;; @001f v43 = iadd v42, v91 ; v91 = 16;; @001f store.i32 user2 v2, v43-;; v77 = load.i32 notrap v95-;; @001f trapz v77, user16-;; v147 = load.i64 notrap aligned readonly can_move v0+8-;; v148 = load.i64 notrap aligned readonly can_move v147+32-;; @001f v46 = uextend.i64 v77-;; @001f v48 = iadd v148, v46-;; @001f v50 = iadd v48, v94 ; v94 = 16+;; @001f trapz v41, user16+;; v143 = load.i64 notrap aligned readonly can_move v0+8+;; v144 = load.i64 notrap aligned readonly can_move v143+32+;; @001f v46 = uextend.i64 v41+;; @001f v48 = iadd v144, v46+;; @001f v50 = iadd v48, v91 ; v91 = 16;; @001f v51 = load.i32 user2 readonly v50;; @001f v52 = uextend.i64 v51;; @001f v57 = icmp.i64 ugt v5, v52;; @001f trapnz v57, user17-;; @001f v68 = load.i64 notrap aligned v147+40+;; @001f v68 = load.i64 notrap aligned v143+40;; v85 = iconst.i64 20;; @001f v61 = iadd v48, v85 ; v85 = 20-;; @001f v70 = uadd_overflow_trap v61, v103, user2-;; @001f v69 = iadd v148, v68+;; @001f v70 = uadd_overflow_trap v61, v99, user2+;; @001f v69 = iadd v144, v68;; @001f v71 = icmp ugt v70, v69;; @001f trapnz v71, user2+;; v123 = iconst.i64 0+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0;; @001f v44 = iconst.i32 0-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0-;; v74 = load.i32 notrap v95-;; @0022 jump block1+;; v97 = iconst.i64 4+;; @001f v72 = iadd v61, v99+;; @001f brif v73, block6, block5(v61);;-;; block1:-;; @0022 return v74+;; block5(v74: i64):+;; v145 = iconst.i32 0+;; @001f store user2 little v145, v74 ; v145 = 0+;; v146 = iconst.i64 4+;; v147 = iadd v74, v146 ; v146 = 4+;; @001f v76 = icmp eq v147, v72+;; @001f brif v76, block6, block5(v147)+;;+;; block6:+;; @0022 jump block1(v41)+;;+;; block1(v3: i32):+;; @0022 return v3;; }diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.watindex baa936d1dd..45901103e3 100644--- a/tests/disas/gc/array-new-default-funcref.wat+++ b/tests/disas/gc/array-new-default-funcref.wat@@ -20,22 +20,22 @@;; gv5 = load.i64 notrap aligned readonly can_move gv4+32;; gv6 = load.i64 notrap aligned gv4+40;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail-;; sig1 = (i64 vmctx, i64, i32, i64) tail+;; sig1 = (i64 vmctx, i64) -> i64 tail;; fn0 = colocated u805306368:24 sig0-;; fn1 = colocated u805306368:2 sig1+;; fn1 = colocated u805306368:25 sig1;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):;; @001f v5 = uextend.i64 v2-;; v102 = iconst.i64 2-;; v103 = ishl v5, v102 ; v102 = 2-;; v100 = iconst.i64 32-;; @001f v7 = ushr v103, v100 ; v100 = 32+;; v110 = iconst.i64 2+;; v111 = ishl v5, v110 ; v110 = 2+;; v108 = iconst.i64 32+;; @001f v7 = ushr v111, v108 ; v108 = 32;; @001f trapnz v7, user18;; @001f v4 = iconst.i32 20-;; v109 = iconst.i32 2-;; v110 = ishl v2, v109 ; v109 = 2-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20+;; v117 = iconst.i32 2+;; v118 = ishl v2, v117 ; v117 = 2+;; @001f v9 = uadd_overflow_trap v4, v118, user18 ; v4 = 20;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32;; @001f v12 = load.i32 notrap aligned v11;; @001f v13 = load.i32 notrap aligned v11+4@@ -51,22 +51,22 @@;; @001f brif v22, block2, block3;;;; block2:-;; v118 = iconst.i32 15-;; v119 = iadd.i32 v9, v118 ; v118 = 15-;; v122 = iconst.i32 -16-;; v123 = band v119, v122 ; v122 = -16-;; v125 = iadd.i32 v12, v123-;; @001f store notrap aligned region0 v125, v11-;; v140 = iconst.i32 -1476395002-;; v141 = load.i64 notrap aligned readonly can_move v0+8-;; v142 = load.i64 notrap aligned readonly can_move v141+32-;; @001f v36 = iadd v142, v19-;; @001f store notrap aligned v140, v36 ; v140 = -1476395002-;; v143 = load.i64 notrap aligned readonly can_move v0+40-;; v144 = load.i32 notrap aligned readonly can_move v143-;; @001f store notrap aligned v144, v36+4-;; v145 = band.i64 v17, v16 ; v16 = -16-;; @001f istore32 notrap aligned v145, v36+8+;; v126 = iconst.i32 15+;; v127 = iadd.i32 v9, v126 ; v126 = 15+;; v130 = iconst.i32 -16+;; v131 = band v127, v130 ; v130 = -16+;; v133 = iadd.i32 v12, v131+;; @001f store notrap aligned region0 v133, v11+;; v148 = iconst.i32 -1476395002+;; v149 = load.i64 notrap aligned readonly can_move v0+8+;; v150 = load.i64 notrap aligned readonly can_move v149+32+;; @001f v36 = iadd v150, v19+;; @001f store notrap aligned v148, v36 ; v148 = -1476395002+;; v151 = load.i64 notrap aligned readonly can_move v0+40+;; v152 = load.i32 notrap aligned readonly can_move v151+;; @001f store notrap aligned v152, v36+4+;; v153 = band.i64 v17, v16 ; v16 = -16+;; @001f istore32 notrap aligned v153, v36+8;; @001f jump block4(v12, v36);;;; block3 cold:@@ -75,41 +75,56 @@;; @001f v27 = load.i32 notrap aligned readonly can_move v26;; @001f v28 = iconst.i32 16;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476395002, v28 = 16-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32+;; @001f v104 = load.i64 notrap aligned readonly can_move v0+8+;; @001f v30 = load.i64 notrap aligned readonly can_move v104+32;; @001f v31 = uextend.i64 v29;; @001f v32 = iadd v30, v31;; @001f jump block4(v29, v32);;;; block4(v41: i32, v42: i64):-;; v95 = stack_addr.i64 ss0-;; store notrap v41, v95-;; v94 = iconst.i64 16-;; @001f v43 = iadd v42, v94 ; v94 = 16+;; v103 = stack_addr.i64 ss0+;; store notrap v41, v103+;; v102 = iconst.i64 16+;; @001f v43 = iadd v42, v102 ; v102 = 16;; @001f store.i32 user2 v2, v43-;; v77 = load.i32 notrap v95-;; @001f trapz v77, user16-;; v146 = load.i64 notrap aligned readonly can_move v0+8-;; v147 = load.i64 notrap aligned readonly can_move v146+32-;; @001f v46 = uextend.i64 v77-;; @001f v48 = iadd v147, v46-;; @001f v50 = iadd v48, v94 ; v94 = 16+;; v83 = load.i32 notrap v103+;; @001f trapz v83, user16+;; v154 = load.i64 notrap aligned readonly can_move v0+8+;; v155 = load.i64 notrap aligned readonly can_move v154+32+;; @001f v46 = uextend.i64 v83+;; @001f v48 = iadd v155, v46+;; @001f v50 = iadd v48, v102 ; v102 = 16;; @001f v51 = load.i32 user2 readonly v50;; @001f v52 = uextend.i64 v51;; @001f v57 = icmp.i64 ugt v5, v52;; @001f trapnz v57, user17-;; @001f v68 = load.i64 notrap aligned v146+40-;; v85 = iconst.i64 20-;; @001f v61 = iadd v48, v85 ; v85 = 20-;; @001f v70 = uadd_overflow_trap v61, v103, user2-;; @001f v69 = iadd v147, v68+;; @001f v68 = load.i64 notrap aligned v154+40+;; v93 = iconst.i64 20+;; @001f v61 = iadd v48, v93 ; v93 = 20+;; @001f v70 = uadd_overflow_trap v61, v111, user2+;; @001f v69 = iadd v155, v68;; @001f v71 = icmp ugt v70, v69;; @001f trapnz v71, user2-;; @001f v45 = iconst.i32 0-;; @001f call fn1(v0, v61, v45, v103), stack_map=[i32 @ ss0+0] ; v45 = 0-;; v74 = load.i32 notrap v95+;; @001f v44 = iconst.i64 0+;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0+;; v109 = iconst.i64 4+;; @001f v72 = iadd v61, v111+;; @001f brif v73, block6, block5(v61)+;;+;; block5(v74: i64):+;; v156 = iconst.i64 0+;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0+;; @001f v77 = ireduce.i32 v76+;; @001f store user2 little v77, v74+;; v157 = iconst.i64 4+;; v158 = iadd v74, v157 ; v157 = 4+;; @001f v79 = icmp eq v158, v72+;; @001f brif v79, block6, block5(v158)+;;+;; block6:+;; v80 = load.i32 notrap v103;; @0022 jump block1;;;; block1:-;; @0022 return v74+;; @0022 return v80;; }diff --git a/tests/disas/passive-data.wat b/tests/disas/passive-data.watindex 0fbe3b45fc..0d8605e9c4 100644--- a/tests/disas/passive-data.wat+++ b/tests/disas/passive-data.wat@@ -28,29 +28,30 @@;; @003d v6 = load.i64 notrap aligned v0+64;; @003d v7 = uextend.i64 v2;; @003d v8 = uextend.i64 v4-;; v32 = iconst.i64 1-;; @003d v9 = imul v8, v32 ; v32 = 1+;; v33 = iconst.i64 1+;; @003d v9 = imul v8, v33 ; v33 = 1;; @003d v10 = iadd v7, v9;; @003d v11 = icmp ugt v10, v6;; @003d trapnz v11, heap_oob;; @003d v12 = load.i64 notrap aligned readonly can_move v0+56;; @003d v13 = uextend.i64 v2-;; v30 = iconst.i64 1-;; @003d v14 = imul v13, v30 ; v30 = 1+;; v31 = iconst.i64 1+;; @003d v14 = imul v13, v31 ; v31 = 1;; @003d v15 = iadd v12, v14-;; @003d v17 = uload32 notrap aligned v0+152-;; @003d v18 = uextend.i64 v3-;; @003d v19 = uextend.i64 v4-;; v29 = iconst.i64 1-;; @003d v20 = imul v19, v29 ; v29 = 1-;; @003d v21 = iadd v18, v20-;; @003d v22 = icmp ugt v21, v17-;; @003d trapnz v22, heap_oob-;; @003d v24 = load.i64 notrap aligned v0+144-;; @003d v25 = uextend.i64 v3-;; @003d v26 = iadd v24, v25-;; @003d v27 = uextend.i64 v4-;; @003d call fn0(v0, v15, v26, v27)+;; @003d v17 = load.i32 notrap aligned v0+152+;; @003d v18 = uextend.i64 v17+;; @003d v19 = uextend.i64 v3+;; @003d v20 = uextend.i64 v4+;; v30 = iconst.i64 1+;; @003d v21 = imul v20, v30 ; v30 = 1+;; @003d v22 = iadd v19, v21+;; @003d v23 = icmp ugt v22, v18+;; @003d trapnz v23, heap_oob+;; @003d v25 = load.i64 notrap aligned v0+144+;; @003d v26 = uextend.i64 v3+;; @003d v27 = iadd v25, v26+;; @003d v28 = uextend.i64 v4+;; @003d call fn0(v0, v15, v27, v28);; @0041 jump block1;;;; block1:diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.watnew file mode 100644index 0000000000..95e2e5b187--- /dev/null+++ b/tests/disas/startup-data-active.wat@@ -0,0 +1,62 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'+;;! flags = '-Wgc -Wfunction-references'++(module+ (memory 1)++ (data (i32.const 1) "hi")+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned gv0+64+;; gv2 = load.i64 notrap aligned readonly can_move gv0+56+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:1 sig0+;;+;; block0(v0: i64, v1: i64):+;; v3 = load.i64 notrap aligned v0+112+;; v4 = iconst.i64 0+;; v5 = icmp eq v3, v4 ; v4 = 0+;; brif v5, block2, block1+;;+;; block1:+;; v8 = load.i32 notrap aligned v0+120+;; v11 = load.i64 notrap aligned v0+64+;; v13 = uextend.i64 v8+;; v16 = icmp ugt v13, v11+;; trapnz v16, heap_oob+;; v17 = load.i64 notrap aligned readonly can_move v0+56+;; call fn0(v0, v17, v3, v13)+;; jump block2+;;+;; block2:+;; return+;; }diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.watnew file mode 100644index 0000000000..10e6af2848--- /dev/null+++ b/tests/disas/startup-elem-active.wat@@ -0,0 +1,77 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'+;;! flags = '-Wgc -Wfunction-references'++(module+ (table 10 anyref)++ (elem (i32.const 1) (ref i31)+ (item (ref.i31 (i32.const 10)))+ (item (ref.i31 (i32.const 11)))+ (item (ref.i31 (i32.const 12)))+ )+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; region0 = 1 "table"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned gv0+48+;; gv2 = load.i64 notrap aligned gv0+56+;;+;; block0(v0: i64, v1: i64):+;; v4 = load.i64 notrap aligned v0+56+;; v5 = ireduce.i32 v4+;; v6 = uextend.i64 v5+;; v86 = iconst.i64 4+;; v92 = icmp ult v6, v86 ; v86 = 4+;; trapnz v92, user6+;; v12 = load.i64 notrap aligned v0+48+;; v103 = iconst.i32 21+;; v2 = iconst.i32 1+;; v114 = icmp ule v5, v2 ; v2 = 1+;; v79 = iconst.i64 0+;; v15 = iadd v12, v86 ; v86 = 4+;; v28 = select_spectre_guard v114, v79, v15 ; v79 = 0+;; store user6 aligned region0 v103, v28 ; v103 = 21+;; v117 = iconst.i32 23+;; v123 = iconst.i32 2+;; v129 = icmp ule v5, v123 ; v123 = 2+;; v131 = iconst.i64 8+;; v39 = iadd v12, v131 ; v131 = 8+;; v41 = select_spectre_guard v129, v79, v39 ; v79 = 0+;; store user6 aligned region0 v117, v41 ; v117 = 23+;; v133 = iconst.i32 25+;; v3 = iconst.i32 3+;; v144 = icmp ule v5, v3 ; v3 = 3+;; v146 = iconst.i64 12+;; v52 = iadd v12, v146 ; v146 = 12+;; v54 = select_spectre_guard v144, v79, v52 ; v79 = 0+;; store user6 aligned region0 v133, v54 ; v133 = 25+;; return+;; }diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.watnew file mode 100644index 0000000000..49da0905df--- /dev/null+++ b/tests/disas/startup-global.wat@@ -0,0 +1,42 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'++(module+ (global i64 i64.const 0 i64.const 0 i64.add)+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; region0 = 1 "table"+;; gv0 = vmctx+;;+;; block0(v0: i64, v1: i64):+;; v2 = iconst.i64 0+;; store notrap aligned region0 v2, v0+48 ; v2 = 0+;; return+;; }diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.watnew file mode 100644index 0000000000..eaec68a403--- /dev/null+++ b/tests/disas/startup-passive-segment.wat@@ -0,0 +1,50 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'+;;! flags = '-Wgc'++(module+ (elem (ref i31) (item (ref.i31 (i32.const 0))) (item (ref.i31 (i32.const 1))))+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; gv0 = vmctx+;; sig0 = (i64 vmctx, i32) -> i64 tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i64, v1: i64):+;; v3 = iconst.i32 0+;; v4 = call fn0(v0, v3) ; v3 = 0+;; v18 = iconst.i32 1+;; store user2 little v18, v4 ; v18 = 1+;; v25 = iconst.i32 3+;; v13 = iconst.i64 16+;; v12 = iadd v4, v13 ; v13 = 16+;; store user2 little v25, v12 ; v25 = 3+;; return+;; }diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.watnew file mode 100644index 0000000000..aac06f2446--- /dev/null+++ b/tests/disas/startup-start.wat@@ -0,0 +1,42 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'++(module+ (func $f)+ (start $f)+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u0:0 sig0+;;+;; block0(v0: i64, v1: i64):+;; call fn0(v0, v0)+;; return+;; }diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.watnew file mode 100644index 0000000000..657a42ecbb--- /dev/null+++ b/tests/disas/startup-table-initial-value.wat@@ -0,0 +1,65 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! filter = 'module_start'+;;! flags = '-Wgc -Wfunction-references'++(module+ (table 10 (ref i31) (ref.i31 (i32.const 0)))+)+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; sig0 = (i64 vmctx, i64) tail+;; fn0 = colocated u2415919104:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i64 notrap aligned v0+8+;; v5 = get_frame_pointer.i64+;; store notrap aligned v5, v4+72+;; v6 = get_stack_pointer.i64+;; store notrap aligned v6, v4+64+;; v7 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned v7, v4+80+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]+;;+;; block2:+;; v8 = iconst.i8 1+;; return v8 ; v8 = 1+;;+;; block3:+;; v9 = iconst.i8 0+;; return v9 ; v9 = 0+;; }+;;+;; function u2415919104:0(i64 vmctx, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned gv0+48+;; gv2 = load.i64 notrap aligned gv0+56+;;+;; block0(v0: i64, v1: i64):+;; v7 = load.i64 notrap aligned v0+56+;; v8 = ireduce.i32 v7+;; v9 = uextend.i64 v8+;; v41 = iconst.i64 10+;; v53 = icmp ult v9, v41 ; v41 = 10+;; trapnz v53, user6+;; v15 = load.i64 notrap aligned v0+48+;; v34 = iconst.i32 1+;; v83 = iconst.i64 36+;; v85 = iadd v15, v83 ; v83 = 36+;; v29 = iconst.i64 4+;; jump block1(v15)+;;+;; block1(v23: i64):+;; v88 = iconst.i32 1+;; store notrap aligned v88, v23 ; v88 = 1+;; v89 = iadd.i64 v15, v83 ; v83 = 36+;; v90 = icmp eq v23, v89+;; v91 = iconst.i64 4+;; v92 = iadd v23, v91 ; v91 = 4+;; brif v90, block2, block1(v92)+;;+;; block2:+;; return+;; }diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.watindex ae2e8eaf43..e9ccc89ad7 100644--- a/tests/disas/winch/x64/table/init_copy_drop.wat+++ b/tests/disas/winch/x64/table/init_copy_drop.wat@@ -303,7 +303,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x114d+;; callq 0x1178;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -388,7 +388,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x114d+;; callq 0x1178;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -473,7 +473,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x114d+;; callq 0x1178;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -558,7 +558,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x114d+;; callq 0x1178;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -643,7 +643,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x114d+;; callq 0x1178;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -754,7 +754,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 0xc(%rsp), %edx-;; callq 0x114d+;; callq 0x1178;; addq $0xc, %rsp;; addq $4, %rsp;; movq 0x18(%rsp), %r14diff --git a/tests/misc_testsuite/gc/array-fill-null-funcref.wast b/tests/misc_testsuite/gc/array-fill-null-funcref.wastnew file mode 100644index 0000000000..2e74c3384d--- /dev/null+++ b/tests/misc_testsuite/gc/array-fill-null-funcref.wast@@ -0,0 +1,10 @@+;;! gc = true++(module+ (type $a (array funcref))+ (func (export "hi") (result funcref)+ (array.get $a (array.new_default $a (i32.const 10)) (i32.const 3))+ )+)++(assert_return (invoke "hi") (ref.null func))diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex be5eaefc49..ffa64484d0 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -1722,7 +1722,7 @@ whereself.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?;self.context.free_reg(dst);- let passive_data_index = match self.env.translation.passive_data_map[segment] {+ let runtime_data_index = match self.env.translation.runtime_data_map[segment] {Some(i) => i,// Active data segments always have length zero, so this is only@@ -1747,7 +1747,7 @@ wherelet data_segment_length_offset = self.env.vmoffsets- .vmctx_passive_data_length(passive_data_index);+ .vmctx_runtime_data_length(runtime_data_index);let tmp1 = self.context.any_gpr(self.masm)?;let tmp2 = self.context.any_gpr(self.masm)?;self.masm.load(@@ -1774,7 +1774,7 @@ wherelet data_segment_base_offset = self.env.vmoffsets- .vmctx_passive_data_base(passive_data_index);+ .vmctx_runtime_data_base(runtime_data_index);self.masm.load(self.masm.address_at_vmctx(data_segment_base_offset)?,writable!(tmp1),@@ -1804,7 +1804,7 @@ where}pub fn emit_data_drop(&mut self, data_index: DataIndex) -> Result<()> {- let passive_data_index = match self.env.translation.passive_data_map[data_index] {+ let runtime_data_index = match self.env.translation.runtime_data_map[data_index] {Some(idx) => idx,// Active data segments do nothing when dropped, so this is a noop.None => return Ok(()),@@ -1812,7 +1812,7 @@ wherelet data_segment_offset = self.env.vmoffsets- .vmctx_passive_data_length(passive_data_index);+ .vmctx_runtime_data_length(runtime_data_index);let len_addr = self.masm.address_at_vmctx(data_segment_offset)?;self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32)}