Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Move most module initialization to compiled code (#13487)

Alex Crichton5f3b67ea0558
diff --git a/crates/core/src/slab.rs b/crates/core/src/slab.rs
index 38a483a839..030d6294a2 100644
--- a/crates/core/src/slab.rs
+++ b/crates/core/src/slab.rs
@@ -390,15 +390,11 @@ impl<T> Slab<T> {
///
/// Returns `None` if the value has since been deallocated.
///
- /// If `id` comes from a different `Slab` instance, this method may panic,
- /// return `None`, or return an arbitrary value.
+ /// If `id` comes from a different `Slab` instance, this method may return
+ /// `None`, or return an arbitrary value.
#[inline]
pub fn get(&self, id: Id) -> Option<&T> {
- match self
- .entries
- .get(id.0.index())
- .expect("id from different slab")
- {
+ match self.entries.get(id.0.index())? {
Entry::Occupied(x) => Some(x),
Entry::Free { .. } => None,
}
@@ -408,15 +404,11 @@ impl<T> Slab<T> {
///
/// Returns `None` if the value has since been deallocated.
///
- /// If `id` comes from a different `Slab` instance, this method may panic,
- /// return `None`, or return an arbitrary value.
+ /// If `id` comes from a different `Slab` instance, this method may return
+ /// `None`, or return an arbitrary value.
#[inline]
pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {
- match self
- .entries
- .get_mut(id.0.index())
- .expect("id from different slab")
- {
+ match self.entries.get_mut(id.0.index())? {
Entry::Occupied(x) => Some(x),
Entry::Free { .. } => None,
}
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index a2c76c9f8a..5bc5005503 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -39,9 +39,9 @@ use wasmtime_environ::{
Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,
DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,
FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, HostCall, Inlining,
- InliningCompiler, ModulePC, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection,
- StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType,
- WasmValType, prelude::*,
+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,
+ StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,
+ WasmFuncType, WasmValType, prelude::*,
};
use wasmtime_unwinder::ExceptionTableBuilder;
@@ -199,6 +199,230 @@ impl Compiler {
builder.ins().call_indirect(sig, addr, args)
}
+
+ fn compile_wasm_to_array_trampoline(
+ &self,
+ wasm_func_ty: &WasmFuncType,
+ key: FuncKey,
+ symbol: &str,
+ ) -> Result<CompiledFunctionBody, CompileError> {
+ log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}");
+
+ let isa = &*self.isa;
+ let pointer_type = isa.pointer_type();
+ let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables);
+ let array_call_sig = array_call_signature(isa);
+
+ let mut compiler = self.function_compiler();
+ let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig);
+ let (mut builder, block0) = compiler.builder(func);
+
+ let args = builder.func.dfg.block_params(block0).to_vec();
+ let callee_vmctx = args[0];
+ let caller_vmctx = args[1];
+
+ // We are exiting Wasm, so save our PC and FP.
+ //
+ // Assert that the caller vmctx really is a core Wasm vmctx, since
+ // that's what we are assuming with our offsets below.
+ self.debug_assert_vmctx_kind(
+ &mut builder,
+ caller_vmctx,
+ wasmtime_environ::VMCONTEXT_MAGIC,
+ );
+ let ptr = isa.pointer_bytes();
+ let vm_store_context = builder.ins().load(
+ pointer_type,
+ MemFlagsData::trusted(),
+ caller_vmctx,
+ i32::from(ptr.vmcontext_store_context()),
+ );
+ save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context);
+
+ // Spill all wasm arguments to the stack in `ValRaw` slots.
+ let (args_base, args_len) =
+ self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);
+ let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));
+
+ // Load the actual callee out of the
+ // `VMArrayCallHostFuncContext::host_func`.
+ let ptr_size = isa.pointer_bytes();
+ let callee = builder.ins().load(
+ pointer_type,
+ MemFlagsData::trusted(),
+ callee_vmctx,
+ ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),
+ );
+
+ // Do an indirect call to the callee.
+ let callee_signature = builder.func.import_signature(array_call_sig);
+ let call = self.call_indirect_host(
+ &mut builder,
+ HostCall::ArrayCall,
+ callee_signature,
+ callee,
+ &[callee_vmctx, caller_vmctx, args_base, args_len],
+ );
+
+ // Increment the "execution version" on the VMStoreContext if
+ // guest debugging is enabled.
+ if self.tunables.debug_guest {
+ let vmstore_ctx_ptr = builder.ins().load(
+ pointer_type,
+ MemFlagsData::trusted().with_readonly(),
+ caller_vmctx,
+ i32::from(ptr_size.vmctx_store_context()),
+ );
+ let old_version = builder.ins().load(
+ ir::types::I64,
+ MemFlagsData::trusted(),
+ vmstore_ctx_ptr,
+ i32::from(ptr_size.vmstore_context_execution_version()),
+ );
+ let new_version = builder.ins().iadd_imm(old_version, 1);
+ builder.ins().store(
+ MemFlagsData::trusted(),
+ new_version,
+ vmstore_ctx_ptr,
+ i32::from(ptr_size.vmstore_context_execution_version()),
+ );
+ }
+
+ // Invoke `raise` if the callee (host) returned an error.
+ let succeeded = builder.func.dfg.inst_results(call)[0];
+ self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded);
+
+ // Return results from the array as native return values.
+ let results =
+ self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);
+ builder.ins().return_(&results);
+ builder.finalize();
+
+ Ok(CompiledFunctionBody {
+ code: box_dyn_any_compiler_context(Some(compiler.cx)),
+ needs_gc_heap: false,
+ })
+ }
+
+ fn compile_wasm_to_builtin(
+ &self,
+ key: FuncKey,
+ symbol: &str,
+ ) -> Result<CompiledFunctionBody, CompileError> {
+ log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}");
+
+ let isa = &*self.isa;
+ let ptr_size = isa.pointer_bytes();
+ let pointer_type = isa.pointer_type();
+ let sigs = BuiltinFunctionSignatures::new(self);
+
+ let (builtin_func_index, wasm_sig) = match key {
+ FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)),
+ FuncKey::PatchableToBuiltinTrampoline(builtin) => {
+ let mut sig = sigs.wasm_signature(builtin);
+ // Patchable functions cannot return anything. We
+ // raise any errors that occur below so this is fine.
+ sig.returns.clear();
+ sig.call_conv = CallConv::PreserveAll;
+ (builtin, sig)
+ }
+ _ => unreachable!(),
+ };
+ let host_sig = sigs.host_signature(builtin_func_index);
+
+ let mut compiler = self.function_compiler();
+ let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone());
+ let (mut builder, block0) = compiler.builder(func);
+ let vmctx = builder.block_params(block0)[0];
+
+ // Debug-assert that this is the right kind of vmctx, and then
+ // additionally perform the "routine of the exit trampoline" of saving
+ // fp/pc/etc.
+ self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC);
+ let vm_store_context = builder.ins().load(
+ pointer_type,
+ MemFlagsData::trusted(),
+ vmctx,
+ ptr_size.vmcontext_store_context(),
+ );
+ save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context);
+
+ // Now it's time to delegate to the actual builtin. Forward all our own
+ // arguments to the libcall itself.
+ let args = builder.block_params(block0).to_vec();
+ let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig);
+ let results = builder.func.dfg.inst_results(call).to_vec();
+
+ // Libcalls do not explicitly jump/raise on traps but instead return a
+ // code indicating whether they trapped or not. This means that it's the
+ // responsibility of the trampoline to check for an trapping return
+ // value and raise a trap as appropriate. With the `results` above check
+ // what `index` is and for each libcall that has a trapping return value
+ // process it here.
+ match builtin_func_index.trap_sentinel() {
+ Some(TrapSentinel::Falsy) => {
+ self.raise_if_host_trapped(&mut builder, vmctx, results[0]);
+ }
+ Some(TrapSentinel::NegativeTwo) => {
+ let ty = builder.func.dfg.value_type(results[0]);
+ let trapped = builder.ins().iconst(ty, -2);
+ let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped);
+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+ }
+ Some(TrapSentinel::Negative) => {
+ let ty = builder.func.dfg.value_type(results[0]);
+ let zero = builder.ins().iconst(ty, 0);
+ let succeeded =
+ builder
+ .ins()
+ .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero);
+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+ }
+ Some(TrapSentinel::NegativeOne) => {
+ let ty = builder.func.dfg.value_type(results[0]);
+ let minus_one = builder.ins().iconst(ty, -1);
+ let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one);
+ self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+ }
+ None => {}
+ }
+
+ // And finally, return all the results of this libcall.
+ if !wasm_sig.returns.is_empty() {
+ builder.ins().return_(&results);
+ } else {
+ builder.ins().return_(&[]);
+ }
+ builder.finalize();
+
+ Ok(CompiledFunctionBody {
+ code: box_dyn_any_compiler_context(Some(compiler.cx)),
+ needs_gc_heap: false,
+ })
+ }
+
+ fn compile_module_startup(
+ &self,
+ translation: &ModuleTranslation<'_>,
+ types: &ModuleTypesBuilder,
+ key: FuncKey,
+ ty: &WasmFuncType,
+ ) -> Result<CompiledFunctionBody, CompileError> {
+ let mut compiler = self.function_compiler();
+ let context = &mut compiler.cx.codegen_context;
+ context.func.signature = wasm_call_signature(&*self.isa, ty, &self.tunables);
+ let (namespace, index) = key.into_raw_parts();
+ context.func.name = UserFuncName::User(UserExternalName { namespace, index });
+ let mut func_env = FuncEnvironment::new(self, translation, types, ty, key);
+ compiler
+ .cx
+ .func_translator
+ .translate_module_startup(&mut context.func, &mut func_env)?;
+ Ok(CompiledFunctionBody {
+ code: box_dyn_any_compiler_context(Some(compiler.cx)),
+ needs_gc_heap: func_env.needs_gc_heap(),
+ })
+ }
}
fn box_dyn_any_compiled_function(f: CompiledFunction) -> Box<dyn Any + Send + Sync> {
@@ -364,130 +588,82 @@ impl wasmtime_environ::Compiler for Compiler {
})
}
- fn compile_array_to_wasm_trampoline(
+ fn compile_trampoline(
&self,
- translation: &ModuleTranslation<'_>,
- types: &ModuleTypesBuilder,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- let (module_index, def_func_index) = key.unwrap_array_to_wasm_trampoline();
- let func_index = translation.module.func_index(def_func_index);
- let sig = translation.module.functions[func_index]
- .signature
- .unwrap_module_type_index();
- self.array_to_wasm_trampoline(
- key,
- FuncKey::DefinedWasmFunction(module_index, def_func_index),
- types[sig].unwrap_func(),
- symbol,
- self.isa.pointer_bytes().vmctx_store_context().into(),
- wasmtime_environ::VMCONTEXT_MAGIC,
- )
- }
-
- fn compile_wasm_to_array_trampoline(
- &self,
- wasm_func_ty: &WasmFuncType,
+ translation: Option<&ModuleTranslation<'_>>,
key: FuncKey,
+ types: &ModuleTypesBuilder,
symbol: &str,
) -> Result<CompiledFunctionBody, CompileError> {
- log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}");
-
- let isa = &*self.isa;
- let pointer_type = isa.pointer_type();
- let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables);
- let array_call_sig = array_call_signature(isa);
-
- let mut compiler = self.function_compiler();
- let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig);
- let (mut builder, block0) = compiler.builder(func);
+ match key {
+ FuncKey::ArrayToWasmTrampoline(module_index, def_func_index) => {
+ let translation = translation.unwrap();
+ let func_index = translation.module.func_index(def_func_index);
+ let sig = translation.module.functions[func_index]
+ .signature
+ .unwrap_module_type_index();
+ self.array_to_wasm_trampoline(
+ key,
+ FuncKey::DefinedWasmFunction(module_index, def_func_index),
+ types[sig].unwrap_func(),
+ symbol,
+ self.isa.pointer_bytes().vmctx_store_context().into(),
+ wasmtime_environ::VMCONTEXT_MAGIC,
+ )
+ }
- let args = builder.func.dfg.block_params(block0).to_vec();
- let callee_vmctx = args[0];
- let caller_vmctx = args[1];
+ FuncKey::WasmToArrayTrampoline(ty) => {
+ let ty = types[ty].unwrap_func();
+ self.compile_wasm_to_array_trampoline(ty, key, symbol)
+ }
+ #[cfg(feature = "component-model")]
+ FuncKey::ResourceDropTrampoline => {
+ let ty = types.find_resource_drop_signature().unwrap();
+ let ty = types[ty].unwrap_func();
+ self.compile_wasm_to_array_trampoline(ty, key, symbol)
+ }
- // We are exiting Wasm, so save our PC and FP.
- //
- // Assert that the caller vmctx really is a core Wasm vmctx, since
- // that's what we are assuming with our offsets below.
- self.debug_assert_vmctx_kind(
- &mut builder,
- caller_vmctx,
- wasmtime_environ::VMCONTEXT_MAGIC,
- );
- let ptr = isa.pointer_bytes();
- let vm_store_context = builder.ins().load(
- pointer_type,
- MemFlagsData::trusted(),
- caller_vmctx,
- i32::from(ptr.vmcontext_store_context()),
- );
- save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context);
+ FuncKey::DefinedWasmFunction(..) => {
+ panic!("use `compile_function` instead")
+ }
- // Spill all wasm arguments to the stack in `ValRaw` slots.
- let (args_base, args_len) =
- self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);
- let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));
+ FuncKey::WasmToBuiltinTrampoline(..) | FuncKey::PatchableToBuiltinTrampoline(_) => {
+ self.compile_wasm_to_builtin(key, symbol)
+ }
- // Load the actual callee out of the
- // `VMArrayCallHostFuncContext::host_func`.
- let ptr_size = isa.pointer_bytes();
- let callee = builder.ins().load(
- pointer_type,
- MemFlagsData::trusted(),
- callee_vmctx,
- ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),
- );
+ FuncKey::PulleyHostCall(_) => unreachable!(),
- // Do an indirect call to the callee.
- let callee_signature = builder.func.import_signature(array_call_sig);
- let call = self.call_indirect_host(
- &mut builder,
- HostCall::ArrayCall,
- callee_signature,
- callee,
- &[callee_vmctx, caller_vmctx, args_base, args_len],
- );
+ #[cfg(feature = "component-model")]
+ FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => {
+ unreachable!()
+ }
- // Increment the "execution version" on the VMStoreContext if
- // guest debugging is enabled.
- if self.tunables.debug_guest {
- let vmstore_ctx_ptr = builder.ins().load(
- pointer_type,
- MemFlagsData::trusted().with_readonly(),
- caller_vmctx,
- i32::from(ptr_size.vmctx_store_context()),
- );
- let old_version = builder.ins().load(
- ir::types::I64,
- MemFlagsData::trusted(),
- vmstore_ctx_ptr,
- i32::from(ptr_size.vmstore_context_execution_version()),
- );
- let new_version = builder.ins().iadd_imm(old_version, 1);
- builder.ins().store(
- MemFlagsData::trusted(),
- new_version,
- vmstore_ctx_ptr,
- i32::from(ptr_size.vmstore_context_execution_version()),
- );
+ FuncKey::ModuleStartup(abi, module) => {
+ let translation = translation.unwrap();
+ let ty = match translation.module.startup {
+ ModuleStartup::None => unreachable!(),
+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => {
+ t.unwrap_module_type_index()
+ }
+ };
+ let ty = types[ty].unwrap_func();
+ match abi {
+ // This is a thin array-to-wasm shim around the actual
+ // implementation.
+ Abi::Array => self.array_to_wasm_trampoline(
+ key,
+ FuncKey::ModuleStartup(Abi::Wasm, module),
+ ty,
+ symbol,
+ self.isa.pointer_bytes().vmctx_store_context().into(),
+ wasmtime_environ::VMCONTEXT_MAGIC,
+ ),
+ // Delegate to a helper to finish compiling this.
+ Abi::Wasm => self.compile_module_startup(translation, types, key, ty),
+ Abi::Patchable => unreachable!(),
+ }
+ }
}
-
- // Invoke `raise` if the callee (host) returned an error.
- let succeeded = builder.func.dfg.inst_results(call)[0];
- self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded);
-
- // Return results from the array as native return values.
- let results =
- self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);
- builder.ins().return_(&results);
- builder.finalize();
-
- Ok(CompiledFunctionBody {
- code: box_dyn_any_compiler_context(Some(compiler.cx)),
- needs_gc_heap: false,
- })
}
fn append_code(
@@ -752,103 +928,6 @@ impl wasmtime_environ::Compiler for Compiler {
self.isa.create_systemv_cie()
}
- fn compile_wasm_to_builtin(
- &self,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}");
-
- let isa = &*self.isa;
- let ptr_size = isa.pointer_bytes();
- let pointer_type = isa.pointer_type();
- let sigs = BuiltinFunctionSignatures::new(self);
-
- let (builtin_func_index, wasm_sig) = match key {
- FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)),
- FuncKey::PatchableToBuiltinTrampoline(builtin) => {
- let mut sig = sigs.wasm_signature(builtin);
- // Patchable functions cannot return anything. We
- // raise any errors that occur below so this is fine.
- sig.returns.clear();
- sig.call_conv = CallConv::PreserveAll;
- (builtin, sig)
- }
- _ => unreachable!(),
- };
- let host_sig = sigs.host_signature(builtin_func_index);
-
- let mut compiler = self.function_compiler();
- let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone());
- let (mut builder, block0) = compiler.builder(func);
- let vmctx = builder.block_params(block0)[0];
-
- // Debug-assert that this is the right kind of vmctx, and then
- // additionally perform the "routine of the exit trampoline" of saving
- // fp/pc/etc.
- self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC);
- let vm_store_context = builder.ins().load(
- pointer_type,
- MemFlagsData::trusted(),
- vmctx,
- ptr_size.vmcontext_store_context(),
- );
- save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context);
-
- // Now it's time to delegate to the actual builtin. Forward all our own
- // arguments to the libcall itself.
- let args = builder.block_params(block0).to_vec();
- let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig);
- let results = builder.func.dfg.inst_results(call).to_vec();
-
- // Libcalls do not explicitly jump/raise on traps but instead return a
- // code indicating whether they trapped or not. This means that it's the
- // responsibility of the trampoline to check for an trapping return
- // value and raise a trap as appropriate. With the `results` above check
- // what `index` is and for each libcall that has a trapping return value
- // process it here.
- match builtin_func_index.trap_sentinel() {
- Some(TrapSentinel::Falsy) => {
- self.raise_if_host_trapped(&mut builder, vmctx, results[0]);
- }
- Some(TrapSentinel::NegativeTwo) => {
- let ty = builder.func.dfg.value_type(results[0]);
- let trapped = builder.ins().iconst(ty, -2);
- let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped);
- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
- }
- Some(TrapSentinel::Negative) => {
- let ty = builder.func.dfg.value_type(results[0]);
- let zero = builder.ins().iconst(ty, 0);
- let succeeded =
- builder
- .ins()
- .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero);
- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
- }
- Some(TrapSentinel::NegativeOne) => {
- let ty = builder.func.dfg.value_type(results[0]);
- let minus_one = builder.ins().iconst(ty, -1);
- let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one);
- self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
- }
- None => {}
- }
-
- // And finally, return all the results of this libcall.
- if !wasm_sig.returns.is_empty() {
- builder.ins().return_(&results);
- } else {
- builder.ins().return_(&[]);
- }
- builder.finalize();
-
- Ok(CompiledFunctionBody {
- code: box_dyn_any_compiler_context(Some(compiler.cx)),
- needs_gc_heap: false,
- })
- }
-
fn compiled_function_relocation_targets<'a>(
&'a self,
func: &'a dyn Any,
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index ec972bb7b3..d10c1370d3 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1588,7 +1588,7 @@ impl TranslateTrap for TrapTranslator<'_> {
}
impl ComponentCompiler for Compiler {
- fn compile_trampoline(
+ fn compile_component_trampoline(
&self,
component: &ComponentTranslation,
types: &ComponentTypesBuilder,
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index b3568e4070..c6f97a67a5 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -13,7 +13,7 @@ use crate::{
};
use cranelift_codegen::cursor::FuncCursor;
use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
-use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm};
+use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, V128Imm};
use cranelift_codegen::ir::{
self, BlockArg, Endianness, ExceptionTableData, ExceptionTableItem, types,
};
@@ -29,10 +29,12 @@ use std::mem;
use wasmparser::{FuncValidator, Operator, WasmFeatures, WasmModuleResources};
use wasmtime_core::math::f64_cvt_to_int_bounds;
use wasmtime_environ::{
- BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,
- EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,
- GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,
- ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,
+ BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex,
+ DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex,
+ FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,
+ IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module,
+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize,
+ RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements,
TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
};
@@ -1481,8 +1483,12 @@ impl FuncEnvironment<'_> {
if !self.module.globals[index].mutability {
if let Some(index) = self.module.defined_global_index(index) {
- let init = &self.module.global_initializers[index];
- if let Some(value) = init.const_eval() {
+ if let Ok(i) = self
+ .module
+ .global_initializers
+ .binary_search_by_key(&index, |(def_index, _)| *def_index)
+ {
+ let (_, value) = self.module.global_initializers[i];
return GlobalVariable::Constant { value };
}
}
@@ -2618,7 +2624,7 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let table_data = self.get_or_create_table(builder.func, table_index);
let (dst, flags) = table_data.prepare_table_addr(self, builder, index);
- self.emit_table_set(builder, table_index, dst, flags, value)
+ self.emit_table_set(builder, table_index, dst, flags, value, true)
}
/// Helper to store `value` into the table address at `addr` using `flags`.
@@ -2632,19 +2638,32 @@ impl FuncEnvironment<'_> {
addr: ir::Value,
flags: ir::MemFlagsData,
value: ir::Value,
+ initialized: bool,
) -> WasmResult<()> {
let table = self.module.tables[table_index];
match table.ref_type.heap_type.top() {
// GC-managed types.
WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
- gc::gc_compiler(self)?.translate_write_gc_reference(
- self,
- builder,
- table.ref_type,
- addr,
- value,
- flags,
- )
+ let mut gc = gc::gc_compiler(self)?;
+ if initialized {
+ gc.translate_write_gc_reference(
+ self,
+ builder,
+ table.ref_type,
+ addr,
+ value,
+ flags,
+ )
+ } else {
+ gc.translate_init_gc_reference(
+ self,
+ builder,
+ table.ref_type,
+ addr,
+ value,
+ flags,
+ )
+ }
}
// Function types.
@@ -2693,7 +2712,16 @@ impl FuncEnvironment<'_> {
val: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
- self.translate_entity_fill(builder, table_index, dst, val, len)
+ self.translate_entity_fill(
+ builder,
+ CheckedEntity::Table {
+ table: table_index,
+ initialized: true,
+ },
+ dst,
+ val,
+ len,
+ )
}
pub fn translate_ref_i31(
@@ -3164,6 +3192,21 @@ impl FuncEnvironment<'_> {
builder: &mut FunctionBuilder<'_>,
global_index: GlobalIndex,
val: ir::Value,
+ ) -> WasmResult<()> {
+ self.emit_global_set(builder, global_index, val, true)
+ }
+
+ /// Helper to translate the `global.set` instruction.
+ ///
+ /// The main difference with `translate_global_set`, the main entrypoint, is
+ /// the `initialized` flag to indicate whether the global was previously
+ /// initialized to a valid value.
+ fn emit_global_set(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ global_index: GlobalIndex,
+ val: ir::Value,
+ initialized: bool,
) -> WasmResult<()> {
match self.get_or_create_global(builder.func, global_index) {
GlobalVariable::Constant { .. } => {
@@ -3197,14 +3240,26 @@ impl FuncEnvironment<'_> {
let gv = builder.ins().global_value(self.pointer_type(), gv);
let src = builder.ins().iadd_imm(gv, i64::from(offset));
- gc::gc_compiler(self)?.translate_write_gc_reference(
- self,
- builder,
- ty,
- src,
- val,
- ir::MemFlagsData::trusted(),
- )?
+ let mut gc = gc::gc_compiler(self)?;
+ if initialized {
+ gc.translate_write_gc_reference(
+ self,
+ builder,
+ ty,
+ src,
+ val,
+ ir::MemFlagsData::trusted(),
+ )?;
+ } else {
+ gc.translate_init_gc_reference(
+ self,
+ builder,
+ ty,
+ src,
+ val,
+ ir::MemFlagsData::trusted(),
+ )?;
+ }
}
}
Ok(())
@@ -3731,7 +3786,7 @@ impl FuncEnvironment<'_> {
},
);
}
- CheckedEntity::Table(table) => {
+ CheckedEntity::Table { table, initialized } => {
self.emit_raw_array_or_table_fill(
builder,
entity,
@@ -3739,7 +3794,14 @@ impl FuncEnvironment<'_> {
val,
len_ptr,
&|env, builder, addr, value| {
- env.emit_table_set(builder, table, addr, ir::MemFlagsData::trusted(), value)
+ env.emit_table_set(
+ builder,
+ table,
+ addr,
+ ir::MemFlagsData::trusted(),
+ value,
+ initialized,
+ )
},
)?;
}
@@ -3761,7 +3823,9 @@ impl FuncEnvironment<'_> {
)?;
}
// Not allowed to be written to in wasm.
- CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(),
+ CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {
+ unreachable!()
+ }
}
Ok(())
@@ -3993,7 +4057,7 @@ impl FuncEnvironment<'_> {
// Lookup the passive data segment corresponding to this data segment.
// If this is an active data segment then it already has length 0 so
// there's nothing to do.
- let passive_index = match self.translation.passive_data_map[seg_index] {
+ let runtime_index = match self.translation.runtime_data_map[seg_index] {
Some(idx) => idx,
None => return Ok(()),
};
@@ -4006,7 +4070,7 @@ impl FuncEnvironment<'_> {
ir::MemFlagsData::trusted(),
new_length,
vmctx,
- i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)).unwrap(),
+ i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(),
);
Ok(())
@@ -4080,7 +4144,9 @@ impl FuncEnvironment<'_> {
CheckedEntity::Memory(_) => {
assert!(matches!(
src_entity,
- CheckedEntity::Memory(_) | CheckedEntity::Data { .. }
+ CheckedEntity::Memory(_)
+ | CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_)
));
// A memory's elements are bytes, so the element count is already
// the byte length.
@@ -4098,7 +4164,7 @@ impl FuncEnvironment<'_> {
// Tables/arrays are sometimes a memcpy, sometimes a per-element
// loop. Delegate further to figure that out.
- CheckedEntity::Table(_) | CheckedEntity::Array { .. } => self
+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => self
.emit_raw_array_or_table_copy(
builder,
dst_entity,
@@ -4111,7 +4177,9 @@ impl FuncEnvironment<'_> {
),
// Cannot copy into a data or element segment in wasm.
- CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(),
+ CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {
+ unreachable!()
+ }
}
}
@@ -4136,26 +4204,16 @@ impl FuncEnvironment<'_> {
// Load the entity size, as `pointer_type`.
let entity_size = match entity {
CheckedEntity::Memory(i) => self.memory_size_in_bytes(&mut builder.cursor(), i),
- CheckedEntity::Table(i) => {
- let size = self.translate_table_size(builder.cursor(), i);
+ CheckedEntity::Table { table, .. } => {
+ let size = self.translate_table_size(builder.cursor(), table);
self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), size, idx_type)
}
- CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment]
+ CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment]
{
- Some(passive_index) => {
- let vmctx = self.vmctx_val(&mut builder.cursor());
- let offset =
- i32::try_from(self.offsets.vmctx_passive_data_length(passive_index))
- .unwrap();
- let flags = ir::MemFlagsData::trusted();
- match pointer_type {
- I32 => builder.ins().load(I32, flags, vmctx, offset),
- I64 => builder.ins().uload32(flags, vmctx, offset),
- _ => unreachable!(),
- }
- }
+ Some(i) => self.load_runtime_data_length_as_pointer(builder, i),
None => builder.ins().iconst(pointer_type, 0),
},
+ CheckedEntity::RuntimeData(i) => self.load_runtime_data_length_as_pointer(builder, i),
CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] {
Some(passive_index) => {
let vmctx = self.vmctx_val(&mut builder.cursor());
@@ -4225,30 +4283,20 @@ impl FuncEnvironment<'_> {
let heap = &self.heaps()[heap];
builder.ins().global_value(pointer_type, heap.base)
}
- CheckedEntity::Table(i) => {
- let table = self.get_or_create_table(builder.func, i);
+ CheckedEntity::Table { table, .. } => {
+ let table = self.get_or_create_table(builder.func, table);
builder.ins().global_value(pointer_type, table.base_gv)
}
- CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment]
+ CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment]
{
- Some(passive_index) => {
- let vmctx = self.vmctx_val(&mut builder.cursor());
- let offset =
- i32::try_from(self.offsets.vmctx_passive_data_base(passive_index)).unwrap();
- builder.ins().load(
- self.pointer_type(),
- ir::MemFlagsData::trusted(),
- vmctx,
- offset,
- )
- }
-
+ Some(runtime_index) => self.load_runtime_data_base(builder, runtime_index),
// Any address should do for an active data segment, but pick
// something non-null for now. Note that the length of an active
// data segment is always 0, so we know that the memcpy, if any,
// will be 0 elements, so the actual value here doesn't matter.
None => builder.ins().iconst(pointer_type, 1),
},
+ CheckedEntity::RuntimeData(i) => self.load_runtime_data_base(builder, i),
// Element segments are quite similar to data segments just above.
CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] {
Some(passive_index) => {
@@ -4295,6 +4343,41 @@ impl FuncEnvironment<'_> {
Ok(builder.ins().iadd(base, byte_offset))
}
+ fn load_runtime_data_length(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ runtime_index: RuntimeDataIndex,
+ ) -> ir::Value {
+ let vmctx = self.vmctx_val(&mut builder.cursor());
+ let offset = i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap();
+ let flags = ir::MemFlagsData::trusted();
+ builder.ins().load(I32, flags, vmctx, offset)
+ }
+
+ fn load_runtime_data_length_as_pointer(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ runtime_index: RuntimeDataIndex,
+ ) -> ir::Value {
+ let length = self.load_runtime_data_length(builder, runtime_index);
+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), length, IndexType::I32)
+ }
+
+ fn load_runtime_data_base(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ runtime_index: RuntimeDataIndex,
+ ) -> ir::Value {
+ let vmctx = self.vmctx_val(&mut builder.cursor());
+ let offset = i32::try_from(self.offsets.vmctx_runtime_data_base(runtime_index)).unwrap();
+ builder.ins().load(
+ self.pointer_type(),
+ ir::MemFlagsData::trusted(),
+ vmctx,
+ offset,
+ )
+ }
+
pub fn translate_table_copy(
&mut self,
builder: &mut FunctionBuilder<'_>,
@@ -4304,7 +4387,20 @@ impl FuncEnvironment<'_> {
src: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
- self.translate_entity_copy(builder, dst_table_index, src_table_index, dst, src, len)
+ self.translate_entity_copy(
+ builder,
+ CheckedEntity::Table {
+ table: dst_table_index,
+ initialized: true,
+ },
+ CheckedEntity::Table {
+ table: src_table_index,
+ initialized: true,
+ },
+ dst,
+ src,
+ len,
+ )
}
/// Emits a copy between two WebAssembly table or array entities.
@@ -4377,7 +4473,7 @@ impl FuncEnvironment<'_> {
// would mean that memcpy isn't suitable. If lazy init is
// disabled though then funcrefs are just pointers so a
// memcpy can be used.
- CheckedEntity::Table(_) => self.tunables.table_lazy_init,
+ CheckedEntity::Table { .. } => self.tunables.table_lazy_init,
// The GC heap has integers representing funcrefs, so memcpy
// is fine.
CheckedEntity::Array { .. } => false,
@@ -4385,7 +4481,9 @@ impl FuncEnvironment<'_> {
// can't work.
CheckedEntity::Elem(_) => true,
// Not possible
- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(),
+ CheckedEntity::Memory(_)
+ | CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_) => unreachable!(),
},
},
};
@@ -4439,7 +4537,10 @@ impl FuncEnvironment<'_> {
// it's easier right now to share the internals of
// `translate_table_get` which are a bit tricky with
// funcrefs.
- CheckedEntity::Table(i) => this.translate_table_get(builder, i, src_index)?,
+ CheckedEntity::Table { table, initialized } => {
+ assert!(initialized);
+ this.translate_table_get(builder, table, src_index)?
+ }
CheckedEntity::Array { initialized, .. } => {
assert!(initialized);
let read_ty = src_entity.storage_type(this);
@@ -4468,11 +4569,20 @@ impl FuncEnvironment<'_> {
}
}
}
- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(),
+ CheckedEntity::Memory(_)
+ | CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_) => unreachable!(),
};
match dst_entity {
- CheckedEntity::Table(i) => {
- this.emit_table_set(builder, i, dst, ir::MemFlagsData::trusted(), val)?;
+ CheckedEntity::Table { table, initialized } => {
+ this.emit_table_set(
+ builder,
+ table,
+ dst,
+ ir::MemFlagsData::trusted(),
+ val,
+ initialized,
+ )?;
}
CheckedEntity::Array { initialized, .. } => {
if initialized {
@@ -4483,6 +4593,7 @@ impl FuncEnvironment<'_> {
}
CheckedEntity::Memory(_)
| CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_)
| CheckedEntity::Elem(_) => unreachable!(),
}
Ok(())
@@ -4842,7 +4953,10 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
self.translate_entity_copy(
builder,
- table_index,
+ CheckedEntity::Table {
+ table: table_index,
+ initialized: true,
+ },
CheckedEntity::Elem(ElemIndex::from_u32(seg_index)),
dst,
src,
@@ -5645,6 +5759,386 @@ impl FuncEnvironment<'_> {
pub fn is_reachable(&self) -> bool {
self.stacks.reachable()
}
+
+ /// Generates the body of a `FuncKey::ModuleStartup(..)` function.
+ ///
+ /// This will perform all initialization of a module before any code in the
+ /// module itself starts executing. This will, for example, execute all
+ /// constant expressions for global initializers.
+ pub fn translate_module_startup(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
+ for (i, expr) in self.translation.global_initializers.iter() {
+ self.module_initialize_global(builder, *i, expr)?;
+ }
+ for (i, exprs) in self.translation.passive_elements.iter() {
+ self.module_initialize_passive_element(builder, i, exprs)?;
+ }
+ for (i, init) in self.translation.table_initialization.initial_values.iter() {
+ match init {
+ TableInitialValue::Null => {}
+ TableInitialValue::Expr(expr) => {
+ self.module_initialize_table_with_fill(builder, i, expr)?;
+ }
+ }
+ }
+ for segment in self.translation.table_initialization.segments.iter() {
+ self.module_initialize_table_with_segment(builder, segment)?;
+ }
+ match &self.translation.memory_init {
+ MemoryInit::Unprocessed(_) => unreachable!(),
+ MemoryInit::Processed(segments) => {
+ for (memory, offset, data) in segments.iter() {
+ self.module_initialize_memory_segment(builder, *memory, offset, *data)?;
+ }
+ }
+ }
+ if let Some(i) = self.translation.start_func {
+ self.module_start(builder, i)?;
+ }
+ Ok(())
+ }
+
+ /// Initializes all "complicated" globals in a module.
+ ///
+ /// This will translate the given constant expression and then initialize
+ /// the global. Note that this translation is done with the global not
+ /// initialized, meaning that GC barriers are slightly tweaked.
+ fn module_initialize_global(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ global: DefinedGlobalIndex,
+ expr: &ConstExpr,
+ ) -> WasmResult<()> {
+ let index = self.module.global_index(global);
+ let val = self.translate_const_expr(builder, expr)?;
+ self.emit_global_set(builder, index, val, false)
+ }
+
+ /// Initializes all passive element segments for a module.
+ ///
+ /// This will execute all constant expressions for all passive element
+ /// segments and initialize all `ValRaw` storage on the host. This
+ /// implements the semantics of how passive element segments are evaluated
+ /// once in a wasm instance.
+ fn module_initialize_passive_element(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ elem: PassiveElemIndex,
+ exprs: &TableSegmentElements,
+ ) -> WasmResult<()> {
+ let vmctx = self.vmctx_val(&mut builder.cursor());
+ let libcall = self
+ .builtin_functions
+ .passive_elem_segment_base(builder.func);
+ let idx = builder.ins().iconst(I32, i64::from(elem.as_u32()));
+ let call = builder.ins().call(libcall, &[vmctx, idx]);
+ let base = builder.func.dfg.first_result(call);
+
+ // Values in `ValRaw` are always stored in little-endian.
+ let flags = ir::MemFlagsData::trusted().with_endianness(Endianness::Little);
+
+ match exprs {
+ TableSegmentElements::Functions(indices) => {
+ for (i, func) in indices.iter().enumerate() {
+ let func = self.translate_ref_func(builder.cursor(), *func)?;
+ builder.ins().store(
+ flags,
+ func,
+ base,
+ i32::try_from(i.checked_mul(16).unwrap()).unwrap(),
+ );
+ }
+ }
+ TableSegmentElements::Expressions { exprs, ty } => {
+ for (i, expr) in exprs.iter().enumerate() {
+ let val = self.translate_const_expr(builder, expr)?;
+
+ let dst = builder
+ .ins()
+ .iadd_imm(base, i64::try_from(i.checked_mul(16).unwrap()).unwrap());
+ match ty.heap_type.top() {
+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {
+ let ty = WasmStorageType::Val(WasmValType::Ref(*ty));
+ gc::init_field_at_addr(self, builder, ty, dst, val)?;
+ }
+ WasmHeapTopType::Func | WasmHeapTopType::Cont => {
+ builder.ins().store(
+ flags,
+ val,
+ base,
+ i32::try_from(i.checked_mul(16).unwrap()).unwrap(),
+ );
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+
+ /// Initializes all tables with non-null initializers.
+ ///
+ /// This function will morally execute a `table.fill` for the specified
+ /// defined table in this module for the result of the constant expression
+ /// provided. Note that the fill operation is done on an initialized table
+ /// which slightly alters the barriers emitted for GC types.
+ fn module_initialize_table_with_fill(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ table: DefinedTableIndex,
+ init: &ConstExpr,
+ ) -> WasmResult<()> {
+ let table = self.module.table_index(table);
+ let ty = self.module.tables[table];
+ let val = self.translate_const_expr(builder, init)?;
+ let dst = builder.ins().iconst(index_type_to_ir_type(ty.idx_type), 0);
+ let len = builder.ins().iconst(
+ index_type_to_ir_type(ty.idx_type),
+ ty.limits.min.cast_signed(),
+ );
+ self.translate_entity_fill(
+ builder,
+ CheckedEntity::Table {
+ table,
+ initialized: false,
+ },
+ dst,
+ val,
+ len,
+ )
+ }
+
+ /// Executes initialization for an active element segment in a module.
+ ///
+ /// Note that this isn't done for "precomputed" tables which won't have
+ /// initializers present. Additionally note that this doesn't use
+ /// `table.init` because the element segment won't actually be present
+ /// anywhere at runtime.
+ fn module_initialize_table_with_segment(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ segment: &TableSegment,
+ ) -> WasmResult<()> {
+ let offset = self.translate_const_expr(builder, &segment.offset)?;
+ let segment_len = builder.ins().iconst(
+ index_type_to_ir_type(self.module.tables[segment.table_index].idx_type),
+ i64::try_from(segment.elements.len()).unwrap(),
+ );
+
+ // Check the bounds first before mutating anything.
+ self.translate_entity_bounds_check(
+ builder,
+ CheckedEntity::Table {
+ table: segment.table_index,
+ initialized: true,
+ },
+ offset,
+ segment_len,
+ )?;
+
+ // Re-use the `table.set` translation for making this a simple function
+ // to define. That re-executes the bounds check which is a bit
+ // inefficient, but can be optimized in the future.
+ match &segment.elements {
+ TableSegmentElements::Functions(indices) => {
+ for (i, func) in indices.iter().enumerate() {
+ let func = self.translate_ref_func(builder.cursor(), *func)?;
+ let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap());
+ self.translate_table_set(builder, segment.table_index, func, index)?;
+ }
+ }
+ TableSegmentElements::Expressions { exprs, ty: _ } => {
+ for (i, expr) in exprs.iter().enumerate() {
+ let val = self.translate_const_expr(builder, expr)?;
+ let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap());
+ self.translate_table_set(builder, segment.table_index, val, index)?;
+ }
+ }
+ }
+ Ok(())
+ }
+
+ /// Peform initialization of an active data segment in a module.
+ fn module_initialize_memory_segment(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ memory: MemoryIndex,
+ offset: &MemorySegmentOffset,
+ data: RuntimeDataIndex,
+ ) -> WasmResult<()> {
+ let mut end = None;
+ let offset = match offset {
+ // This is a bit subtle, but the goal here is to make a dynamic
+ // deduction to see if this initialization is actually necessary
+ // based on state at runtime. With a static initializer the goal is
+ // to enable CoW initialization at runtime, but not all hosts have
+ // that configured or supported. If it's supported then we shouldn't
+ // do anything here, but if it's unsupported then this must actually
+ // execute initialization.
+ //
+ // The host already put the addresses of all data segments into the
+ // `VMContext` during `VMContext::initialize_vmctx`, and this reads
+ // out the base pointer. If it's null then the host doesn't actually
+ // need initialization, so this is skipped, and otherwise it's the
+ // actual base pointer that's going to be used.
+ MemorySegmentOffset::Static(n) => {
+ let current_block = builder.current_block().unwrap();
+ let init_block = builder.create_block();
+ let end_block = builder.create_block();
+ end = Some(end_block);
+
+ builder.ensure_inserted_block();
+ builder.insert_block_after(init_block, current_block);
+ builder.insert_block_after(end_block, init_block);
+
+ let base = self.load_runtime_data_base(builder, data);
+ let null = builder.ins().iconst(self.pointer_type(), 0);
+ let is_null = builder.ins().icmp(IntCC::Equal, base, null);
+ builder.ins().brif(is_null, end_block, &[], init_block, &[]);
+ builder.switch_to_block(init_block);
+ builder.seal_block(init_block);
+ let ty = index_type_to_ir_type(self.module.memories[memory].idx_type);
+ builder.ins().iconst(ty, n.cast_signed())
+ }
+ MemorySegmentOffset::Expr(expr) => self.translate_const_expr(builder, expr)?,
+ };
+
+ // Model the initialization here as a `memory.init`.
+ let len = self.load_runtime_data_length(builder, data);
+ let start = builder.ins().iconst(I32, 0);
+ self.translate_entity_copy(builder, memory, data, offset, start, len)?;
+
+ // Finalize control-flow for the `MemorySegmentOffset::Static` case
+ // above.
+ if let Some(end) = end {
+ builder.ins().jump(end, &[]);
+ builder.switch_to_block(end);
+ builder.seal_block(end);
+ }
+ Ok(())
+ }
+
+ /// Translates the final step of module initialization, executing the
+ /// `start` function.
+ fn module_start(&mut self, builder: &mut FunctionBuilder, func: FuncIndex) -> WasmResult<()> {
+ // Manuall manage fuel around the call as the `Call` opcode does for
+ // normal wasm to ensure that it's correctly accounted for.
+ if self.tunables.consume_fuel {
+ self.fuel_consumed += 1;
+ self.fuel_increment_var(builder);
+ self.fuel_save_from_var(builder);
+ }
+ let ty = self.module.functions[func]
+ .signature
+ .unwrap_module_type_index();
+ let sig_ref = self.get_or_create_interned_sig_ref(builder.func, ty);
+ self.translate_call(builder, Default::default(), func, sig_ref, &[])?;
+ if self.tunables.consume_fuel {
+ self.fuel_load_into_var(builder);
+ }
+ Ok(())
+ }
+
+ /// Helper function to evaluate `expr` into a value.
+ ///
+ /// This primarily reuses all the `translate_*` functions elsewhere on this
+ /// builder toe deduplicate implementation details.
+ fn translate_const_expr(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ expr: &ConstExpr,
+ ) -> WasmResult<ir::Value> {
+ let mut stack = Vec::new();
+ for op in expr.ops() {
+ if self.tunables.consume_fuel {
+ self.fuel_consumed += 1;
+ }
+ match op {
+ ConstOp::I32Const(i) => {
+ stack.push(builder.ins().iconst(I32, i64::from(*i)));
+ }
+ ConstOp::I64Const(i) => {
+ stack.push(builder.ins().iconst(I64, *i));
+ }
+ ConstOp::F32Const(i) => {
+ stack.push(builder.ins().f32const(Ieee32::with_bits(*i)));
+ }
+ ConstOp::F64Const(i) => {
+ stack.push(builder.ins().f64const(Ieee64::with_bits(*i)));
+ }
+ ConstOp::V128Const(i) => {
+ let data = i.to_le_bytes().to_vec().into();
+ let handle = builder.func.dfg.constants.insert(data);
+ stack.push(builder.ins().vconst(ir::types::I8X16, handle));
+ }
+ ConstOp::GlobalGet(i) => {
+ stack.push(self.translate_global_get(builder, *i)?);
+ }
+ ConstOp::RefI31 => {
+ let val = stack.pop().unwrap();
+ stack.push(self.translate_ref_i31(builder.cursor(), val)?);
+ }
+ ConstOp::RefNull(ty) => {
+ stack.push(self.translate_ref_null(builder.cursor(), *ty)?);
+ }
+ ConstOp::RefFunc(i) => {
+ stack.push(self.translate_ref_func(builder.cursor(), *i)?);
+ }
+ ConstOp::I32Add | ConstOp::I64Add => {
+ let b = stack.pop().unwrap();
+ let a = stack.pop().unwrap();
+ stack.push(builder.ins().iadd(a, b));
+ }
+ ConstOp::I32Sub | ConstOp::I64Sub => {
+ let b = stack.pop().unwrap();
+ let a = stack.pop().unwrap();
+ stack.push(builder.ins().isub(a, b));
+ }
+ ConstOp::I32Mul | ConstOp::I64Mul => {
+ let b = stack.pop().unwrap();
+ let a = stack.pop().unwrap();
+ stack.push(builder.ins().imul(a, b));
+ }
+ ConstOp::StructNew { struct_type_index } => {
+ let arity = self.struct_fields_len(*struct_type_index)?;
+ let fields = stack.drain(stack.len() - arity..).collect::<_>();
+ stack.push(self.translate_struct_new(builder, *struct_type_index, fields)?);
+ }
+ ConstOp::StructNewDefault { struct_type_index } => {
+ stack.push(self.translate_struct_new_default(builder, *struct_type_index)?);
+ }
+ ConstOp::ArrayNew { array_type_index } => {
+ let len = stack.pop().unwrap();
+ let elem = stack.pop().unwrap();
+ stack.push(self.translate_array_new(builder, *array_type_index, elem, len)?);
+ }
+ ConstOp::ArrayNewDefault { array_type_index } => {
+ let len = stack.pop().unwrap();
+ stack.push(self.translate_array_new_default(
+ builder,
+ *array_type_index,
+ len,
+ )?);
+ }
+ ConstOp::ArrayNewFixed {
+ array_type_index,
+ array_size,
+ } => {
+ let array_size = usize::try_from(*array_size).unwrap();
+ let elems = stack.drain(stack.len() - array_size..).collect::<Vec<_>>();
+ stack.push(self.translate_array_new_fixed(
+ builder,
+ *array_type_index,
+ &elems,
+ )?);
+ }
+ ConstOp::ExternConvertAny => {}
+ ConstOp::AnyConvertExtern => {}
+ }
+ }
+ let ret = stack.pop().unwrap();
+ assert!(stack.is_empty());
+ Ok(ret)
+ }
}
// Helper function to convert an `IndexType` to an `ir::Type`.
@@ -5698,7 +6192,12 @@ enum CheckedEntity {
/// A WebAssembly linear memory.
Memory(MemoryIndex),
/// A WebAssembly table.
- Table(TableIndex),
+ Table {
+ /// The index of the table that's being accessed.
+ table: TableIndex,
+ /// Whether or not this table has been initialized yet.
+ initialized: bool,
+ },
/// A WebAssembly data segment loaded in chunks of `element_size` bytes.
///
/// For `memory.init` this will have `element_size = 1`, but for
@@ -5711,6 +6210,11 @@ enum CheckedEntity {
/// or possibly more for array initialization.
element_size: u32,
},
+ /// A refinement of `Data` above which uses a `RuntimeDataIndex`.
+ ///
+ /// This isn't reachable from core wasm but is used during module
+ /// startup.
+ RuntimeData(RuntimeDataIndex),
/// A WebAssembly passive element segment.
Elem(ElemIndex),
/// An `arrayref` with the specified type.
@@ -5731,9 +6235,9 @@ impl From<MemoryIndex> for CheckedEntity {
}
}
-impl From<TableIndex> for CheckedEntity {
- fn from(table_index: TableIndex) -> Self {
- CheckedEntity::Table(table_index)
+impl From<RuntimeDataIndex> for CheckedEntity {
+ fn from(runtime_data_index: RuntimeDataIndex) -> Self {
+ CheckedEntity::RuntimeData(runtime_data_index)
}
}
@@ -5742,10 +6246,11 @@ impl CheckedEntity {
fn index_type(&self, env: &FuncEnvironment) -> IndexType {
match *self {
CheckedEntity::Memory(i) => env.memory(i).idx_type,
- CheckedEntity::Table(i) => env.table(i).idx_type,
- CheckedEntity::Data { .. } | CheckedEntity::Array { .. } | CheckedEntity::Elem(_) => {
- IndexType::I32
- }
+ CheckedEntity::Table { table, .. } => env.table(table).idx_type,
+ CheckedEntity::Data { .. }
+ | CheckedEntity::Array { .. }
+ | CheckedEntity::Elem(_)
+ | CheckedEntity::RuntimeData(_) => IndexType::I32,
}
}
@@ -5756,9 +6261,9 @@ impl CheckedEntity {
func: &mut ir::Function,
) -> WasmResult<u32> {
Ok(match *self {
- CheckedEntity::Memory(_) => 1,
+ CheckedEntity::Memory(_) | CheckedEntity::RuntimeData(_) => 1,
CheckedEntity::Data { element_size, .. } => element_size,
- CheckedEntity::Table(table) => env.get_or_create_table(func, table).element_size,
+ CheckedEntity::Table { table, .. } => env.get_or_create_table(func, table).element_size,
CheckedEntity::Array { ty, .. } => env.array_layout(ty)?.elem_size,
CheckedEntity::Elem(_) => 16,
})
@@ -5770,8 +6275,9 @@ impl CheckedEntity {
CheckedEntity::Memory(_)
| CheckedEntity::Data {
element_size: 1, ..
- } => WasmStorageType::I8,
- CheckedEntity::Table(table) => {
+ }
+ | CheckedEntity::RuntimeData(_) => WasmStorageType::I8,
+ CheckedEntity::Table { table, .. } => {
WasmStorageType::Val(WasmValType::Ref(env.table(table).ref_type))
}
CheckedEntity::Array { ty, .. } => {
@@ -5786,10 +6292,10 @@ impl CheckedEntity {
/// Returns the trap code to use when an index into this entity is out-of-bounds.
fn oob_trap_code(&self) -> ir::TrapCode {
match self {
- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => {
- ir::TrapCode::HEAP_OUT_OF_BOUNDS
- }
- CheckedEntity::Table(_) | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS,
+ CheckedEntity::Memory(_)
+ | CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_) => ir::TrapCode::HEAP_OUT_OF_BOUNDS,
+ CheckedEntity::Table { .. } | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS,
CheckedEntity::Array { .. } => TRAP_ARRAY_OUT_OF_BOUNDS,
}
}
@@ -5798,13 +6304,22 @@ impl CheckedEntity {
/// bit patterns.
fn allows_memset(&self, env: &FuncEnvironment) -> bool {
match self {
- CheckedEntity::Memory(_) | CheckedEntity::Data { .. } | CheckedEntity::Array { .. } => {
- true
+ CheckedEntity::Memory(_)
+ | CheckedEntity::Data { .. }
+ | CheckedEntity::RuntimeData(_) => true,
+ CheckedEntity::Array { ty, .. } => {
+ let array_ty = env.types.unwrap_array(*ty).unwrap();
+ // Most GC references need barriers, funcrefs need intern-ing,
+ // etc. Disallow memset on all reference types.
+ !matches!(
+ array_ty.0.element_type,
+ WasmStorageType::Val(WasmValType::Ref(_))
+ )
}
CheckedEntity::Elem(_) => false,
// Tables that are lazily initialized can't be memset because the
// initialized bit needs to be set when storing values.
- CheckedEntity::Table(_) => !env.tunables.table_lazy_init,
+ CheckedEntity::Table { .. } => !env.tunables.table_lazy_init,
}
}
}
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index 9857146f03..7dd7b5905f 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -162,6 +162,20 @@ pub trait GcCompiler {
flags: ir::MemFlagsData,
) -> WasmResult<()>;
+ /// Same as [`Self::translate_write_gc_reference`] except that the
+ /// destination address has not been previously initialized.
+ ///
+ /// This will, for example, skip barriers on the destination address.
+ fn translate_init_gc_reference(
+ &mut self,
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ty: WasmRefType,
+ dst: ir::Value,
+ new_val: ir::Value,
+ flags: ir::MemFlagsData,
+ ) -> WasmResult<()>;
+
/// Stores `val` into `addr` with the `ty` specified.
///
/// This initializes a previously-uninitialized field, so barriers on the
diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rs
index 5c0e89a244..79cbe1841e 100644
--- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs
@@ -400,6 +400,19 @@ impl GcCompiler for CopyingCompiler {
unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)
}
+ fn translate_init_gc_reference(
+ &mut self,
+ _func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ty: WasmRefType,
+ dst: ir::Value,
+ new_val: ir::Value,
+ flags: ir::MemFlagsData,
+ ) -> WasmResult<()> {
+ // No write barrier needed for the copying collector.
+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)
+ }
+
fn init_field(
&mut self,
func_env: &mut FuncEnvironment<'_>,
diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rs
index 7938257a4b..d59c3dc6b2 100644
--- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs
@@ -330,109 +330,6 @@ impl DrcCompiler {
);
builder.ins().store(GC_MEMFLAGS, new_reserved, ptr, 0);
}
-
- /// Write to an uninitialized GC reference field, initializing it.
- ///
- /// ```text
- /// *dst = new_val
- /// ```
- ///
- /// Doesn't need to do a full write barrier: we don't have an old reference
- /// that is being overwritten and needs its refcount decremented, just a new
- /// reference whose count should be incremented.
- fn translate_init_gc_reference(
- &mut self,
- func_env: &mut FuncEnvironment<'_>,
- builder: &mut FunctionBuilder,
- ty: WasmRefType,
- dst: ir::Value,
- new_val: ir::Value,
- flags: ir::MemFlagsData,
- ) -> WasmResult<()> {
- let (ref_ty, _) = func_env.reference_type(ty.heap_type);
-
- // Special case for references to uninhabited bottom types: see
- // `translate_write_gc_reference` for details.
- if let WasmHeapType::None = ty.heap_type {
- if ty.nullable {
- let null = builder.ins().iconst(ref_ty, 0);
- builder.ins().store(flags, null, dst, 0);
- } else {
- let zero = builder.ins().iconst(ir::types::I32, 0);
- builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);
- }
- return Ok(());
- };
-
- // Special case for `i31ref`s: no need for any barriers.
- if let WasmHeapType::I31 = ty.heap_type {
- return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags);
- }
-
- // Our initialization barrier for GC references being copied out of the
- // stack and initializing a table/global/struct field/etc... is roughly
- // equivalent to the following pseudo-CLIF:
- //
- // ```
- // current_block:
- // ...
- // let new_val_is_null_or_i31 = ...
- // brif new_val_is_null_or_i31, continue_block, inc_ref_block
- //
- // inc_ref_block:
- // let ref_count = load new_val.ref_count
- // let new_ref_count = iadd_imm ref_count, 1
- // store new_val.ref_count, new_ref_count
- // jump check_old_val_block
- //
- // continue_block:
- // store dst, new_val
- // ...
- // ```
- //
- // This write barrier is responsible for ensuring that the new value's
- // ref count is incremented now that the table/global/struct/etc... is
- // holding onto it.
-
- let current_block = builder.current_block().unwrap();
- let inc_ref_block = builder.create_block();
- let continue_block = builder.create_block();
-
- builder.ensure_inserted_block();
- builder.insert_block_after(inc_ref_block, current_block);
- builder.insert_block_after(continue_block, inc_ref_block);
-
- // Current block: check whether the new value is non-null and
- // non-i31. If so, branch to the `inc_ref_block`.
- log::trace!("DRC initialization barrier: check if the value is null or i31");
- let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val);
- builder.ins().brif(
- new_val_is_null_or_i31,
- continue_block,
- &[],
- inc_ref_block,
- &[],
- );
-
- // Block to increment the ref count of the new value when it is non-null
- // and non-i31.
- builder.switch_to_block(inc_ref_block);
- builder.seal_block(inc_ref_block);
- log::trace!("DRC initialization barrier: increment the ref count of the initial value");
- self.mutate_ref_count(func_env, builder, new_val, 1);
- builder.ins().jump(continue_block, &[]);
-
- // Join point after we're done with the GC barrier: do the actual store
- // to initialize the field.
- builder.switch_to_block(continue_block);
- builder.seal_block(continue_block);
- log::trace!(
- "DRC initialization barrier: finally, store into {dst:?} to initialize the field"
- );
- unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?;
-
- Ok(())
- }
}
impl GcCompiler for DrcCompiler {
@@ -959,6 +856,109 @@ impl GcCompiler for DrcCompiler {
Ok(())
}
+ /// Write to an uninitialized GC reference field, initializing it.
+ ///
+ /// ```text
+ /// *dst = new_val
+ /// ```
+ ///
+ /// Doesn't need to do a full write barrier: we don't have an old reference
+ /// that is being overwritten and needs its refcount decremented, just a new
+ /// reference whose count should be incremented.
+ fn translate_init_gc_reference(
+ &mut self,
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ty: WasmRefType,
+ dst: ir::Value,
+ new_val: ir::Value,
+ flags: ir::MemFlagsData,
+ ) -> WasmResult<()> {
+ let (ref_ty, _) = func_env.reference_type(ty.heap_type);
+
+ // Special case for references to uninhabited bottom types: see
+ // `translate_write_gc_reference` for details.
+ if let WasmHeapType::None = ty.heap_type {
+ if ty.nullable {
+ let null = builder.ins().iconst(ref_ty, 0);
+ builder.ins().store(flags, null, dst, 0);
+ } else {
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+ builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);
+ }
+ return Ok(());
+ };
+
+ // Special case for `i31ref`s: no need for any barriers.
+ if let WasmHeapType::I31 = ty.heap_type {
+ return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags);
+ }
+
+ // Our initialization barrier for GC references being copied out of the
+ // stack and initializing a table/global/struct field/etc... is roughly
+ // equivalent to the following pseudo-CLIF:
+ //
+ // ```
+ // current_block:
+ // ...
+ // let new_val_is_null_or_i31 = ...
+ // brif new_val_is_null_or_i31, continue_block, inc_ref_block
+ //
+ // inc_ref_block:
+ // let ref_count = load new_val.ref_count
+ // let new_ref_count = iadd_imm ref_count, 1
+ // store new_val.ref_count, new_ref_count
+ // jump check_old_val_block
+ //
+ // continue_block:
+ // store dst, new_val
+ // ...
+ // ```
+ //
+ // This write barrier is responsible for ensuring that the new value's
+ // ref count is incremented now that the table/global/struct/etc... is
+ // holding onto it.
+
+ let current_block = builder.current_block().unwrap();
+ let inc_ref_block = builder.create_block();
+ let continue_block = builder.create_block();
+
+ builder.ensure_inserted_block();
+ builder.insert_block_after(inc_ref_block, current_block);
+ builder.insert_block_after(continue_block, inc_ref_block);
+
+ // Current block: check whether the new value is non-null and
+ // non-i31. If so, branch to the `inc_ref_block`.
+ log::trace!("DRC initialization barrier: check if the value is null or i31");
+ let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val);
+ builder.ins().brif(
+ new_val_is_null_or_i31,
+ continue_block,
+ &[],
+ inc_ref_block,
+ &[],
+ );
+
+ // Block to increment the ref count of the new value when it is non-null
+ // and non-i31.
+ builder.switch_to_block(inc_ref_block);
+ builder.seal_block(inc_ref_block);
+ log::trace!("DRC initialization barrier: increment the ref count of the initial value");
+ self.mutate_ref_count(func_env, builder, new_val, 1);
+ builder.ins().jump(continue_block, &[]);
+
+ // Join point after we're done with the GC barrier: do the actual store
+ // to initialize the field.
+ builder.switch_to_block(continue_block);
+ builder.seal_block(continue_block);
+ log::trace!(
+ "DRC initialization barrier: finally, store into {dst:?} to initialize the field"
+ );
+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?;
+
+ Ok(())
+ }
+
/// Write to an uninitialized field or element inside a GC object.
fn init_field(
&mut self,
diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/enabled/null.rs
index 70818ab69a..789bdfb8ea 100644
--- a/crates/cranelift/src/func_environ/gc/enabled/null.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled/null.rs
@@ -371,6 +371,18 @@ impl GcCompiler for NullCompiler {
unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)
}
+ fn translate_init_gc_reference(
+ &mut self,
+ _func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ty: WasmRefType,
+ dst: ir::Value,
+ new_val: ir::Value,
+ flags: ir::MemFlagsData,
+ ) -> WasmResult<()> {
+ unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)
+ }
+
fn init_field(
&mut self,
func_env: &mut FuncEnvironment<'_>,
diff --git a/crates/cranelift/src/translate/func_translator.rs b/crates/cranelift/src/translate/func_translator.rs
index b687574550..80323def5c 100644
--- a/crates/cranelift/src/translate/func_translator.rs
+++ b/crates/cranelift/src/translate/func_translator.rs
@@ -96,6 +96,26 @@ impl FuncTranslator {
log::trace!("translated Wasm to CLIF:\n{}", func.display());
Ok(())
}
+
+ /// Translate the `FuncKey::ModuleStartup` function.
+ pub fn translate_module_startup(
+ &mut self,
+ func: &mut ir::Function,
+ environ: &mut FuncEnvironment<'_>,
+ ) -> WasmResult<()> {
+ let mut builder = FunctionBuilder::new(func, &mut self.func_ctx);
+ let entry_block = builder.create_block();
+ builder.append_block_params_for_function_params(entry_block);
+ builder.switch_to_block(entry_block);
+ builder.seal_block(entry_block);
+ builder.ensure_inserted_block();
+ environ.before_translate_function(&mut builder)?;
+ environ.translate_module_startup(&mut builder)?;
+ environ.after_translate_function(&mut builder)?;
+ builder.ins().return_(&[]);
+ builder.finalize();
+ Ok(())
+ }
}
/// Declare local variables for the signature parameters that correspond to WebAssembly locals.
diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs
index d3cd69a872..4dc2fa6d01 100644
--- a/crates/environ/src/compile/mod.rs
+++ b/crates/environ/src/compile/mod.rs
@@ -5,7 +5,7 @@ use crate::error::Result;
use crate::prelude::*;
use crate::{
DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex,
- TripleExt, Tunables, WasmError, WasmFuncType, obj,
+ TripleExt, Tunables, WasmError, obj,
};
use object::write::{Object, SymbolId};
use object::{Architecture, BinaryFormat, FileFlags};
@@ -261,38 +261,11 @@ pub trait Compiler: Send + Sync {
///
/// The trampoline should save the necessary state to record the
/// host-to-Wasm transition (e.g. registers used for fast stack walking).
- fn compile_array_to_wasm_trampoline(
- &self,
- translation: &ModuleTranslation<'_>,
- types: &ModuleTypesBuilder,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError>;
-
- /// Compile a trampoline for a Wasm caller calling a array callee with the
- /// given signature.
- ///
- /// The trampoline should save the necessary state to record the
- /// Wasm-to-host transition (e.g. registers used for fast stack walking).
- fn compile_wasm_to_array_trampoline(
- &self,
- wasm_func_ty: &WasmFuncType,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError>;
-
- /// Creates a trampoline that can be used to call Wasmtime's implementation
- /// of the builtin function specified by `index`.
- ///
- /// The trampoline created can technically have any ABI but currently has
- /// the native ABI. This will then perform all the necessary duties of an
- /// exit trampoline from wasm and then perform the actual dispatch to the
- /// builtin function. Builtin functions in Wasmtime are stored in an array
- /// in all `VMContext` pointers, so the call to the host is an indirect
- /// call.
- fn compile_wasm_to_builtin(
+ fn compile_trampoline(
&self,
+ translation: Option<&ModuleTranslation<'_>>,
key: FuncKey,
+ types: &ModuleTypesBuilder,
symbol: &str,
) -> Result<CompiledFunctionBody, CompileError>;
diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs
index ca409dae97..c9e1f302ad 100644
--- a/crates/environ/src/compile/module_artifacts.rs
+++ b/crates/environ/src/compile/module_artifacts.rs
@@ -5,8 +5,7 @@ use crate::WasmChecksum;
use crate::error::{Result, bail};
use crate::prelude::*;
use crate::{
- CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata,
- ModuleTranslation, Tunables, obj,
+ CompiledModuleInfo, DebugInfoData, FunctionName, Metadata, ModuleTranslation, Tunables, obj,
};
use object::SectionKind;
use object::write::{Object, SectionId, StandardSegment, WritableBuffer};
@@ -118,9 +117,8 @@ impl<'a> ObjectBuilder<'a> {
mut module,
debuginfo,
has_unparsed_debuginfo,
- data,
data_align,
- passive_data,
+ runtime_data,
wasm,
..
} = translation;
@@ -128,20 +126,15 @@ impl<'a> ObjectBuilder<'a> {
// Place all data from the wasm module into a section which will the
// source of the data later at runtime. This additionally keeps track of
// the offset of
- let mut total_data_len = 0;
let data_offset = self
.obj
.append_section_data(self.data, &[], data_align.unwrap_or(1));
- for (i, data) in data.iter().enumerate() {
- // The first data segment has its alignment specified as the alignment
- // for the entire section, but everything afterwards is adjacent so it
- // has alignment of 1.
+ for (i, (_, data)) in runtime_data.iter().enumerate() {
+ // The first data segment has its alignment specified as the
+ // alignment for the entire section, but everything afterwards is
+ // adjacent so it has alignment of 1.
let align = if i == 0 { data_align.unwrap_or(1) } else { 1 };
self.obj.append_section_data(self.data, data, align);
- total_data_len += data.len();
- }
- for data in passive_data.iter() {
- self.obj.append_section_data(self.data, data, 1);
}
// If any names are present in the module then the `ELF_NAME_DATA` section
@@ -172,34 +165,12 @@ impl<'a> ObjectBuilder<'a> {
}
}
- // Data offsets in `MemoryInitialization` are offsets within the
- // `translation.data` list concatenated which is now present in the data
- // segment that's appended to the object. Increase the offsets by
- // `self.data_size` to account for any previously added module.
- let data_offset = u32::try_from(data_offset).unwrap();
- match &mut module.memory_initialization {
- MemoryInitialization::Segmented(list) => {
- for segment in list {
- segment.data.start = segment.data.start.checked_add(data_offset).unwrap();
- segment.data.end = segment.data.end.checked_add(data_offset).unwrap();
- }
- }
- MemoryInitialization::Static { map } => {
- for (_, segment) in map {
- if let Some(segment) = segment {
- segment.data.start = segment.data.start.checked_add(data_offset).unwrap();
- segment.data.end = segment.data.end.checked_add(data_offset).unwrap();
- }
- }
- }
- }
-
// Data offsets for passive data are relative to the start of
- // `translation.passive_data` which was appended to the data segment
+ // `translation.runtime_data` which was appended to the data segment
// of this object, after active data in `translation.data`. Update the
// offsets to account prior modules added in addition to active data.
- let data_offset = data_offset + u32::try_from(total_data_len).unwrap();
- for (_, range) in module.passive_data.iter_mut() {
+ let data_offset = u32::try_from(data_offset).unwrap();
+ for (_, range) in module.runtime_data.iter_mut() {
range.start = range.start.checked_add(data_offset).unwrap();
range.end = range.end.checked_add(data_offset).unwrap();
}
diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs
index be2fc09c0d..b5b02b9e6f 100644
--- a/crates/environ/src/compile/module_environ.rs
+++ b/crates/environ/src/compile/module_environ.rs
@@ -1,20 +1,19 @@
use crate::error::{OutOfMemory, Result, bail};
use crate::module::{
- FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment,
- TableSegmentElements,
+ FuncRefIndex, Initializer, MemoryInitialization, Module, TableSegment, TableSegmentElements,
};
use crate::prelude::*;
use crate::{
- ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex,
- EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex,
- ModuleInternedTypeIndex, ModuleTypesBuilder, PanicOnOom as _, PassiveDataIndex,
- PassiveElemIndex, PrimaryMap, SizeOverflow, StaticMemoryInitializer, StaticModuleIndex,
- TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError,
+ ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, DefinedGlobalIndex, ElemIndex,
+ EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType,
+ MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex, ModuleStartup, ModuleTypesBuilder,
+ PanicOnOom as _, PassiveElemIndex, PrimaryMap, RuntimeDataIndex, StaticModuleIndex, TableIndex,
+ TableInitialValue, TableInitialization, Tag, TagIndex, Tunables, TypeConvert, TypeIndex,
WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,
};
+use alloc::borrow::Cow;
use cranelift_entity::SecondaryMap;
use cranelift_entity::packed_option::ReservedValue;
-use std::borrow::Cow;
use std::collections::HashMap;
use std::mem;
use std::path::PathBuf;
@@ -84,13 +83,6 @@ pub struct ModuleTranslation<'data> {
/// configuration.
pub has_unparsed_debuginfo: bool,
- /// List of data segments found in this module which should be concatenated
- /// together for the final compiled artifact.
- ///
- /// These data segments, when concatenated, are indexed by the
- /// `MemoryInitializer` type.
- pub data: Vec<Cow<'data, [u8]>>,
-
/// The desired alignment of `data` in the final data section of the object
/// file that we'll emit.
///
@@ -100,20 +92,21 @@ pub struct ModuleTranslation<'data> {
pub data_align: Option<u64>,
/// Map from a data segment to whether it's a passive data segment or not.
- pub passive_data_map: SecondaryMap<DataIndex, Option<PassiveDataIndex>>,
+ pub runtime_data_map: SecondaryMap<DataIndex, Option<RuntimeDataIndex>>,
/// Map from an elem segment to whether it's a passive elem segment or not.
pub passive_elem_map: SecondaryMap<ElemIndex, Option<PassiveElemIndex>>,
- /// Total size of all data pushed onto `data` so far.
- total_data: u32,
-
/// List of passive element segments found in this module which will get
/// concatenated for the final artifact.
- pub passive_data: Vec<&'data [u8]>,
+ pub runtime_data: PrimaryMap<RuntimeDataIndex, Cow<'data, [u8]>>,
- /// Total size of all passive data pushed into `passive_data` so far.
- total_passive_data: u32,
+ /// Record of all passive data segments that this module contains.
+ ///
+ /// These are processed during [`ModuleTranslation::finalize_memory_init`]
+ /// and eventually moved over into the `runtime_data` list above. Until
+ /// then, however, their `RuntimeDataIndex` is not yet assigned.
+ passive_data: Vec<(DataIndex, &'data [u8])>,
/// When we're parsing the code section this will be incremented so we know
/// which function is currently being defined.
@@ -122,6 +115,62 @@ pub struct ModuleTranslation<'data> {
/// The type information of the current module made available at the end of the
/// validation process.
types: Option<Types>,
+
+ /// The WebAssembly `start` function, if defined.
+ pub start_func: Option<FuncIndex>,
+
+ /// Initializers for `global` values which aren't considered "simple".
+ ///
+ /// These initializers are later compiled into a "module startup" function.
+ pub global_initializers: Vec<(DefinedGlobalIndex, ConstExpr)>,
+
+ /// Definitions of all passive elements found within a module.
+ ///
+ /// This maps passive element segments to their definition, either functions
+ /// or expressions-basd.
+ pub passive_elements: PrimaryMap<PassiveElemIndex, TableSegmentElements>,
+
+ /// WebAssembly table initialization data, per table.
+ ///
+ /// This keeps track of all per-table initialization (e.g. initial value for
+ /// non-null tables) as well as active element segments. This is processed
+ /// and refined by [`ModuleTranslation::finalize_table_init`] after
+ /// translation.
+ pub table_initialization: TableInitialization,
+
+ /// WebAssembly memory initialization.
+ ///
+ /// This is held here in an `Unprocessed` form during translation, and then
+ /// this is later finished with [`ModuleTranslation::finalize_memory_init`].
+ pub memory_init: MemoryInit<'data>,
+}
+
+/// Different forms of memory initialization that happens for a module.
+pub enum MemoryInit<'a> {
+ /// Raw active data segments that are being applied for an instance.
+ ///
+ /// This list contains the raw data which hasn't yet been processed into
+ /// `RuntimeDataIndex`, for example. This is later processed during
+ /// [`ModuleTranslation::finalize_memory_init`] to optionally shuffle things
+ /// around.
+ Unprocessed(Vec<MemoryInitializer<'a>>),
+
+ /// Finalized memory initialization to be executed after
+ /// [`ModuleTranslation::finalize_memory_init`] has run. This represents
+ /// active data segments which may have been merged from the `Unprocessed`
+ /// list above, and may or may not have statically know offsets.
+ Processed(Vec<(MemoryIndex, MemorySegmentOffset, RuntimeDataIndex)>),
+}
+
+/// Offset within [`MemoryInit::Processed`] which indicates the initial offset
+/// a data segment is applied at.
+pub enum MemorySegmentOffset {
+ /// A "complicated" constant expression deferred to get evaluated at runtime
+ /// with compiled code.
+ Expr(ConstExpr),
+
+ /// A statically known, in-bounds, constant value.
+ Static(u64),
}
impl<'data> ModuleTranslation<'data> {
@@ -136,15 +185,18 @@ impl<'data> ModuleTranslation<'data> {
exported_signatures: Vec::default(),
debuginfo: DebugInfoData::default(),
has_unparsed_debuginfo: false,
- data: Vec::default(),
data_align: None,
- total_data: 0,
- passive_data: Vec::default(),
- total_passive_data: 0,
+ runtime_data: Default::default(),
code_index: 0,
types: None,
- passive_data_map: Default::default(),
+ runtime_data_map: Default::default(),
passive_elem_map: Default::default(),
+ start_func: None,
+ global_initializers: Vec::new(),
+ passive_elements: Default::default(),
+ table_initialization: Default::default(),
+ memory_init: MemoryInit::Unprocessed(Vec::new()),
+ passive_data: Default::default(),
}
}
@@ -417,9 +469,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
self.result.module.needs_gc_heap |= table.ref_type.is_vmgcref_type();
self.result.module.tables.push(table)?;
let init = match init {
- wasmparser::TableInit::RefNull => TableInitialValue::Null {
- precomputed: TryVec::new(),
- },
+ wasmparser::TableInit::RefNull => TableInitialValue::Null,
wasmparser::TableInit::Expr(expr) => {
let (init, escaped) = ConstExpr::from_wasmparser(self, expr)?;
for f in escaped {
@@ -428,11 +478,11 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
TableInitialValue::Expr(init)
}
};
+ self.result.table_initialization.initial_values.push(init)?;
self.result
.module
.table_initialization
- .initial_values
- .push(init)?;
+ .push(Default::default())?;
}
}
@@ -475,8 +525,24 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
self.flag_func_escaped(f);
}
let ty = self.convert_global_type(&ty)?;
- self.result.module.globals.push(ty)?;
- self.result.module.global_initializers.push(initializer)?;
+ let index = self.result.module.globals.push(ty)?;
+ let defined_index = self.result.module.defined_global_index(index).unwrap();
+ match initializer.const_eval() {
+ Some(val) => {
+ self.result
+ .module
+ .global_initializers
+ .push((defined_index, val))?;
+ }
+ None => {
+ // "Complicated" global initializers are deferred
+ // to get evaluated in the startup function.
+ self.require_startup_func();
+ self.result
+ .global_initializers
+ .push((defined_index, initializer));
+ }
+ }
}
}
@@ -508,9 +574,12 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
self.validator.start_section(func, &range)?;
let func_index = FuncIndex::from_u32(func);
- self.flag_func_escaped(func_index);
- debug_assert!(self.result.module.start_func.is_none());
- self.result.module.start_func = Some(func_index);
+ debug_assert!(self.result.start_func.is_none());
+ self.result.start_func = Some(func_index);
+
+ // To make startup a bit easier, invoking the `start` function
+ // is a responsibility deferred to the startup function.
+ self.require_startup_func();
}
Payload::ElementSection(elements) => {
@@ -566,19 +635,27 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;
debug_assert!(escaped.is_empty());
- self.result.module.table_initialization.segments.push(
- TableSegment {
+ self.result
+ .table_initialization
+ .segments
+ .push(TableSegment {
table_index,
offset,
elements,
- },
- )?;
+ })?;
None
}
ElementKind::Passive => {
- let passive_index =
- self.result.module.passive_elements.push(elements)?;
+ let passive_index = self
+ .result
+ .module
+ .passive_elements
+ .push((elements.ty(), elements.len()))?;
+ self.result.passive_elements.push(elements);
+ // One-time initialization of passive element
+ // segments is deferred to the startup function.
+ self.require_startup_func();
Some(passive_index)
}
@@ -639,14 +716,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
Payload::DataSection(data) => {
self.validator.data_section(&data)?;
- let initializers = match &mut self.result.module.memory_initialization {
- MemoryInitialization::Segmented(i) => i,
- _ => unreachable!(),
- };
-
- let cnt = usize::try_from(data.count()).unwrap();
- initializers.reserve_exact(cnt)?;
- self.result.data.reserve_exact(cnt);
+ assert!(self.result.module.memory_initialization.is_segmented());
for (index, entry) in data.into_iter().enumerate() {
let wasmparser::Data {
@@ -655,53 +725,28 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
range: _,
} = entry?;
let data_index = DataIndex::from_u32(index.try_into().unwrap());
- let mk_range = |total: &mut u32| -> Result<_, WasmError> {
- let range = u32::try_from(data.len())
- .ok()
- .and_then(|size| {
- let start = *total;
- let end = start.checked_add(size)?;
- Some(start..end)
- })
- .ok_or_else(|| {
- WasmError::Unsupported(format!(
- "more than 4 gigabytes of data in wasm module",
- ))
- })?;
- *total += range.end - range.start;
- Ok(range)
- };
- let passive_index = match kind {
+ match kind {
DataKind::Active {
memory_index,
offset_expr,
} => {
- let range = mk_range(&mut self.result.total_data)?;
let memory_index = MemoryIndex::from_u32(memory_index);
let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;
debug_assert!(escaped.is_empty());
- let initializers = match &mut self.result.module.memory_initialization {
- MemoryInitialization::Segmented(i) => i,
- _ => unreachable!(),
+ let MemoryInit::Unprocessed(list) = &mut self.result.memory_init else {
+ panic!("memory initializers should be unprocessed at this point");
};
- initializers.push(MemoryInitializer {
+ list.push(MemoryInitializer {
memory_index,
offset,
- data: range,
- })?;
- self.result.data.push(data.into());
- None
+ data,
+ });
}
DataKind::Passive => {
- let range = mk_range(&mut self.result.total_passive_data)?;
- self.result.passive_data.push(data);
- Some(self.result.module.passive_data.push(range)?)
+ self.result.passive_data.push((data_index, data));
}
- };
- self.result
- .passive_data_map
- .insert(data_index, passive_index);
+ }
}
}
@@ -952,6 +997,10 @@ and for re-adding support for interface types you can see this issue:
}
Ok(())
}
+
+ fn require_startup_func(&mut self) {
+ self.result.require_startup_func(self.types);
+ }
}
impl TypeConvert for ModuleEnvironment<'_, '_> {
@@ -971,6 +1020,75 @@ impl TypeConvert for ModuleEnvironment<'_, '_> {
}
impl ModuleTranslation<'_> {
+ /// Called after translation is complete this will finalize the memory
+ /// initialization strategy for this module.
+ ///
+ /// This will notably use `Self::try_static_init` to attempt to massage
+ /// data segments to being CoW-init-friendly. Afterwards the
+ /// `self.memory_init` field is transitioned from `Unprocessed` to
+ /// `Processed`.
+ pub fn finalize_memory_init(
+ &mut self,
+ tunables: &Tunables,
+ page_size: u64,
+ max_image_size_always_allowed: u64,
+ types: &mut ModuleTypesBuilder,
+ ) {
+ if tunables.memory_init_cow {
+ self.try_static_init(page_size, max_image_size_always_allowed);
+ }
+
+ // If any memory is statically initialized, and if that memory has an
+ // initial data segment, then a startup function is at least
+ // conditionally needed if the memory needs initialization. Flag as such
+ // here.
+ if let MemoryInitialization::Static { map } = &self.module.memory_initialization {
+ if map.iter().any(|(_, v)| v.is_some()) {
+ self.require_startup_func_if_memories_need_init(types);
+ }
+ }
+
+ // If, after `try_static_init`, initializers are still `Unprocessed`
+ // then this is the catch-all fallback path for initialization. All
+ // segments are promoted into `self.runtime_data` and then the
+ // initialization is rewritten to `Processed`.
+ if let MemoryInit::Unprocessed(list) = &mut self.memory_init {
+ let segments = mem::take(list);
+ let mut new_initializers = Vec::new();
+ for segment in segments {
+ new_initializers.push((
+ segment.memory_index,
+ MemorySegmentOffset::Expr(segment.offset),
+ self.runtime_data.push(segment.data.into()),
+ ));
+ }
+ if !new_initializers.is_empty() {
+ self.require_startup_func(types);
+ }
+ self.memory_init = MemoryInit::Processed(new_initializers);
+ }
+
+ // At this point append all passive data to the `runtime_data` list.
+ // This notably occurs after `try_static_init` above to ensure that the
+ // page-aligned data for static initialization, if applicable, comes
+ // first.
+ for (data_index, segment) in self.passive_data.iter() {
+ let runtime_index = self.runtime_data.push((*segment).into());
+ self.runtime_data_map
+ .insert(*data_index, Some(runtime_index));
+ }
+
+ // And, finally, record all chunks from `self.runtime_data` within
+ // `self.module.runtime_data` as well.
+ let mut cur = 0;
+ for (idx, data) in self.runtime_data.iter() {
+ let len = u32::try_from(data.len()).unwrap();
+ let i = self.module.runtime_data.push(cur..cur + len).panic_on_oom();
+ cur += len;
+ assert_eq!(idx, i);
+ }
+ }
+
/// Attempts to convert segmented memory initialization into static
/// initialization for the module that this translation represents.
///
@@ -1001,24 +1119,20 @@ impl ModuleTranslation<'_> {
/// modules that have some dynamically-placed data segments. But,
/// for now, this is sufficient to allow a system that "knows what
/// it's doing" to always get static init.
- pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) {
- // This method only attempts to transform a `Segmented` memory init
- // into a `Static` one, no other state.
- if !self.module.memory_initialization.is_segmented() {
- return;
- }
+ fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) {
+ let segments = match &mut self.memory_init {
+ MemoryInit::Unprocessed(list) => list,
+ _ => return,
+ };
// First a dry run of memory initialization is performed. This
// collects information about the extent of memory initialized for each
// memory as well as the size of all data segments being copied in.
- struct Memory {
+ struct Memory<'a> {
data_size: u64,
min_addr: u64,
max_addr: u64,
- // The `usize` here is a pointer into `self.data` which is the list
- // of data segments corresponding to what was found in the original
- // wasm module.
- segments: Vec<(usize, StaticMemoryInitializer)>,
+ segments: Vec<(u64, &'a [u8])>,
}
let mut info = PrimaryMap::with_capacity(self.module.memories.len());
for _ in 0..self.module.memories.len() {
@@ -1030,59 +1144,64 @@ impl ModuleTranslation<'_> {
});
}
- struct InitMemoryAtCompileTime<'a> {
- module: &'a Module,
- info: &'a mut PrimaryMap<MemoryIndex, Memory>,
- idx: usize,
- }
- impl InitMemory for InitMemoryAtCompileTime<'_> {
- fn memory_size_in_bytes(
- &mut self,
- memory_index: MemoryIndex,
- ) -> Result<u64, SizeOverflow> {
- self.module.memories[memory_index].minimum_byte_size()
+ for initializer in segments.iter() {
+ let &MemoryInitializer {
+ memory_index,
+ ref offset,
+ ref data,
+ } = initializer;
+
+ // Currently `Static` only applies to locally-defined memories,
+ // so if a data segment references an imported memory then
+ // transitioning to a `Static` memory initializer is not
+ // possible.
+ if self.module.defined_memory_index(memory_index).is_none() {
+ return;
}
- fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option<u64> {
- match (expr.ops(), self.module.memories[memory_index].idx_type) {
- (&[ConstOp::I32Const(offset)], IndexType::I32) => {
- Some(offset.cast_unsigned().into())
+ // First up determine the start/end range and verify that they're
+ // in-bounds for the initial size of the memory at `memory_index`.
+ // Note that this can bail if we don't have access to globals yet
+ // (e.g. this is a task happening before instantiation at
+ // compile-time).
+ let start = match (offset.ops(), self.module.memories[memory_index].idx_type) {
+ (&[ConstOp::I32Const(offset)], IndexType::I32) => offset.cast_unsigned().into(),
+ (&[ConstOp::I64Const(offset)], IndexType::I64) => offset.cast_unsigned(),
+ _ => return,
+ };
+ let len = u64::try_from(data.len()).unwrap();
+ let end = match start.checked_add(len) {
+ Some(end) => end,
+ None => return,
+ };
+
+ match self.module.memories[memory_index].minimum_byte_size() {
+ Ok(max) => {
+ if end > max {
+ return;
}
- (&[ConstOp::I64Const(offset)], IndexType::I64) => Some(offset.cast_unsigned()),
- _ => None,
}
+
+ // Note that computing the minimum can overflow if the page
+ // size is the default 64KiB and the memory's minimum size in
+ // pages is `1 << 48`, the maximum number of minimum pages for
+ // 64-bit memories. We don't return `false` to signal an error
+ // here and instead defer the error to runtime, when it will be
+ // impossible to allocate that much memory anyways.
+ Err(_) => return,
}
- fn write(&mut self, memory: MemoryIndex, init: &StaticMemoryInitializer) -> bool {
- // Currently `Static` only applies to locally-defined memories,
- // so if a data segment references an imported memory then
- // transitioning to a `Static` memory initializer is not
- // possible.
- if self.module.defined_memory_index(memory).is_none() {
- return false;
- };
- let info = &mut self.info[memory];
- let data_len = u64::from(init.data.end - init.data.start);
- if data_len > 0 {
- info.data_size += data_len;
- info.min_addr = info.min_addr.min(init.offset);
- info.max_addr = info.max_addr.max(init.offset + data_len);
- info.segments.push((self.idx, init.clone()));
- }
- self.idx += 1;
- true
+ // Skip empty in-bounds data segments.
+ if data.is_empty() {
+ continue;
}
- }
- let ok = self
- .module
- .memory_initialization
- .init_memory(&mut InitMemoryAtCompileTime {
- idx: 0,
- module: &self.module,
- info: &mut info,
- });
- if !ok {
- return;
+
+ let info = &mut info[memory_index];
+ let len64 = u64::try_from(data.len()).unwrap();
+ info.data_size += len64;
+ info.min_addr = info.min_addr.min(start);
+ info.max_addr = info.max_addr.max(start + len64);
+ info.segments.push((start, data));
}
// Validate that the memory information collected is indeed valid for
@@ -1127,9 +1246,8 @@ impl ModuleTranslation<'_> {
// Here's where we've now committed to changing to static memory. The
// memory initialization image is built here from the page data and then
// it's converted to a single initializer.
- let data = mem::replace(&mut self.data, Vec::new());
let mut map = TryPrimaryMap::with_capacity(info.len()).panic_on_oom();
- let mut module_data_size = 0u32;
+ let mut new_initializers = Vec::new();
for (memory, info) in info.iter() {
// Create the in-memory `image` which is the initialized contents of
// this linear memory.
@@ -1139,10 +1257,8 @@ impl ModuleTranslation<'_> {
0
};
let mut image = Vec::with_capacity(extent);
- for (idx, init) in info.segments.iter() {
- let data = &data[*idx];
- assert_eq!(data.len(), init.data.len());
- let offset = usize::try_from(init.offset - info.min_addr).unwrap();
+ for (offset, data) in info.segments.iter() {
+ let offset = usize::try_from(*offset - info.min_addr).unwrap();
if image.len() < offset {
image.resize(offset, 0u8);
image.extend_from_slice(data);
@@ -1181,50 +1297,72 @@ impl ModuleTranslation<'_> {
// the front and back with extra zeros as necessary
if offset % page_size != 0 {
let zero_padding = offset % page_size;
- self.data.push(vec![0; zero_padding as usize].into());
+ image.splice(0..0, std::iter::repeat(0).take(zero_padding as usize));
offset -= zero_padding;
len += zero_padding;
}
- self.data.push(image.into());
if len % page_size != 0 {
let zero_padding = page_size - (len % page_size);
- self.data.push(vec![0; zero_padding as usize].into());
+ image.extend(std::iter::repeat(0).take(zero_padding as usize));
len += zero_padding;
}
+ let runtime_index = if image.is_empty() {
+ None
+ } else {
+ Some(self.runtime_data.push(image.into()))
+ };
// Offset/length should now always be page-aligned.
assert!(offset % page_size == 0);
assert!(len % page_size == 0);
- // Create the `StaticMemoryInitializer` which describes this image,
+ // Record the static memory initializer which describes this image,
// only needed if the image is actually present and has a nonzero
// length. The `offset` has been calculates above, originally
// sourced from `info.min_addr`. The `data` field is the extent
// within the final data segment we'll emit to an ELF image, which
// is the concatenation of `self.data`, so here it's the size of
// the section-so-far plus the current segment we're appending.
- let len = u32::try_from(len).unwrap();
- let init = if len > 0 {
- Some(StaticMemoryInitializer {
- offset,
- data: module_data_size..module_data_size + len,
- })
- } else {
- None
- };
- let idx = map.push(init).panic_on_oom();
+ let idx = map.push(runtime_index.map(|i| (offset, i))).panic_on_oom();
assert_eq!(idx, memory);
- module_data_size += len;
+ if let Some(runtime_index) = runtime_index {
+ new_initializers.push((idx, MemorySegmentOffset::Static(offset), runtime_index));
+ }
}
self.data_align = Some(page_size);
self.module.memory_initialization = MemoryInitialization::Static { map };
+ self.memory_init = MemoryInit::Processed(new_initializers);
+ }
+
+ /// Finalizes the initialization of tables.
+ ///
+ /// This is invoked after translation and notably uses
+ /// `Self::try_func_table_init` to attempt to optimize initialization of
+ /// tables into static precomputed images.
+ pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) {
+ if tunables.table_lazy_init {
+ self.try_func_table_init();
+ }
+
+ // If any table has a non-null initializers, or if there's any active
+ // data segments, then a startup function is unconditionally required to
+ // configure the table.
+ if self
+ .table_initialization
+ .initial_values
+ .iter()
+ .any(|(_, v)| !matches!(v, TableInitialValue::Null))
+ || !self.table_initialization.segments.is_empty()
+ {
+ self.require_startup_func(types);
+ }
}
/// Attempts to convert the module's table initializers to
/// FuncTable form where possible. This enables lazy table
/// initialization later by providing a one-to-one map of initial
/// table values, without having to parse all segments.
- pub fn try_func_table_init(&mut self) {
+ fn try_func_table_init(&mut self) {
// This should be large enough to support very large Wasm
// modules with huge funcref tables, but small enough to avoid
// OOMs or DoS on truly sparse tables.
@@ -1232,32 +1370,27 @@ impl ModuleTranslation<'_> {
// First convert any element-initialized tables to images of just that
// single function if the minimum size of the table allows doing so.
- for ((_, init), (_, table)) in self
- .module
- .table_initialization
- .initial_values
- .iter_mut()
- .zip(
- self.module
- .tables
- .iter()
- .skip(self.module.num_imported_tables),
- )
- {
+ for ((i, init), (_, table)) in self.table_initialization.initial_values.iter_mut().zip(
+ self.module
+ .tables
+ .iter()
+ .skip(self.module.num_imported_tables),
+ ) {
let table_size = table.limits.min;
if table_size > MAX_FUNC_TABLE_SIZE {
continue;
}
if let TableInitialValue::Expr(expr) = init {
if let [ConstOp::RefFunc(f)] = expr.ops() {
- *init = TableInitialValue::Null {
- precomputed: try_vec![*f; table_size as usize].panic_on_oom(),
- };
+ assert!(self.module.table_initialization[i].is_empty());
+ self.module.table_initialization[i] =
+ try_vec![*f; table_size as usize].panic_on_oom();
+ *init = TableInitialValue::Null;
}
}
}
- let mut segments = mem::take(&mut self.module.table_initialization.segments)
+ let mut segments = mem::take(&mut self.table_initialization.segments)
.into_iter()
.peekable();
@@ -1326,18 +1459,18 @@ impl ModuleTranslation<'_> {
TableSegmentElements::Expressions { .. } => break,
};
- let precomputed =
- match &mut self.module.table_initialization.initial_values[defined_index] {
- TableInitialValue::Null { precomputed } => precomputed,
+ match &self.table_initialization.initial_values[defined_index] {
+ TableInitialValue::Null => {}
- // If this table is still listed as an initial value here
- // then that means the initial size of the table doesn't
- // support a precomputed function list, so skip this.
- // Technically this won't trap so it's possible to process
- // further initializers, but that's left as a future
- // optimization.
- TableInitialValue::Expr(_) => break,
- };
+ // If this table is still listed as an initial value here
+ // then that means the initial size of the table doesn't
+ // support a precomputed function list, so skip this.
+ // Technically this won't trap so it's possible to process
+ // further initializers, but that's left as a future
+ // optimization.
+ TableInitialValue::Expr(_) => break,
+ }
+ let precomputed = &mut self.module.table_initialization[defined_index];
// At this point we're committing to pre-initializing the table
// with the `segment` that's being iterated over. This segment is
@@ -1355,6 +1488,27 @@ impl ModuleTranslation<'_> {
// advance the iterator to see the next segment
let _ = segments.next();
}
- self.module.table_initialization.segments = segments.try_collect().panic_on_oom();
+ self.table_initialization.segments = segments.try_collect().panic_on_oom();
+ }
+
+ /// Helper function to ratchet the `startup` function for this module as
+ /// `Always`.
+ fn require_startup_func(&mut self, types: &mut ModuleTypesBuilder) {
+ let ty = match self.module.startup {
+ ModuleStartup::None => types.startup_func_type().into(),
+ ModuleStartup::Always(_) => return,
+ ModuleStartup::IfMemoriesNeedInit(ty) => ty,
+ };
+ self.module.startup = ModuleStartup::Always(ty);
+ }
+
+ /// Helper function to ratchet the `startup` function for this module as
+ /// `IfMemoriesNeedInit`.
+ fn require_startup_func_if_memories_need_init(&mut self, types: &mut ModuleTypesBuilder) {
+ let ty = match self.module.startup {
+ ModuleStartup::None => types.startup_func_type().into(),
+ ModuleStartup::Always(_) | ModuleStartup::IfMemoriesNeedInit(_) => return,
+ };
+ self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty);
}
}
diff --git a/crates/environ/src/compile/module_types.rs b/crates/environ/src/compile/module_types.rs
index a82843dc9a..082b3cab23 100644
--- a/crates/environ/src/compile/module_types.rs
+++ b/crates/environ/src/compile/module_types.rs
@@ -2,7 +2,7 @@ use crate::{
EngineOrModuleTypeIndex, EntityRef, ModuleInternedRecGroupIndex, ModuleInternedTypeIndex,
ModuleTypes, PanicOnOom as _, TypeConvert, TypeIndex, WasmArrayType, WasmCompositeInnerType,
WasmCompositeType, WasmExnType, WasmFieldType, WasmFuncType, WasmHeapType, WasmResult,
- WasmStorageType, WasmStructType, WasmSubType,
+ WasmStorageType, WasmStructType, WasmSubType, WasmValType,
collections::{TryClone as _, TryCow},
wasm_unsupported,
};
@@ -59,6 +59,9 @@ pub struct ModuleTypesBuilder {
/// If we are in the middle of defining a recursion group, this is the
/// metadata about the recursion group we started defining.
defining_rec_group: Option<RecGroupStart>,
+
+ /// Cache of the return value of [`Self::startup_func_type`].
+ startup_func_type: Option<ModuleInternedTypeIndex>,
}
impl ModuleTypesBuilder {
@@ -72,6 +75,7 @@ impl ModuleTypesBuilder {
wasmparser_to_wasmtime: HashMap::default(),
already_seen: HashMap::default(),
defining_rec_group: None,
+ startup_func_type: None,
}
}
@@ -462,6 +466,45 @@ impl ModuleTypesBuilder {
}
Ok(composite_type.inner.unwrap_func())
}
+
+ /// Gets a type for the "module startup" function, or `[] -> []` in the wasm
+ /// type system.
+ pub fn startup_func_type(&mut self) -> ModuleInternedTypeIndex {
+ *self.startup_func_type.get_or_insert_with(|| {
+ let idx = self.types.push(WasmSubType {
+ is_final: true,
+ supertype: None,
+ composite_type: WasmCompositeType {
+ inner: WasmCompositeInnerType::Func(WasmFuncType::new([], []).panic_on_oom()),
+ shared: false,
+ },
+ });
+ let next = self.types.next_ty();
+ self.types.push_rec_group(idx..next);
+ idx
+ })
+ }
+
+ /// Smaller helper method to find a `ModuleInternedTypeIndex` which
+ /// corresponds to the `resource.drop` intrinsic in components, namely a
+ /// core wasm function type which takes one `i32` argument and has no
+ /// results.
+ ///
+ /// This is a bit of a hack right now as ideally this find operation
+ /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself
+ /// would be threaded through appropriately, but that's left for a future
+ /// refactoring. Try not to lean too hard on this method though.
+ pub fn find_resource_drop_signature(&self) -> Option<ModuleInternedTypeIndex> {
+ self.wasm_types()
+ .find(|(_, ty)| {
+ ty.as_func().map_or(false, |sig| {
+ sig.params().len() == 1
+ && sig.results().len() == 0
+ && sig.params()[0] == WasmValType::I32
+ })
+ })
+ .map(|(i, _)| i)
+ }
}
// Forward the indexing impl to the internal `ModuleTypes`
diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs
index 9e98fcac64..2bfc28a361 100644
--- a/crates/environ/src/component/compiler.rs
+++ b/crates/environ/src/component/compiler.rs
@@ -10,7 +10,7 @@ pub trait ComponentCompiler: Send + Sync {
/// Each trampoline is a member of the `Trampoline` enumeration and has a
/// unique purpose and is translated differently. See the implementation of
/// this trait for Cranelift for more information.
- fn compile_trampoline(
+ fn compile_component_trampoline(
&self,
component: &ComponentTranslation,
types: &ComponentTypesBuilder,
diff --git a/crates/environ/src/component/types_builder.rs b/crates/environ/src/component/types_builder.rs
index 3f8f7d9495..d6cd8ec85b 100644
--- a/crates/environ/src/component/types_builder.rs
+++ b/crates/environ/src/component/types_builder.rs
@@ -2,8 +2,8 @@ use crate::component::*;
use crate::error::{Result, bail};
use crate::prelude::*;
use crate::{
- EngineOrModuleTypeIndex, EntityType, ModuleInternedTypeIndex, ModuleTypes, ModuleTypesBuilder,
- PrimaryMap, TypeConvert, WasmHeapType, WasmValType,
+ EngineOrModuleTypeIndex, EntityType, ModuleTypes, ModuleTypesBuilder, PrimaryMap, TypeConvert,
+ WasmHeapType,
};
use cranelift_entity::EntityRef;
use std::collections::HashMap;
@@ -163,28 +163,6 @@ impl ComponentTypesBuilder {
(self.component_types, ty)
}
- /// Smaller helper method to find a `ModuleInternedTypeIndex` which
- /// corresponds to the `resource.drop` intrinsic in components, namely a
- /// core wasm function type which takes one `i32` argument and has no
- /// results.
- ///
- /// This is a bit of a hack right now as ideally this find operation
- /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself
- /// would be threaded through appropriately, but that's left for a future
- /// refactoring. Try not to lean too hard on this method though.
- pub fn find_resource_drop_signature(&self) -> Option<ModuleInternedTypeIndex> {
- self.module_types
- .wasm_types()
- .find(|(_, ty)| {
- ty.as_func().map_or(false, |sig| {
- sig.params().len() == 1
- && sig.results().len() == 0
- && sig.params()[0] == WasmValType::I32
- })
- })
- .map(|(i, _)| i)
- }
-
/// Returns the underlying builder used to build up core wasm module types.
///
/// Note that this is shared across all modules found within a component to
diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs
index 8f0fd0e794..66bc7c189a 100644
--- a/crates/environ/src/key.rs
+++ b/crates/environ/src/key.rs
@@ -46,6 +46,10 @@ pub enum FuncKeyKind {
/// A Wasmtime unsafe intrinsic function.
#[cfg(feature = "component-model")]
UnsafeIntrinsic = FuncKey::new_kind(0b1000),
+
+ /// Initialization function for a module, such as initializing "complicated"
+ /// globals and passive element segments.
+ ModuleStartup = FuncKey::new_kind(0b1001),
}
impl From<FuncKeyKind> for u32 {
@@ -73,6 +77,7 @@ impl FuncKeyKind {
Self::PatchableToBuiltinTrampoline
}
x if x == Self::PulleyHostCall.into() => Self::PulleyHostCall,
+ x if x == Self::ModuleStartup.into() => Self::ModuleStartup,
#[cfg(feature = "component-model")]
x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline,
@@ -125,7 +130,9 @@ impl FuncKeyNamespace {
/// Panics when given invalid raw representations.
pub fn from_raw(raw: u32) -> Self {
match FuncKeyKind::from_raw(raw & FuncKey::KIND_MASK) {
- FuncKeyKind::DefinedWasmFunction | FuncKeyKind::ArrayToWasmTrampoline => Self(raw),
+ FuncKeyKind::DefinedWasmFunction
+ | FuncKeyKind::ArrayToWasmTrampoline
+ | FuncKeyKind::ModuleStartup => Self(raw),
FuncKeyKind::WasmToArrayTrampoline
| FuncKeyKind::WasmToBuiltinTrampoline
| FuncKeyKind::PatchableToBuiltinTrampoline
@@ -205,7 +212,6 @@ pub enum Abi {
Patchable = 2,
}
-#[cfg(feature = "component-model")]
impl Abi {
fn from_raw(raw: u32) -> Self {
match raw {
@@ -254,6 +260,13 @@ pub enum FuncKey {
/// A Wasmtime intrinsic function.
#[cfg(feature = "component-model")]
UnsafeIntrinsic(Abi, component::UnsafeIntrinsic),
+
+ /// Initialization function for a module, such as initializing "complicated"
+ /// globals and passive element segments.
+ ///
+ /// This function has the `Abi` specified and will initialize the module
+ /// specified.
+ ModuleStartup(Abi, StaticModuleIndex),
}
impl Ord for FuncKey {
@@ -342,6 +355,13 @@ impl FuncKey {
let index = intrinsic.index();
(namespace, index)
}
+
+ FuncKey::ModuleStartup(abi, module) => {
+ assert_eq!(module.as_u32() & Self::KIND_MASK, 0);
+ let namespace = FuncKeyKind::ModuleStartup.into_raw() | module.as_u32();
+ let index = abi.into_raw();
+ (namespace, index)
+ }
};
(FuncKeyNamespace(namespace), FuncKeyIndex(index))
}
@@ -376,6 +396,7 @@ impl FuncKey {
FuncKey::ResourceDropTrampoline => Abi::Wasm,
#[cfg(feature = "component-model")]
FuncKey::UnsafeIntrinsic(abi, _) => abi,
+ FuncKey::ModuleStartup(abi, _) => abi,
}
}
@@ -462,6 +483,12 @@ impl FuncKey {
let intrinsic = component::UnsafeIntrinsic::from_u32(b);
Self::UnsafeIntrinsic(abi, intrinsic)
}
+
+ FuncKeyKind::ModuleStartup => {
+ let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK);
+ let abi = Abi::from_raw(b);
+ Self::ModuleStartup(abi, module)
+ }
}
}
@@ -556,7 +583,9 @@ impl FuncKey {
/// looking up the StoreCode (or having access to the Store).
pub fn is_store_invariant(&self) -> bool {
match self {
- Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false,
+ Self::DefinedWasmFunction(..)
+ | Self::ArrayToWasmTrampoline(..)
+ | Self::ModuleStartup(..) => false,
Self::WasmToArrayTrampoline(..)
| Self::WasmToBuiltinTrampoline(..)
| Self::PatchableToBuiltinTrampoline(..)
diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs
index 1983b9729b..45e228449b 100644
--- a/crates/environ/src/module.rs
+++ b/crates/environ/src/module.rs
@@ -7,8 +7,8 @@ use cranelift_entity::{EntityRef, packed_option::ReservedValue};
use serde_derive::{Deserialize, Serialize};
/// A WebAssembly linear memory initializer.
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct MemoryInitializer {
+#[derive(Clone, Debug)]
+pub struct MemoryInitializer<'a> {
/// The index of a linear memory to initialize.
pub memory_index: MemoryIndex,
/// The base offset to start this segment at.
@@ -17,19 +17,7 @@ pub struct MemoryInitializer {
///
/// This range indexes into a separately stored data section which will be
/// provided with the compiled module's code as well.
- pub data: Range<u32>,
-}
-
-/// Similar to the above `MemoryInitializer` but only used when memory
-/// initializers are statically known to be valid.
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct StaticMemoryInitializer {
- /// The 64-bit offset, in bytes, of where this initializer starts.
- pub offset: u64,
-
- /// The range of data to write at `offset`, where these indices are indexes
- /// into the compiled wasm module's data section.
- pub data: Range<u32>,
+ pub data: &'a [u8],
}
/// The type of WebAssembly linear memory initialization to use for a module.
@@ -47,7 +35,7 @@ pub enum MemoryInitialization {
/// data segments when the module is instantiated.
///
/// This is the default memory initialization type.
- Segmented(TryVec<MemoryInitializer>),
+ Segmented,
/// Memory initialization is statically known and involves a single `memcpy`
/// or otherwise simply making the defined data visible.
@@ -81,13 +69,13 @@ pub enum MemoryInitialization {
///
/// The offset, range base, and range end are all guaranteed to be page
/// aligned to the page size passed in to `try_static_init`.
- map: TryPrimaryMap<MemoryIndex, Option<StaticMemoryInitializer>>,
+ map: TryPrimaryMap<MemoryIndex, Option<(u64, RuntimeDataIndex)>>,
},
}
impl Default for MemoryInitialization {
fn default() -> Self {
- Self::Segmented(TryVec::new())
+ Self::Segmented
}
}
@@ -96,121 +84,10 @@ impl MemoryInitialization {
/// `MemoryInitialization::Segmented`.
pub fn is_segmented(&self) -> bool {
match self {
- MemoryInitialization::Segmented(_) => true,
+ MemoryInitialization::Segmented => true,
_ => false,
}
}
-
- /// Performs the memory initialization steps for this set of initializers.
- ///
- /// This will perform wasm initialization in compliance with the wasm spec
- /// and how data segments are processed. This doesn't need to necessarily
- /// only be called as part of initialization, however, as it's structured to
- /// allow learning about memory ahead-of-time at compile time possibly.
- ///
- /// This function will return true if all memory initializers are processed
- /// successfully. If any initializer hits an error or, for example, a
- /// global value is needed but `None` is returned, then false will be
- /// returned. At compile-time this typically means that the "error" in
- /// question needs to be deferred to runtime, and at runtime this means
- /// that an invalid initializer has been found and a trap should be
- /// generated.
- pub fn init_memory(&self, state: &mut dyn InitMemory) -> bool {
- let initializers = match self {
- // Fall through below to the segmented memory one-by-one
- // initialization.
- MemoryInitialization::Segmented(list) => list,
-
- // If previously switched to static initialization then pass through
- // all those parameters here to the `write` callback.
- //
- // Note that existence of `Static` already guarantees that all
- // indices are in-bounds.
- MemoryInitialization::Static { map } => {
- for (index, init) in map {
- if let Some(init) = init {
- let result = state.write(index, init);
- if !result {
- return result;
- }
- }
- }
- return true;
- }
- };
-
- for initializer in initializers {
- let &MemoryInitializer {
- memory_index,
- ref offset,
- ref data,
- } = initializer;
-
- // First up determine the start/end range and verify that they're
- // in-bounds for the initial size of the memory at `memory_index`.
- // Note that this can bail if we don't have access to globals yet
- // (e.g. this is a task happening before instantiation at
- // compile-time).
- let start = match state.eval_offset(memory_index, offset) {
- Some(start) => start,
- None => return false,
- };
- let len = u64::try_from(data.len()).unwrap();
- let end = match start.checked_add(len) {
- Some(end) => end,
- None => return false,
- };
-
- match state.memory_size_in_bytes(memory_index) {
- Ok(max) => {
- if end > max {
- return false;
- }
- }
-
- // Note that computing the minimum can overflow if the page size
- // is the default 64KiB and the memory's minimum size in pages
- // is `1 << 48`, the maximum number of minimum pages for 64-bit
- // memories. We don't return `false` to signal an error here and
- // instead defer the error to runtime, when it will be
- // impossible to allocate that much memory anyways.
- Err(_) => {}
- }
-
- // The limits of the data segment have been validated at this point
- // so the `write` callback is called with the range of data being
- // written. Any erroneous result is propagated upwards.
- let init = StaticMemoryInitializer {
- offset: start,
- data: data.clone(),
- };
- let result = state.write(memory_index, &init);
- if !result {
- return result;
- }
- }
-
- return true;
- }
-}
-
-/// The various callbacks provided here are used to drive the smaller bits of
-/// memory initialization.
-pub trait InitMemory {
- /// Returns the size, in bytes, of the memory specified. For compile-time
- /// purposes this would be the memory type's minimum size.
- fn memory_size_in_bytes(&mut self, memory_index: MemoryIndex) -> Result<u64, SizeOverflow>;
-
- /// Returns the value of the constant expression, as a `u64`. Note that
- /// this may involve zero-extending a 32-bit global to a 64-bit number. May
- /// return `None` to indicate that the expression involves a value which is
- /// not available yet.
- fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option<u64>;
-
- /// A callback used to actually write data. This indicates that the
- /// specified memory must receive the specified range of data at the
- /// specified offset. This can return false on failure.
- fn write(&mut self, memory_index: MemoryIndex, init: &StaticMemoryInitializer) -> bool;
}
/// Table initialization data for all tables in the module.
@@ -240,16 +117,7 @@ pub struct TableInitialization {
pub enum TableInitialValue {
/// Initialize each table element to null, optionally setting some elements
/// to non-null given the precomputed image.
- Null {
- /// A precomputed image of table initializers for this table.
- ///
- /// This image is constructed during `try_func_table_init` and
- /// null-initialized elements are represented with
- /// `FuncIndex::reserved_value()`. Note that this image is empty by
- /// default and may not encompass the entire span of the table in which
- /// case the elements are initialized to null.
- precomputed: TryVec<FuncIndex>,
- },
+ Null,
/// An arbitrary const expression.
Expr(ConstExpr),
}
@@ -285,6 +153,14 @@ pub enum TableSegmentElements {
}
impl TableSegmentElements {
+ /// Returns the type of this segment.
+ pub fn ty(&self) -> WasmRefType {
+ match self {
+ Self::Functions(_) => WasmRefType::FUNCREF,
+ Self::Expressions { ty, .. } => *ty,
+ }
+ }
+
/// Returns the number of elements in this segment.
pub fn len(&self) -> u64 {
match self {
@@ -313,20 +189,38 @@ pub struct Module {
/// Exported entities.
pub exports: TryIndexMap<Atom, EntityIndex>,
- /// The module "start" function, if present.
- pub start_func: Option<FuncIndex>,
+ /// Whether or not this module has a start function,
+ pub startup: ModuleStartup,
- /// WebAssembly table initialization data, per table.
- pub table_initialization: TableInitialization,
+ /// Precompute per-table static images, if applicable.
+ ///
+ /// This map tracks, for each defined table in this module, the initial
+ /// precomputed contents of the table. This is only applicable for funcref
+ /// tables and the `TryVec` here uses `FuncIndex::reserved_value()` for null
+ /// entries. This structure is filled in if table initialization is detected
+ /// to be infallible as part of [`ModuleTranslation::finalize_table_init`].
+ pub table_initialization: TryPrimaryMap<DefinedTableIndex, TryVec<FuncIndex>>,
/// WebAssembly linear memory initializer.
+ ///
+ /// This will track how memory is initialized, either exclusively via
+ /// segments or if some memories can be initialized with static images. This
+ /// is computed during [`ModuleTranslation::finalize_memory_init`].
pub memory_initialization: MemoryInitialization,
/// WebAssembly passive elements.
- pub passive_elements: TryPrimaryMap<PassiveElemIndex, TableSegmentElements>,
+ ///
+ /// This is a map of all passive element segments to their type and the
+ /// initial size of the segment. Note that the contents of the segment are
+ /// initialized by compiled code.
+ pub passive_elements: TryPrimaryMap<PassiveElemIndex, (WasmRefType, u64)>,
- /// Where passive data segments are located in the module's image.
- pub passive_data: TryPrimaryMap<PassiveDataIndex, Range<u32>>,
+ /// Where runtime data segments are located in the module's image.
+ ///
+ /// Note that this does not directly correspond to either active or passive
+ /// data segments. Those are massaged during
+ /// [`ModuleTranslation::finalize_memory_init`] into the form used here.
+ pub runtime_data: TryPrimaryMap<RuntimeDataIndex, Range<u32>>,
/// Types declared in the wasm module.
pub types: TryPrimaryMap<TypeIndex, EngineOrModuleTypeIndex>,
@@ -368,8 +262,18 @@ pub struct Module {
/// WebAssembly global variables.
pub globals: TryPrimaryMap<GlobalIndex, Global>,
- /// WebAssembly global initializers for locally-defined globals.
- pub global_initializers: TryPrimaryMap<DefinedGlobalIndex, ConstExpr>,
+ /// "Simple" WebAssembly global initializers for locally-defined globals.
+ ///
+ /// This map does not track initialization of all globals in this module,
+ /// but only those considered "simple" which can be easily evaluated at
+ /// compile-time. For example an initialization expression of `i32.const N`
+ /// is considered simple. These globals are manually initialized in the
+ /// host.
+ ///
+ /// This is all in contrast to [`ModuleTranslation::global_initializers`]
+ /// which is processed in compiled code and initialized after the instance
+ /// has been created.
+ pub global_initializers: TryVec<(DefinedGlobalIndex, GlobalConstValue)>,
/// WebAssembly exception and control tags.
pub tags: TryPrimaryMap<TagIndex, Tag>,
@@ -400,11 +304,11 @@ impl Module {
name: Default::default(),
initializers: Default::default(),
exports: Default::default(),
- start_func: Default::default(),
+ startup: ModuleStartup::None,
table_initialization: Default::default(),
memory_initialization: Default::default(),
passive_elements: Default::default(),
- passive_data: Default::default(),
+ runtime_data: Default::default(),
types: Default::default(),
num_imported_funcs: Default::default(),
num_imported_tables: Default::default(),
@@ -690,11 +594,11 @@ impl TypeTrace for Module {
name: _,
initializers: _,
exports: _,
- start_func: _,
+ startup,
table_initialization: _,
memory_initialization: _,
passive_elements: _,
- passive_data: _,
+ runtime_data: _,
types,
num_imported_funcs: _,
num_imported_tables: _,
@@ -726,6 +630,7 @@ impl TypeTrace for Module {
for t in tags.values() {
t.trace(func)?;
}
+ startup.trace(func)?;
Ok(())
}
@@ -741,11 +646,11 @@ impl TypeTrace for Module {
name: _,
initializers: _,
exports: _,
- start_func: _,
+ startup,
table_initialization: _,
memory_initialization: _,
passive_elements: _,
- passive_data: _,
+ runtime_data: _,
types,
num_imported_funcs: _,
num_imported_tables: _,
@@ -777,10 +682,33 @@ impl TypeTrace for Module {
for t in tags.values_mut() {
t.trace_mut(func)?;
}
+ startup.trace_mut(func)?;
Ok(())
}
}
+impl TypeTrace for ModuleStartup {
+ fn trace<F, E>(&self, func: &mut F) -> Result<(), E>
+ where
+ F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>,
+ {
+ match self {
+ ModuleStartup::None => Ok(()),
+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(*t),
+ }
+ }
+
+ fn trace_mut<F, E>(&mut self, func: &mut F) -> Result<(), E>
+ where
+ F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>,
+ {
+ match self {
+ ModuleStartup::None => Ok(()),
+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(t),
+ }
+ }
+}
+
/// Type information about functions in a wasm module.
#[derive(Debug, Serialize, Deserialize)]
pub struct FunctionType {
@@ -821,3 +749,34 @@ impl FunctionType {
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
pub struct FuncRefIndex(u32);
cranelift_entity::entity_impl!(FuncRefIndex);
+
+/// Different means of startup for a wasm module.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum ModuleStartup {
+ /// No startup is necessary.
+ None,
+
+ /// Startup is always required, for example to apply active table segments.
+ ///
+ /// The type of the startup function, of wasm signature `[] -> []`, is
+ /// provided here.
+ Always(EngineOrModuleTypeIndex),
+
+ /// Startup is only required if some linear memory within this module, at
+ /// runtime, says `needs_init() == true`.
+ ///
+ /// This special mode of startup indicates that the startup function has no
+ /// purpose other than to initialize the initial contents of
+ /// `MemoryInitialization::Static` linear memories. In this situation if all
+ /// memories say `needs_init() == false` then the startup function won't
+ /// actually do anything meaning that it can be optimized slightly by
+ /// skipping it entirely.
+ IfMemoriesNeedInit(EngineOrModuleTypeIndex),
+}
+
+impl ModuleStartup {
+ /// Returns if this is `ModuleStartup::None`.
+ pub fn is_none(&self) -> bool {
+ matches!(self, Self::None)
+ }
+}
diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs
index 0f78e765ca..c3173b3819 100644
--- a/crates/environ/src/module_artifacts.rs
+++ b/crates/environ/src/module_artifacts.rs
@@ -552,7 +552,8 @@ impl CompiledFunctionsTable {
FuncKeyKind::ArrayToWasmTrampoline
| FuncKeyKind::WasmToBuiltinTrampoline
- | FuncKeyKind::PatchableToBuiltinTrampoline => false,
+ | FuncKeyKind::PatchableToBuiltinTrampoline
+ | FuncKeyKind::ModuleStartup => false,
#[cfg(feature = "component-model")]
FuncKeyKind::ComponentTrampoline
@@ -569,7 +570,8 @@ impl CompiledFunctionsTable {
| FuncKeyKind::WasmToArrayTrampoline
| FuncKeyKind::WasmToBuiltinTrampoline
| FuncKeyKind::PatchableToBuiltinTrampoline
- | FuncKeyKind::PulleyHostCall => false,
+ | FuncKeyKind::PulleyHostCall
+ | FuncKeyKind::ModuleStartup => false,
#[cfg(feature = "component-model")]
FuncKeyKind::ComponentTrampoline
| FuncKeyKind::ResourceDropTrampoline
diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs
index 0ee06f9378..9841f45ebc 100644
--- a/crates/environ/src/types.rs
+++ b/crates/environ/src/types.rs
@@ -1699,13 +1699,19 @@ impl Default for VMSharedTypeIndex {
pub struct DataIndex(u32);
entity_impl_with_try_clone!(DataIndex);
-/// Index type of a passive data segment inside the WebAssembly module.
+/// Index into data segments needed at runtime by a module.
///
-/// Not a spec-level concept, just used to get dense index spaces for passive
-/// data segments inside of Wasmtime.
+/// This does not directly correspond to either active or passive data segments
+/// in the wasm spec. Instead this is a concept purely for Wasmtime and
+/// organizing memory initialization within the
+/// `ModuleTranslation::finalize_memory_init` function, for example.
+///
+/// Passive data segments at runtime all have a corresponding
+/// `RuntimeDataIndex`, but active data segments maybe coalesced or mutated if
+/// they're statically evaluated.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
-pub struct PassiveDataIndex(u32);
-entity_impl_with_try_clone!(PassiveDataIndex);
+pub struct RuntimeDataIndex(u32);
+entity_impl_with_try_clone!(RuntimeDataIndex);
/// Index type of an element segment inside the WebAssembly module.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
@@ -2007,7 +2013,7 @@ impl ConstExpr {
/// A global's constant value, known at compile time.
#[expect(missing_docs, reason = "self-describing variants")]
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
pub enum GlobalConstValue {
I32(i32),
I64(i64),
@@ -2027,7 +2033,7 @@ pub enum ConstOp {
V128Const(u128),
GlobalGet(GlobalIndex),
RefI31,
- RefNull(WasmHeapTopType),
+ RefNull(WasmHeapType),
RefFunc(FuncIndex),
I32Add,
I32Sub,
@@ -2069,7 +2075,7 @@ impl ConstOp {
O::F32Const { value } => Self::F32Const(value.bits()),
O::F64Const { value } => Self::F64Const(value.bits()),
O::V128Const { value } => Self::V128Const(u128::from_le_bytes(*value.bytes())),
- O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?.top()),
+ O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?),
O::RefFunc { function_index } => Self::RefFunc(FuncIndex::from_u32(function_index)),
O::GlobalGet { global_index } => Self::GlobalGet(GlobalIndex::from_u32(global_index)),
O::RefI31 => Self::RefI31,
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 74c50dd201..3437c14674 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -31,13 +31,14 @@
// globals: [VMGlobalDefinition; module.num_defined_globals],
// tags: [VMTagDefinition; module.num_defined_tags],
// func_refs: [VMFuncRef; module.num_escaped_funcs],
-// passive_data_bases: [*const u8; module.num_passive_data],
-// passive_data_lengths: [u32; module.num_passive_data],
+// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0],
+// runtime_data_bases: [*const u8; module.num_runtime_data],
+// runtime_data_lengths: [u32; module.num_runtime_data],
// }
use crate::{
DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,
- FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, PassiveDataIndex, TableIndex,
+ FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,
TagIndex,
};
use cranelift_entity::packed_option::ReservedValue;
@@ -90,8 +91,10 @@ pub struct VMOffsets<P> {
/// The number of escaped functions in the module, the size of the func_refs
/// array.
pub num_escaped_funcs: u32,
- /// The number of passive data segments in the module.
- pub num_passive_data: u32,
+ /// The number of runtime data segments in the module.
+ pub num_runtime_data: u32,
+ /// Whether or not the module has a start function.
+ pub has_startup_func: bool,
// precalculated offsets of various member fields
imported_functions: u32,
@@ -105,8 +108,9 @@ pub struct VMOffsets<P> {
defined_globals: u32,
defined_tags: u32,
defined_func_refs: u32,
- passive_data_bases: u32,
- passive_data_lengths: u32,
+ startup_func_ref: u32,
+ runtime_data_bases: u32,
+ runtime_data_lengths: u32,
size: u32,
}
@@ -607,8 +611,10 @@ pub struct VMOffsetsFields<P> {
/// The number of escaped functions in the module, the size of the function
/// references array.
pub num_escaped_funcs: u32,
- /// The number of passive data segments in the module.
- pub num_passive_data: u32,
+ /// The number of runtime data segments in the module.
+ pub num_runtime_data: u32,
+ /// Whether or not the module has a start function.
+ pub has_startup_func: bool,
}
impl<P: PtrSize> VMOffsets<P> {
@@ -635,7 +641,8 @@ impl<P: PtrSize> VMOffsets<P> {
num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals),
num_defined_tags: cast_to_u32(module.tags.len() - module.num_imported_tags),
num_escaped_funcs: cast_to_u32(module.num_escaped_funcs),
- num_passive_data: cast_to_u32(module.passive_data.len()),
+ num_runtime_data: cast_to_u32(module.runtime_data.len()),
+ has_startup_func: !module.startup.is_none(),
})
}
@@ -667,7 +674,8 @@ impl<P: PtrSize> VMOffsets<P> {
num_defined_tags: _,
num_owned_memories: _,
num_escaped_funcs: _,
- num_passive_data: _,
+ num_runtime_data: _,
+ has_startup_func: _,
// used as the initial size below
size,
@@ -696,8 +704,9 @@ impl<P: PtrSize> VMOffsets<P> {
}
calculate_sizes! {
- passive_data_lengths: "passive data lengths",
- passive_data_bases: "passive data base pointers",
+ runtime_data_lengths: "runtime data lengths",
+ runtime_data_bases: "runtime data base pointers",
+ startup_func_ref: "startup funcref",
defined_func_refs: "module functions",
defined_tags: "defined tags",
defined_globals: "defined globals",
@@ -728,7 +737,8 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
num_defined_globals: fields.num_defined_globals,
num_defined_tags: fields.num_defined_tags,
num_escaped_funcs: fields.num_escaped_funcs,
- num_passive_data: fields.num_passive_data,
+ num_runtime_data: fields.num_runtime_data,
+ has_startup_func: fields.has_startup_func,
imported_functions: 0,
imported_tables: 0,
imported_memories: 0,
@@ -740,8 +750,9 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
defined_globals: 0,
defined_tags: 0,
defined_func_refs: 0,
- passive_data_bases: 0,
- passive_data_lengths: 0,
+ startup_func_ref: 0,
+ runtime_data_bases: 0,
+ runtime_data_lengths: 0,
size: 0,
};
@@ -800,8 +811,13 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
ret.num_escaped_funcs,
ret.ptr.size_of_vm_func_ref(),
),
- size(passive_data_bases) = cmul(ret.num_passive_data, ret.ptr.size()),
- size(passive_data_lengths) = cmul(ret.num_passive_data, 4),
+ size(startup_func_ref) = if ret.has_startup_func {
+ ret.ptr.size_of_vm_func_ref()
+ } else {
+ 0
+ },
+ size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()),
+ size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4),
}
ret.size = next_field_offset;
@@ -1050,16 +1066,16 @@ impl<P: PtrSize> VMOffsets<P> {
self.defined_func_refs
}
- /// The offset of the `passive_data_bases` array.
+ /// The offset of the `runtime_data_bases` array.
#[inline]
- pub fn vmctx_passive_data_bases_begin(&self) -> u32 {
- self.passive_data_bases
+ pub fn vmctx_runtime_data_bases_begin(&self) -> u32 {
+ self.runtime_data_bases
}
- /// The offset of the `passive_data_lengths` array.
+ /// The offset of the `runtime_data_lengths` array.
#[inline]
- pub fn vmctx_passive_data_lengths_begin(&self) -> u32 {
- self.passive_data_lengths
+ pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 {
+ self.runtime_data_lengths
}
/// Return the size of the `VMContext` allocation.
@@ -1154,20 +1170,29 @@ impl<P: PtrSize> VMOffsets<P> {
self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref())
}
- /// Return the offset to the base of the passive data segment at `index`.
+ /// Returns the offset to the `VMFuncRef` for the module startup function.
+ ///
+ /// Panics if this module does not have a startup function.
+ #[inline]
+ pub fn vmctx_startup_func_ref(&self) -> u32 {
+ assert!(self.has_startup_func);
+ self.startup_func_ref
+ }
+
+ /// Return the offset to the base of the runtime data segment at `index`.
#[inline]
- pub fn vmctx_passive_data_base(&self, index: PassiveDataIndex) -> u32 {
+ pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 {
assert!(!index.is_reserved_value());
- assert!(index.as_u32() < self.num_passive_data);
- self.vmctx_passive_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())
+ assert!(index.as_u32() < self.num_runtime_data);
+ self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())
}
- /// Return the offset to the length of the passive data segment at `index`.
+ /// Return the offset to the length of the runtime data segment at `index`.
#[inline]
- pub fn vmctx_passive_data_length(&self, index: PassiveDataIndex) -> u32 {
+ pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 {
assert!(!index.is_reserved_value());
- assert!(index.as_u32() < self.num_passive_data);
- self.vmctx_passive_data_lengths_begin() + index.as_u32() * 4
+ assert!(index.as_u32() < self.num_runtime_data);
+ self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4
}
/// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`.
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index 3696f325ab..2994644c5d 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -85,10 +85,11 @@ pub(crate) fn build_module_artifacts<T: FinishedObject>(
)
.translate(parser, wasm)
.context("failed to parse WebAssembly module")?;
+ prepare_translation(engine, compiler, &mut translation, &mut types);
let functions = mem::take(&mut translation.function_body_inputs);
let compile_inputs = CompileInputs::for_module(&types, &translation, functions);
- let unlinked_compile_outputs = compile_inputs.compile(engine)?;
+ let unlinked_compile_outputs = compile_inputs.compile(engine, &types)?;
let PreLinkOutput {
needs_gc_heap,
compiled_funcs,
@@ -165,6 +166,15 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
.translate(binary)
.context("failed to parse WebAssembly module")?;
+ for (_, translation) in module_translations.iter_mut() {
+ prepare_translation(
+ engine,
+ compiler,
+ translation,
+ types.module_types_builder_mut(),
+ );
+ }
+
let compile_inputs = CompileInputs::for_component(
engine,
&types,
@@ -174,7 +184,7 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
(i, &*translation, functions)
}),
);
- let unlinked_compile_outputs = compile_inputs.compile(&engine)?;
+ let unlinked_compile_outputs = compile_inputs.compile(&engine, types.module_types_builder())?;
let PreLinkOutput {
needs_gc_heap,
@@ -230,6 +240,26 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
Ok((result, Some(artifacts)))
}
+fn prepare_translation(
+ engine: &Engine,
+ compiler: &dyn Compiler,
+ translation: &mut ModuleTranslation<'_>,
+ types: &mut ModuleTypesBuilder,
+) {
+ // If configured attempt to use static memory initialization
+ // which can either at runtime be implemented as a single memcpy
+ // to initialize memory or otherwise enabling
+ // virtual-memory-tricks such as mmap'ing from a file to get
+ // copy-on-write.
+ let align = compiler.page_size_align();
+ let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
+ translation.finalize_memory_init(engine.tunables(), align, max_always_allowed, types);
+
+ // Attempt to convert table initializer segments to FuncTable
+ // representation where possible, to enable table lazy init.
+ translation.finalize_table_init(engine.tunables(), types);
+}
+
type CompileInput<'a> = Box<dyn FnOnce(&dyn Compiler) -> Result<CompileOutput<'a>> + Send + 'a>;
struct CompileOutput<'a> {
@@ -357,7 +387,7 @@ impl<'a> CompileInputs<'a> {
);
let function = compiler
.component_compiler()
- .compile_trampoline(component, types, key, abi, tunables, &symbol)
+ .compile_component_trampoline(component, types, key, abi, tunables, &symbol)
.with_context(|| format!("failed to compile {symbol}"))?;
Ok(CompileOutput {
key,
@@ -378,12 +408,16 @@ impl<'a> CompileInputs<'a> {
// resources from the embedder and otherwise won't be explicitly
// requested through initializers above or such.
if component.component.num_resources > 0 {
- if let Some(sig) = types.find_resource_drop_signature() {
+ if types
+ .module_types_builder()
+ .find_resource_drop_signature()
+ .is_some()
+ {
ret.push_input(move |compiler| {
let key = FuncKey::ResourceDropTrampoline;
let symbol = "resource_drop_trampoline".to_string();
let function = compiler
- .compile_wasm_to_array_trampoline(types[sig].unwrap_func(), key, &symbol)
+ .compile_trampoline(None, key, types.module_types_builder(), &symbol)
.with_context(|| format!("failed to compile `{symbol}`"))?;
Ok(CompileOutput {
key,
@@ -493,7 +527,7 @@ impl<'a> CompileInputs<'a> {
func_index.as_u32()
);
let function = compiler
- .compile_array_to_wasm_trampoline(translation, types, key, &symbol)
+ .compile_trampoline(Some(translation), key, types, &symbol)
.with_context(|| format!("failed to compile: {symbol}"))?;
Ok(CompileOutput {
key,
@@ -506,6 +540,26 @@ impl<'a> CompileInputs<'a> {
});
}
}
+
+ if !translation.module.startup.is_none() {
+ for abi in [Abi::Wasm, Abi::Array] {
+ self.push_input(move |compiler| {
+ let key = FuncKey::ModuleStartup(abi, module);
+ let symbol = format!("module_start[{}]::{abi:?}", module.as_u32());
+ let function = compiler
+ .compile_trampoline(Some(translation), key, types, &symbol)
+ .with_context(|| format!("failed to compile: {symbol}"))?;
+ Ok(CompileOutput {
+ key,
+ function,
+ symbol,
+ start_srcloc: FilePos::default(),
+ translation: None,
+ func_body: None,
+ })
+ });
+ }
+ }
}
let mut trampoline_types_seen = HashSet::new();
@@ -514,7 +568,6 @@ impl<'a> CompileInputs<'a> {
if !is_new {
continue;
}
- let trampoline_func_ty = types[trampoline_type_index].unwrap_func();
self.push_input(move |compiler| {
let key = FuncKey::WasmToArrayTrampoline(trampoline_type_index);
let symbol = format!(
@@ -522,7 +575,7 @@ impl<'a> CompileInputs<'a> {
trampoline_type_index.as_u32()
);
let function = compiler
- .compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol)
+ .compile_trampoline(None, key, types, &symbol)
.with_context(|| format!("failed to compile: {symbol}"))?;
Ok(CompileOutput {
key,
@@ -538,7 +591,11 @@ impl<'a> CompileInputs<'a> {
/// Compile these `CompileInput`s (maybe in parallel) and return the
/// resulting `UnlinkedCompileOutput`s.
- fn compile(self, engine: &Engine) -> Result<UnlinkedCompileOutputs<'a>> {
+ fn compile(
+ self,
+ engine: &Engine,
+ types: &'a ModuleTypesBuilder,
+ ) -> Result<UnlinkedCompileOutputs<'a>> {
let compiler = engine.try_compiler()?;
if self.inputs.len() > 0 && cfg!(miri) {
@@ -597,7 +654,7 @@ the use case.
// wasmtime-builtin functions are necessary. If so those need to be
// collected and then those trampolines additionally need to be
// compiled.
- compile_required_builtins(engine, &mut raw_outputs)?;
+ compile_required_builtins(engine, types, &mut raw_outputs)?;
// Bucket the outputs by kind.
let mut outputs: BTreeMap<FuncKey, CompileOutput> = BTreeMap::new();
@@ -910,7 +967,8 @@ fn is_inlining_function(key: FuncKey) -> bool {
FuncKey::ArrayToWasmTrampoline(..)
| FuncKey::WasmToArrayTrampoline(..)
| FuncKey::WasmToBuiltinTrampoline(..)
- | FuncKey::PatchableToBuiltinTrampoline(..) => false,
+ | FuncKey::PatchableToBuiltinTrampoline(..)
+ | FuncKey::ModuleStartup(..) => false,
#[cfg(feature = "component-model")]
FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,
@@ -934,7 +992,11 @@ fn inlining_functions<'a>(
})
}
-fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {
+fn compile_required_builtins<'a>(
+ engine: &Engine,
+ types: &'a ModuleTypesBuilder,
+ raw_outputs: &mut Vec<CompileOutput<'a>>,
+) -> Result<()> {
let compiler = engine.try_compiler()?;
let mut builtins = HashSet::new();
let mut new_inputs: Vec<CompileInput<'_>> = Vec::new();
@@ -951,7 +1013,7 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutpu
_ => unreachable!(),
};
let mut function = compiler
- .compile_wasm_to_builtin(key, &symbol)
+ .compile_trampoline(None, key, types, &symbol)
.with_context(|| format!("failed to compile `{symbol}`"))?;
if let Some(compiler) = compiler.inlining_compiler() {
compiler.finish_compiling(&mut function, None, &symbol)?;
@@ -1116,26 +1178,7 @@ impl FunctionIndices {
let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);
let modules = translations
.into_iter()
- .map(|(_, mut translation)| {
- // If configured attempt to use static memory initialization
- // which can either at runtime be implemented as a single memcpy
- // to initialize memory or otherwise enabling
- // virtual-memory-tricks such as mmap'ing from a file to get
- // copy-on-write.
- if engine.tunables().memory_init_cow {
- let align = compiler.page_size_align();
- let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
- translation.try_static_init(align, max_always_allowed);
- }
-
- // Attempt to convert table initializer segments to FuncTable
- // representation where possible, to enable table lazy init.
- if engine.tunables().table_lazy_init {
- translation.try_func_table_init();
- }
-
- obj.append(translation)
- })
+ .map(|(_, translation)| obj.append(translation))
.collect::<Result<PrimaryMap<_, _>>>()?;
let artifacts = Artifacts {
diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rs
index 3d6af317c3..1faf0ac073 100644
--- a/crates/wasmtime/src/runtime/code.rs
+++ b/crates/wasmtime/src/runtime/code.rs
@@ -9,11 +9,9 @@ use alloc::boxed::Box;
use alloc::sync::Arc;
use core::ops::{Add, Range, Sub};
use wasmtime_core::error::OutOfMemory;
-use wasmtime_environ::DefinedFuncIndex;
-use wasmtime_environ::ModuleTypes;
-use wasmtime_environ::StaticModuleIndex;
#[cfg(feature = "component-model")]
use wasmtime_environ::component::ComponentTypes;
+use wasmtime_environ::{FuncKey, ModuleTypes, StaticModuleIndex};
macro_rules! define_pc_kind {
($ty:ident) => {
@@ -431,30 +429,10 @@ impl<'a> ModuleWithCode<'a> {
}
/// Returns the slice in the text section of the function that
- /// `index` points to.
+ /// `key` points to.
#[inline]
- pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {
- let range = self
- .module
- .compiled_module()
- .finished_function_range(def_func_index);
+ pub fn function(&self, key: FuncKey) -> &[u8] {
+ let range = self.module.compiled_module().function_range(key);
&self.store_code.text()[range]
}
-
- /// Get the array-to-Wasm trampoline for the function `index`
- /// points to, as a slice of raw code that can be converted to a
- /// callable function pointer.
- ///
- /// If the function `index` points to does not escape, then `None` is
- /// returned.
- ///
- /// These trampolines are used for array callers (e.g. `Func::new`)
- /// calling Wasm callees.
- pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {
- let range = self
- .module
- .compiled_module()
- .array_to_wasm_trampoline_range(def_func_index)?;
- Some(&self.store_code.text()[range])
- }
}
diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs
index 489caf0a8e..7b4f66cee6 100644
--- a/crates/wasmtime/src/runtime/component/component.rs
+++ b/crates/wasmtime/src/runtime/component/component.rs
@@ -967,6 +967,7 @@ mod tests {
use wasmtime_environ::MemoryInitialization;
#[test]
+ #[cfg_attr(miri, ignore)]
fn cow_on_by_default() {
let mut config = Config::new();
config.wasm_component_model(true);
diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
index d03ba277f1..eb2f392270 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
@@ -7,7 +7,7 @@ use core::convert::Infallible;
use core::fmt::{self, Debug};
use core::hash::{Hash, Hasher};
use core::marker;
-use core::ops::{Deref, DerefMut};
+use core::ops::Deref;
mod sealed {
use super::*;
@@ -209,27 +209,3 @@ impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {
match self.inner {}
}
}
-
-pub(crate) struct OpaqueRootScope<S> {
- store: S,
-}
-
-impl<S> Deref for OpaqueRootScope<S> {
- type Target = S;
-
- fn deref(&self) -> &Self::Target {
- &self.store
- }
-}
-
-impl<S> DerefMut for OpaqueRootScope<S> {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.store
- }
-}
-
-impl<S> OpaqueRootScope<S> {
- pub(crate) fn new(store: S) -> Self {
- OpaqueRootScope { store }
- }
-}
diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs
index f2133cc275..1b4a539710 100644
--- a/crates/wasmtime/src/runtime/instance.rs
+++ b/crates/wasmtime/src/runtime/instance.rs
@@ -14,7 +14,6 @@ use crate::{
};
use alloc::sync::Arc;
use core::ptr::NonNull;
-use wasmparser::WasmFeatures;
use wasmtime_environ::{
EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TagIndex, TypeTrace,
};
@@ -255,21 +254,26 @@ impl Instance {
imports: Imports<'_>,
asyncness: Asyncness,
) -> Result<Instance> {
- let (instance, start) = {
+ let instance = {
let (mut limiter, store) = store.0.resource_limiter_and_store_opaque();
// SAFETY: the safety contract of `new_raw` is the same as this
// function.
- unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports, asyncness).await? }
+ unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports).await? }
};
- if let Some(start) = start {
+
+ // If this instance requires startup, which is a dynamic decision made
+ // at this point in conjunction with analysis at compile time, the
+ // instance gets started. Note that this isn't just the wasm start
+ // function itself, but it's finalization of initialization of this
+ // instance, for example for complicated global initialization
+ // expressions.
+ if instance.id.get_mut(store.0).needs_startup() {
if asyncness == Asyncness::No {
- instance.start_raw(store, start)?;
+ instance.start_raw(store)?;
} else {
#[cfg(feature = "async")]
{
- store
- .on_fiber(|store| instance.start_raw(store, start))
- .await??;
+ store.on_fiber(|store| instance.start_raw(store)).await??;
}
#[cfg(not(feature = "async"))]
unreachable!();
@@ -298,8 +302,7 @@ impl Instance {
mut limiter: Option<&mut StoreResourceLimiter<'_>>,
module: &Module,
imports: Imports<'_>,
- asyncness: Asyncness,
- ) -> Result<(Instance, Option<FuncIndex>)> {
+ ) -> Result<Instance> {
if !Engine::same(store.engine(), module.engine()) {
bail!("cross-`Engine` instantiation is not currently supported");
}
@@ -317,8 +320,6 @@ impl Instance {
}
}
- let compiled_module = module.compiled_module();
-
// Register the module just before instantiation to ensure we keep the module
// properly referenced while in use by the store.
let (modules, engine, breakpoints) = store.modules_and_engine_and_breakpoints_mut();
@@ -341,46 +342,12 @@ impl Instance {
.await?
};
- // Additionally, before we start doing fallible instantiation, we
- // do one more step which is to insert an `InstanceData`
- // corresponding to this instance. This `InstanceData` can be used
- // via `Caller::get_export` if our instance's state "leaks" into
- // other instances, even if we don't return successfully from this
- // function.
- //
- // We don't actually load all exports from the instance at this
- // time, instead preferring to lazily load them as they're demanded.
- // For module/instance exports, though, those aren't actually
- // stored in the instance handle so we need to immediately handle
- // those here.
- let instance = Instance::from_wasmtime(id, store);
-
- // Now that we've recorded all information we need to about this
- // instance within a `Store` we can start performing fallible
- // initialization. Note that we still defer the `start` function to
- // later since that may need to run asynchronously.
- //
- // If this returns an error (or if the start function traps) then
- // any other initialization which may have succeeded which placed
- // items from this instance into other instances should be ok when
- // those items are loaded and run we'll have all the metadata to
- // look at them.
- let bulk_memory = store
- .engine()
- .features()
- .contains(WasmFeatures::BULK_MEMORY);
-
- vm::initialize_instance(
- store,
- limiter,
- id,
- compiled_module.module(),
- bulk_memory,
- asyncness,
- )
- .await?;
-
- Ok((instance, compiled_module.module().start_func))
+ // At this point the instance is created and stored within the store,
+ // but it's also not quite usable just yet. Initialization hasn't
+ // completed (e.g. active data/element segments) and the `start`
+ // function additionally has not yet been invoked. That's the
+ // responsibility of the caller to handle, however.
+ Ok(Instance::from_wasmtime(id, store))
}
pub(crate) fn from_wasmtime(id: InstanceId, store: &mut StoreOpaque) -> Instance {
@@ -389,7 +356,7 @@ impl Instance {
}
}
- fn start_raw<T>(&self, store: &mut StoreContextMut<'_, T>, start: FuncIndex) -> Result<()> {
+ fn start_raw<T>(&self, store: &mut StoreContextMut<'_, T>) -> Result<()> {
// If a start function is present, invoke it. Make sure we use all the
// trap-handling configuration in `store` as well.
let store_id = store.0.id();
@@ -399,7 +366,8 @@ impl Instance {
let f = unsafe {
instance
.as_mut()
- .get_exported_func(registry, store_id, start)
+ .get_startup_func(registry, store_id)
+ .expect("should have a startup function")
};
let caller_vmctx = instance.vmctx();
unsafe {
diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs
index 188d005e1f..8640cdf43a 100644
--- a/crates/wasmtime/src/runtime/instantiate.rs
+++ b/crates/wasmtime/src/runtime/instantiate.rs
@@ -11,9 +11,8 @@ use alloc::sync::Arc;
use core::ops::Range;
use core::str;
use wasmtime_environ::{
- CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,
- FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex,
- StaticModuleIndex,
+ CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FuncKey,
+ FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, StaticModuleIndex,
};
/// A compiled wasm module, ready to be instantiated.
@@ -116,55 +115,32 @@ impl CompiledModule {
pub fn finished_function_ranges(
&self,
) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, Range<usize>)> + '_ {
- self.module
- .defined_func_indices()
- .map(|i| (i, self.finished_function_range(i)))
+ self.module.defined_func_indices().map(|i| {
+ let key = FuncKey::DefinedWasmFunction(self.module_index(), i);
+ (i, self.function_range(key))
+ })
}
- /// Returns the offset in the text section of the function that `index` points to.
+ /// Returns the offset in the text section of the function that `key`
+ /// points to.
#[inline]
- pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range<usize> {
- let loc = self.func_loc(def_func_index);
+ pub fn function_range(&self, key: FuncKey) -> Range<usize> {
+ let loc = self.func_loc(key);
let start = usize::try_from(loc.start).unwrap();
let end = usize::try_from(loc.start + loc.length).unwrap();
start..end
}
- /// Get the array-to-Wasm trampoline for the function `index`
- /// points to, as a range in the text segment.
- ///
- /// If the function `index` points to does not escape, then `None` is
- /// returned.
- ///
- /// These trampolines are used for array callers (e.g. `Func::new`)
- /// calling Wasm callees.
- pub fn array_to_wasm_trampoline_range(
- &self,
- def_func_index: DefinedFuncIndex,
- ) -> Option<Range<usize>> {
- assert!(def_func_index.index() < self.module.num_defined_funcs());
- let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);
- let loc = self.index.func_loc(key)?;
- let start = usize::try_from(loc.start).unwrap();
- let end = usize::try_from(loc.start + loc.length).unwrap();
- Some(start..end)
- }
-
/// Get the Wasm-to-array trampoline for the given signature, as a
/// range in the text segment.
///
/// These trampolines are used for filling in
/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs
/// that don't have access to a compiler when created.
- pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {
+ pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] {
let key = FuncKey::WasmToArrayTrampoline(signature);
- let loc = self.index.func_loc(key)?;
- let start = usize::try_from(loc.start).unwrap();
- let end = usize::try_from(loc.start + loc.length).unwrap();
- Some(
- self.engine_code
- .raw_wasm_to_array_trampoline_data(start..end),
- )
+ let range = self.function_range(key);
+ self.engine_code.raw_wasm_to_array_trampoline_data(range)
}
/// Lookups a defined function by a program counter value.
@@ -190,9 +166,7 @@ impl CompiledModule {
}
/// Gets the function location information for a given function index.
- pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc {
- assert!(def_func_index.index() < self.module.num_defined_funcs());
- let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);
+ pub fn func_loc(&self, key: FuncKey) -> &FunctionLoc {
self.index
.func_loc(key)
.expect("defined function should be present")
@@ -200,9 +174,7 @@ impl CompiledModule {
/// Returns the original binary offset in the file that `index` was defined
/// at.
- pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos {
- assert!(def_func_index.index() < self.module.num_defined_funcs());
- let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);
+ pub fn func_start_srcloc(&self, key: FuncKey) -> FilePos {
self.index
.src_loc(key)
.expect("defined function should be present")
diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs
index 9b4294f34f..3952255804 100644
--- a/crates/wasmtime/src/runtime/module.rs
+++ b/crates/wasmtime/src/runtime/module.rs
@@ -19,8 +19,8 @@ use core::ptr::NonNull;
use std::{fs::File, path::Path};
use wasmparser::{Parser, ValidPayload, Validator};
use wasmtime_environ::{
- CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind,
- StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum,
+ CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, FuncKey, HostPtr, ModuleTypes,
+ ObjectKind, StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum,
};
mod registry;
@@ -1092,7 +1092,8 @@ impl Module {
let module = self.compiled_module();
let module_index = self.env_module().module_index;
self.env_module().defined_func_indices().map(move |idx| {
- let loc = module.func_loc(idx);
+ let key = FuncKey::DefinedWasmFunction(module_index, idx);
+ let loc = module.func_loc(key);
let idx = module.module().func_index(idx);
ModuleFunction {
module: module_index,
@@ -1158,7 +1159,6 @@ impl Module {
let ptr = self
.compiled_module()
.wasm_to_array_trampoline(trampoline_module_ty)
- .expect("always have a trampoline for the trampoline type")
.as_ptr()
.cast::<VMWasmCallFunction>()
.cast_mut();
@@ -1275,6 +1275,7 @@ mod tests {
use wasmtime_environ::MemoryInitialization;
#[test]
+ #[cfg_attr(miri, ignore)]
fn cow_on_by_default() {
let engine = Engine::default();
let module = Module::new(
diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rs
index 30234e32bc..336639c798 100644
--- a/crates/wasmtime/src/runtime/profiling.rs
+++ b/crates/wasmtime/src/runtime/profiling.rs
@@ -13,7 +13,7 @@ use fxprof_processed_profile::{
use std::ops::Range;
use std::sync::Arc;
use std::time::{Duration, Instant};
-use wasmtime_environ::demangle_function_name_or_index;
+use wasmtime_environ::{FuncKey, demangle_function_name_or_index};
// TODO: collect more data
// - On non-Windows, measure thread-local CPU usage between events with
@@ -268,7 +268,9 @@ fn module_symbols(name: String, module: &Module) -> Option<LibraryInfo> {
.env_module()
.defined_func_indices()
.map(|defined_idx| {
- let loc = compiled.func_loc(defined_idx);
+ let key =
+ FuncKey::DefinedWasmFunction(module.env_module().module_index, defined_idx);
+ let loc = compiled.func_loc(key);
let func_idx = compiled.module().func_index(defined_idx);
let mut name = String::new();
demangle_function_name_or_index(
diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs
index cc49f4a7cf..e7d79cb4bf 100644
--- a/crates/wasmtime/src/runtime/trap.rs
+++ b/crates/wasmtime/src/runtime/trap.rs
@@ -7,7 +7,7 @@ use core::fmt;
use core::num::NonZeroUsize;
use wasmtime_core::alloc::TryVec;
use wasmtime_environ::{
- CompiledTrap, FilePos, demangle_function_name, demangle_function_name_or_index,
+ CompiledTrap, FilePos, FuncKey, demangle_function_name, demangle_function_name_or_index,
};
/// Representation of a WebAssembly trap and what caused it to occur.
@@ -501,7 +501,8 @@ impl FrameInfo {
pub(crate) fn new(module: Module, text_offset: usize) -> Option<FrameInfo> {
let compiled_module = module.compiled_module();
let index = compiled_module.func_by_text_offset(text_offset)?;
- let func_start = compiled_module.func_start_srcloc(index);
+ let key = FuncKey::DefinedWasmFunction(compiled_module.module().module_index, index);
+ let func_start = compiled_module.func_start_srcloc(key);
let instr =
wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset);
let index = compiled_module.module().func_index(index);
diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs
index 832652a694..dd2830084d 100644
--- a/crates/wasmtime/src/runtime/values.rs
+++ b/crates/wasmtime/src/runtime/values.rs
@@ -4,7 +4,6 @@ use crate::{
StructRef, V128, ValType, prelude::*,
};
use core::ptr;
-use wasmtime_environ::WasmHeapTopType;
pub use crate::runtime::vm::ValRaw;
@@ -125,16 +124,6 @@ impl Val {
Val::AnyRef(None)
}
- pub(crate) const fn null_top(top: WasmHeapTopType) -> Val {
- match top {
- WasmHeapTopType::Func => Val::FuncRef(None),
- WasmHeapTopType::Extern => Val::ExternRef(None),
- WasmHeapTopType::Any => Val::AnyRef(None),
- WasmHeapTopType::Exn => Val::ExnRef(None),
- WasmHeapTopType::Cont => Val::ContRef(None),
- }
- }
-
/// Returns the default value for the given type, if any exists.
///
/// Returns `None` if there is no default value for the given type (for
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index 3a57c66547..a42ec6666b 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -53,7 +53,6 @@ use wasmtime_environ::ModuleInternedTypeIndex;
mod always_mut;
#[cfg(feature = "component-model")]
pub mod component;
-mod const_expr;
mod export;
mod gc;
mod imports;
@@ -100,7 +99,7 @@ pub use crate::runtime::vm::gc::*;
pub use crate::runtime::vm::imports::Imports;
pub use crate::runtime::vm::instance::{
GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
- MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance,
+ MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex,
};
#[cfg(feature = "pooling-allocator")]
pub use crate::runtime::vm::instance::{
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
deleted file mode 100644
index a3d836bacb..0000000000
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ /dev/null
@@ -1,499 +0,0 @@
-//! Evaluating const expressions.
-
-use crate::prelude::*;
-use crate::runtime::vm;
-use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter};
-#[cfg(feature = "gc")]
-use crate::{
- AnyRef, ArrayRef, ArrayRefPre, ArrayType, ExternRef, I31, StructRef, StructRefPre, StructType,
-};
-use crate::{OpaqueRootScope, Val};
-use wasmtime_environ::{ConstExpr, ConstOp, FuncIndex, GlobalConstValue, GlobalIndex};
-#[cfg(feature = "gc")]
-use wasmtime_environ::{VMSharedTypeIndex, WasmCompositeInnerType, WasmCompositeType, WasmSubType};
-
-/// An interpreter for const expressions.
-///
-/// This can be reused across many const expression evaluations to reuse
-/// allocated resources, if any.
-pub struct ConstExprEvaluator {
- stack: TryVec<Val>,
- simple: Val,
-}
-
-impl Default for ConstExprEvaluator {
- fn default() -> ConstExprEvaluator {
- ConstExprEvaluator {
- stack: TryVec::new(),
- simple: Val::I32(0),
- }
- }
-}
-
-/// The context within which a particular const expression is evaluated.
-pub struct ConstEvalContext {
- pub(crate) instance: InstanceId,
- #[cfg_attr(
- not(feature = "gc"),
- expect(dead_code, reason = "easier than conditionally compiling this field")
- )]
- pub(crate) asyncness: Asyncness,
-}
-
-impl ConstEvalContext {
- /// Create a new context.
- pub fn new(instance: InstanceId, asyncness: Asyncness) -> Self {
- Self {
- instance,
- asyncness,
- }
- }
-
- fn global_get(&mut self, store: &mut StoreOpaque, index: GlobalIndex) -> Result<Val> {
- let id = store.id();
- Ok(store
- .instance_mut(self.instance)
- .get_exported_global(id, index)
- ._get(&mut AutoAssertNoGc::new(store)))
- }
-
- fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result<Val> {
- let id = store.id();
- let (instance, registry) = store.instance_and_module_registry_mut(self.instance);
- // SAFETY: `id` is the correct store-owner of the function being looked
- // up
- let func = unsafe { instance.get_exported_func(registry, id, index) };
- Ok(func.into())
- }
-
- #[cfg(feature = "gc")]
- fn struct_fields_len(&self, store: &mut StoreOpaque, shared_ty: VMSharedTypeIndex) -> usize {
- let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty);
- let fields = struct_ty.fields();
- fields.len()
- }
-
- #[cfg(feature = "gc")]
- async fn struct_new(
- &mut self,
- store: &mut StoreOpaque,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- shared_ty: VMSharedTypeIndex,
- fields: &[Val],
- ) -> Result<Val> {
- let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty);
- let allocator = StructRefPre::_new(store, struct_ty);
- let struct_ref =
- StructRef::_new_async(store, limiter, &allocator, &fields, self.asyncness).await?;
- Ok(Val::AnyRef(Some(struct_ref.into())))
- }
-
- #[cfg(feature = "gc")]
- async fn struct_new_default(
- &mut self,
- store: &mut StoreOpaque,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- shared_ty: VMSharedTypeIndex,
- ) -> Result<Val> {
- let module = store
- .instance(self.instance)
- .runtime_module()
- .expect("should never be allocating a struct type defined in a dummy module");
-
- let borrowed = module
- .engine()
- .signatures()
- .borrow(shared_ty)
- .expect("should have a registered type for struct");
- let WasmSubType {
- composite_type:
- WasmCompositeType {
- shared: false,
- inner: WasmCompositeInnerType::Struct(struct_ty),
- },
- ..
- } = &*borrowed
- else {
- unreachable!("registered type should be a struct");
- };
-
- let fields = struct_ty
- .fields
- .iter()
- .map(|ty| match &ty.element_type {
- wasmtime_environ::WasmStorageType::I8 | wasmtime_environ::WasmStorageType::I16 => {
- Val::I32(0)
- }
- wasmtime_environ::WasmStorageType::Val(v) => match v {
- wasmtime_environ::WasmValType::I32 => Val::I32(0),
- wasmtime_environ::WasmValType::I64 => Val::I64(0),
- wasmtime_environ::WasmValType::F32 => Val::F32(0.0f32.to_bits()),
- wasmtime_environ::WasmValType::F64 => Val::F64(0.0f64.to_bits()),
- wasmtime_environ::WasmValType::V128 => Val::V128(0u128.into()),
- wasmtime_environ::WasmValType::Ref(r) => {
- assert!(r.nullable);
- Val::null_top(r.heap_type.top())
- }
- },
- })
- .collect::<smallvec::SmallVec<[_; 8]>>();
-
- self.struct_new(store, limiter, shared_ty, &fields).await
- }
-}
-
-impl ConstExprEvaluator {
- /// Same as [`Self::eval`] except that this is specifically intended for
- /// integral constant expression.
- ///
- /// # Panics
- ///
- /// Panics if `ConstExpr` contains GC ops (e.g. it's not for an integral
- /// type).
- pub fn eval_int(
- &mut self,
- store: &mut StoreOpaque,
- context: &mut ConstEvalContext,
- expr: &ConstExpr,
- ) -> Result<&Val> {
- // Try to evaluate a simple expression first before doing the more
- // complicated eval loop below.
- if self.try_simple(expr).is_some() {
- return Ok(&self.simple);
- }
-
- // Note that `assert_ready` here should be valid as production of an
- // integer cannot involve GC meaning that async operations aren't used.
- let mut scope = OpaqueRootScope::new(store);
- vm::assert_ready(self.eval_loop(&mut scope, None, context, expr))
- }
-
- /// Attempts to peek into `expr` to see if it's trivial to evaluate, e.g.
- /// for `i32.const N`.
- #[inline]
- pub fn try_simple(&mut self, expr: &ConstExpr) -> Option<&Val> {
- match expr.const_eval()? {
- GlobalConstValue::I32(i) => Some(self.return_one(Val::I32(i))),
- GlobalConstValue::I64(i) => Some(self.return_one(Val::I64(i))),
- GlobalConstValue::F32(f) => Some(self.return_one(Val::F32(f))),
- GlobalConstValue::F64(f) => Some(self.return_one(Val::F64(f))),
- GlobalConstValue::V128(i) => Some(self.return_one(Val::V128(i.into()))),
- }
- }
-
- /// Evaluate the given const expression in the given context.
- ///
- /// Note that the `store` argument is an `OpaqueRootScope` which is used to
- /// require that a GC rooting scope external to evaluation of this constant
- /// is required. Constant expression evaluation may perform GC allocations
- /// and itself trigger a GC meaning that all references must be rooted,
- /// hence the external requirement of a rooting scope.
- ///
- /// # Panics
- ///
- /// This function will panic if `expr` is an invalid constant expression.
- pub async fn eval(
- &mut self,
- store: &mut OpaqueRootScope<&mut StoreOpaque>,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- context: &mut ConstEvalContext,
- expr: &ConstExpr,
- ) -> Result<&Val> {
- // Same structure as `eval_int` above, except using `.await` and with a
- // slightly different type signature here for this function.
- if self.try_simple(expr).is_some() {
- return Ok(&self.simple);
- }
- self.eval_loop(store, limiter, context, expr).await
- }
-
- #[inline]
- fn return_one(&mut self, val: Val) -> &Val {
- self.simple = val;
- &self.simple
- }
-
- #[cold]
- async fn eval_loop(
- &mut self,
- store: &mut OpaqueRootScope<&mut StoreOpaque>,
- mut limiter: Option<&mut StoreResourceLimiter<'_>>,
- context: &mut ConstEvalContext,
- expr: &ConstExpr,
- ) -> Result<&Val> {
- log::trace!("evaluating const expr: {expr:?}");
-
- self.stack.clear();
-
- // On GC-less builds ensure that this is always considered used an
- // needed-mutable.
- let _ = &mut limiter;
-
- for op in expr.ops() {
- log::trace!("const-evaluating op: {op:?}");
- match op {
- ConstOp::I32Const(i) => self.stack.push(Val::I32(*i))?,
- ConstOp::I64Const(i) => self.stack.push(Val::I64(*i))?,
- ConstOp::F32Const(f) => self.stack.push(Val::F32(*f))?,
- ConstOp::F64Const(f) => self.stack.push(Val::F64(*f))?,
- ConstOp::V128Const(v) => self.stack.push(Val::V128((*v).into()))?,
- ConstOp::GlobalGet(g) => self.stack.push(context.global_get(store, *g)?)?,
- ConstOp::RefNull(ty) => self.stack.push(Val::null_top(*ty))?,
- ConstOp::RefFunc(f) => self.stack.push(context.ref_func(store, *f)?)?,
- #[cfg(feature = "gc")]
- ConstOp::RefI31 => {
- let i = self.pop()?.unwrap_i32();
- let i31 = I31::wrapping_i32(i);
- let r = AnyRef::_from_i31(&mut AutoAssertNoGc::new(store), i31);
- self.stack.push(Val::AnyRef(Some(r)))?;
- }
- #[cfg(not(feature = "gc"))]
- ConstOp::RefI31 => panic!("should not have validated"),
- ConstOp::I32Add => {
- let b = self.pop()?.unwrap_i32();
- let a = self.pop()?.unwrap_i32();
- self.stack.push(Val::I32(a.wrapping_add(b)))?;
- }
- ConstOp::I32Sub => {
- let b = self.pop()?.unwrap_i32();
- let a = self.pop()?.unwrap_i32();
- self.stack.push(Val::I32(a.wrapping_sub(b)))?;
- }
- ConstOp::I32Mul => {
- let b = self.pop()?.unwrap_i32();
- let a = self.pop()?.unwrap_i32();
- self.stack.push(Val::I32(a.wrapping_mul(b)))?;
- }
- ConstOp::I64Add => {
- let b = self.pop()?.unwrap_i64();
- let a = self.pop()?.unwrap_i64();
- self.stack.push(Val::I64(a.wrapping_add(b)))?;
- }
- ConstOp::I64Sub => {
- let b = self.pop()?.unwrap_i64();
- let a = self.pop()?.unwrap_i64();
- self.stack.push(Val::I64(a.wrapping_sub(b)))?;
- }
- ConstOp::I64Mul => {
- let b = self.pop()?.unwrap_i64();
- let a = self.pop()?.unwrap_i64();
- self.stack.push(Val::I64(a.wrapping_mul(b)))?;
- }
-
- #[cfg(not(feature = "gc"))]
- ConstOp::StructNew { .. }
- | ConstOp::StructNewDefault { .. }
- | ConstOp::ArrayNew { .. }
- | ConstOp::ArrayNewDefault { .. }
- | ConstOp::ArrayNewFixed { .. }
- | ConstOp::ExternConvertAny
- | ConstOp::AnyConvertExtern => {
- bail!(
- "const expr evaluation error: struct operations are not \
- supported without the `gc` feature"
- )
- }
-
- #[cfg(feature = "gc")]
- ConstOp::StructNew { struct_type_index } => {
- let interned_type_index = store.instance(context.instance).env_module().types
- [*struct_type_index]
- .unwrap_engine_type_index();
- let len = context.struct_fields_len(store, interned_type_index);
-
- if self.stack.len() < len {
- bail!(
- "const expr evaluation error: expected at least {len} values on the stack, found {}",
- self.stack.len()
- )
- }
-
- let start = self.stack.len() - len;
- let s = context
- .struct_new(
- store,
- limiter.as_deref_mut(),
- interned_type_index,
- &self.stack[start..],
- )
- .await?;
- self.stack.truncate(start);
- self.stack.push(s)?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::StructNewDefault { struct_type_index } => {
- let ty = store.instance(context.instance).env_module().types
- [*struct_type_index]
- .unwrap_engine_type_index();
- self.stack.push(
- context
- .struct_new_default(store, limiter.as_deref_mut(), ty)
- .await?,
- )?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::ArrayNew { array_type_index } => {
- let ty = store.instance(context.instance).env_module().types[*array_type_index]
- .unwrap_engine_type_index();
- let ty = ArrayType::from_shared_type_index(store.engine(), ty);
-
- let len = self.pop()?.unwrap_i32().cast_unsigned();
-
- let elem = self.pop()?;
-
- let pre = ArrayRefPre::_new(store, ty);
- let array = ArrayRef::_new_async(
- store,
- limiter.as_deref_mut(),
- &pre,
- &elem,
- len,
- context.asyncness,
- )
- .await?;
-
- self.stack.push(Val::AnyRef(Some(array.into())))?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::ArrayNewDefault { array_type_index } => {
- let ty = store.instance(context.instance).env_module().types[*array_type_index]
- .unwrap_engine_type_index();
- let ty = ArrayType::from_shared_type_index(store.engine(), ty);
-
- let len = self.pop()?.unwrap_i32().cast_unsigned();
-
- let elem = Val::default_for_ty(ty.element_type().unpack())
- .expect("type should have a default value");
-
- let pre = ArrayRefPre::_new(store, ty);
- let array = ArrayRef::_new_async(
- store,
- limiter.as_deref_mut(),
- &pre,
- &elem,
- len,
- context.asyncness,
- )
- .await?;
-
- self.stack.push(Val::AnyRef(Some(array.into())))?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::ArrayNewFixed {
- array_type_index,
- array_size,
- } => {
- let ty = store.instance(context.instance).env_module().types[*array_type_index]
- .unwrap_engine_type_index();
- let ty = ArrayType::from_shared_type_index(store.engine(), ty);
-
- let array_size = usize::try_from(*array_size).unwrap();
- if self.stack.len() < array_size {
- bail!(
- "const expr evaluation error: expected at least {array_size} values on the stack, found {}",
- self.stack.len()
- )
- }
-
- let start = self.stack.len() - array_size;
-
- let elems = self
- .stack
- .drain(start..)
- .collect::<smallvec::SmallVec<[_; 8]>>();
-
- let pre = ArrayRefPre::_new(store, ty);
- let array = ArrayRef::_new_fixed_async(
- store,
- limiter.as_deref_mut(),
- &pre,
- &elems,
- context.asyncness,
- )
- .await?;
-
- self.stack.push(Val::AnyRef(Some(array.into())))?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::ExternConvertAny => {
- let mut store = AutoAssertNoGc::new(store);
- let result = match self.pop()?.unwrap_anyref() {
- Some(anyref) => Some(ExternRef::_convert_any(&mut store, *anyref)?),
- None => None,
- };
- self.stack.push(Val::ExternRef(result))?;
- }
-
- #[cfg(feature = "gc")]
- ConstOp::AnyConvertExtern => {
- let mut store = AutoAssertNoGc::new(store);
- let result = match self.pop()?.unwrap_externref() {
- Some(externref) => Some(AnyRef::_convert_extern(&mut store, *externref)?),
- None => None,
- };
- self.stack.push(result.into())?;
- }
- }
- }
-
- if self.stack.len() == 1 {
- log::trace!("const expr evaluated to {:?}", self.stack[0]);
- Ok(&self.stack[0])
- } else {
- bail!(
- "const expr evaluation error: expected 1 resulting value, found {}",
- self.stack.len()
- )
- }
- }
-
- fn pop(&mut self) -> Result<Val> {
- self.stack.pop().ok_or_else(|| {
- format_err!(
- "const expr evaluation error: attempted to pop from an empty \
- evaluation stack"
- )
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::{ConstExprEvaluator, Val};
- use wasmtime_environ::{ConstExpr, ConstOp};
-
- #[test]
- fn pop_empty_stack() {
- let mut evaluator = ConstExprEvaluator::default();
- let err = evaluator.pop().unwrap_err();
- assert!(
- format!("{err}").contains("attempted to pop from an empty"),
- "unexpected error: {err}"
- );
- }
-
- #[test]
- fn try_simple_scalar_types() {
- let mut ev = ConstExprEvaluator::default();
-
- let expr = ConstExpr::new([ConstOp::I32Const(7)]);
- assert!(matches!(ev.try_simple(&expr), Some(Val::I32(7))));
-
- let expr = ConstExpr::new([ConstOp::I64Const(99)]);
- assert!(matches!(ev.try_simple(&expr), Some(Val::I64(99))));
-
- let expr = ConstExpr::new([ConstOp::F32Const(0x3f800000)]);
- assert!(matches!(ev.try_simple(&expr), Some(Val::F32(_))));
-
- let expr = ConstExpr::new([ConstOp::F64Const(0x3ff0000000000000)]);
- assert!(matches!(ev.try_simple(&expr), Some(Val::F64(_))));
-
- let expr = ConstExpr::new([ConstOp::V128Const(0x12345678)]);
- assert!(matches!(ev.try_simple(&expr), Some(Val::V128(_))));
- }
-}
diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs
index 15dbbaae5d..6f8129f47c 100644
--- a/crates/wasmtime/src/runtime/vm/cow.rs
+++ b/crates/wasmtime/src/runtime/vm/cow.rs
@@ -194,7 +194,7 @@ impl ModuleMemoryImages {
// If there's no initialization for this memory known then we don't
// need an image for the memory so push `None` and move on.
- let init = match init {
+ let (offset, runtime_index) = match init {
Some(init) => init,
None => {
memories.push(None)?;
@@ -202,11 +202,14 @@ impl ModuleMemoryImages {
}
};
- let data_range = init.data.start as usize..init.data.end as usize;
+ let data_range = &module.runtime_data[*runtime_index];
+ let data_range = usize::try_from(data_range.start).unwrap()
+ ..usize::try_from(data_range.end).unwrap();
+
if module.memories[memory_index]
.minimum_byte_size()
.map_or(false, |mem_initial_len| {
- init.offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len
+ *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len
})
{
// The image is rounded up to multiples of the host OS page
@@ -220,7 +223,7 @@ impl ModuleMemoryImages {
return Ok(None);
}
- let offset_usize = match usize::try_from(init.offset) {
+ let offset_usize = match usize::try_from(*offset) {
Ok(offset) => offset,
Err(_) => return Ok(None),
};
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 38057577f1..61be8250cd 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -1370,7 +1370,8 @@ mod tests {
num_defined_globals: 0,
num_defined_tags: 0,
num_escaped_funcs: 0,
- num_passive_data: 0,
+ num_runtime_data: 0,
+ has_startup_func: false,
});
assert_eq!(
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index 9b097c43e7..600d3c1a88 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -2,7 +2,6 @@
//! wasm module (except its callstack and register state). An
//! `InstanceHandle` is a reference-counting handle for an `Instance`.
-use crate::Val;
use crate::code::ModuleWithCode;
use crate::module::ModuleRegistry;
use crate::prelude::*;
@@ -18,9 +17,7 @@ use crate::runtime::vm::{
GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGcRef, VMGlobalKind, VMStore,
VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,
};
-use crate::store::{
- AutoAssertNoGc, InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter,
-};
+use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter};
use crate::vm::{VMWasmCallFunction, ValRaw};
use alloc::sync::Arc;
use core::alloc::Layout;
@@ -35,10 +32,11 @@ use core::{mem, ptr};
use wasmtime_environ::ModuleInternedTypeIndex;
use wasmtime_environ::error::OutOfMemory;
use wasmtime_environ::{
- DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, EntityIndex,
- EntityRef, FuncIndex, GlobalIndex, HostPtr, MemoryIndex, PassiveDataIndex, PassiveElemIndex,
- PtrSize, TableIndex, TableInitialValue, TagIndex, VMCONTEXT_MAGIC, VMOffsets,
- VMSharedTypeIndex, WasmRefType, packed_option::ReservedValue,
+ Abi, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
+ DefinedTagIndex, EntityIndex, EntityRef, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,
+ HostPtr, MemoryIndex, MemoryInitialization, ModuleStartup, PassiveElemIndex, PtrSize,
+ RuntimeDataIndex, TableIndex, TagIndex, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex,
+ WasmRefType, packed_option::ReservedValue,
};
#[cfg(feature = "wmemcheck")]
use wasmtime_wmemcheck::Wmemcheck;
@@ -170,7 +168,7 @@ impl Instance {
) -> Result<InstanceHandle, OutOfMemory> {
let module = req.runtime_info.env_module();
let memory_tys = &module.memories;
- let passive_elements = TryVec::with_capacity(module.passive_elements.len())?;
+ let mut passive_elements = TryVec::with_capacity(module.passive_elements.len())?;
#[cfg(feature = "wmemcheck")]
let wmemcheck_state = if req.store.engine().config().wmemcheck {
@@ -188,6 +186,11 @@ impl Instance {
#[cfg(not(feature = "wmemcheck"))]
let _ = memory_tys;
+ for (_, (ty, len)) in req.runtime_info.env_module().passive_elements.iter() {
+ let len = usize::try_from(*len).unwrap();
+ passive_elements.push(PassiveElementSegment::new(*ty, len)?)?;
+ }
+
let mut ret = OwnedInstance::new(Instance {
id: req.id,
runtime_info: req.runtime_info.clone(),
@@ -205,6 +208,7 @@ impl Instance {
unsafe {
ret.get_mut().initialize_vmctx(req.store, req.imports);
}
+
Ok(ret)
}
@@ -447,6 +451,7 @@ impl Instance {
}
/// Get a locally defined or imported memory.
+ #[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {
if let Some(defined_index) = self.env_module().defined_memory_index(index) {
self.memory(defined_index)
@@ -594,6 +599,26 @@ impl Instance {
unsafe { crate::Func::from_vm_func_ref(store, func_ref) }
}
+ /// Returns a `Func` corresponding to the startup function for this
+ /// instance, if generated at compile time.
+ ///
+ /// # Safety
+ ///
+ /// The `store` parameter must be the store that owns this instance and the
+ /// functions that this instance can reference.
+ pub unsafe fn get_startup_func(
+ self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
+ store: StoreId,
+ ) -> Option<crate::Func> {
+ let func_ref = self.get_start_func_ref(registry)?;
+
+ // SAFETY: the validity of `func_ref` is guaranteed by the validity of
+ // `self`, and the contract that `store` must own `func_ref` is a
+ // contract of this function itself.
+ Some(unsafe { crate::Func::from_vm_func_ref(store, func_ref) })
+ }
+
/// Lookup a table by index.
///
/// # Panics
@@ -820,11 +845,86 @@ impl Instance {
return None;
}
- let Some(def_index) = self.env_module().defined_func_index(index) else {
- debug_assert!(self.env_module().is_imported_function(index));
- return Some(self.imported_function(index).as_func_ref().into());
+ match self.env_module().defined_func_index(index) {
+ Some(index) => self.initialize_defined_funcref(registry, index),
+ None => {
+ debug_assert!(self.env_module().is_imported_function(index));
+ Some(self.imported_function(index).as_func_ref().into())
+ }
+ }
+ }
+
+ /// Initializes a defined function's `VMFuncRef` in-place and then returns a
+ /// pointer to that location.
+ fn initialize_defined_funcref(
+ self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
+ def_index: DefinedFuncIndex,
+ ) -> Option<NonNull<VMFuncRef>> {
+ let module = self.env_module();
+ let index = module.func_index(def_index);
+ let func = &module.functions[index];
+ let type_index = func.signature.unwrap_engine_type_index();
+ let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref);
+ let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index);
+ let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index);
+ // SAFETY: the type/offset/keys here are all valid for the defined
+ // function at `def_index`.
+ unsafe {
+ self.initialize_and_return_funcref(
+ registry,
+ type_index,
+ vmctx_offset,
+ array_to_wasm_key,
+ wasm_key,
+ )
+ }
+ }
+
+ fn get_start_func_ref(
+ self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
+ ) -> Option<NonNull<VMFuncRef>> {
+ let module = self.env_module();
+ let type_index = match module.startup {
+ ModuleStartup::None => return None,
+ ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => {
+ t.unwrap_engine_type_index()
+ }
};
+ let vmctx_offset = self.offsets().vmctx_startup_func_ref();
+ let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index);
+ let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index);
+ // SAFETY: the type/offset/keys here are all valid for the module
+ // startup function.
+ unsafe {
+ self.initialize_and_return_funcref(
+ registry,
+ type_index,
+ vmctx_offset,
+ array_to_wasm_key,
+ wasm_key,
+ )
+ }
+ }
+ /// Common implementation of initializing a `VMFuncRef` stored within this
+ /// instance's `VMContext`.
+ ///
+ /// # Safety
+ ///
+ /// This function requires that `type_index` accurately describes this
+ /// function and `vmctx_offset` is indeed the correct offset for the
+ /// functions here. Effectively all the arguments here must be "logically
+ /// correct" for the `VMFuncRef` being initialized.
+ unsafe fn initialize_and_return_funcref(
+ self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
+ type_index: VMSharedTypeIndex,
+ vmctx_offset: u32,
+ array_to_wasm_key: FuncKey,
+ wasm_key: FuncKey,
+ ) -> Option<NonNull<VMFuncRef>> {
// For now, we eagerly initialize an funcref struct in-place whenever
// asked for a reference to it. This is mostly fine, because in practice
// each funcref is unlikely to be requested more than a few times:
@@ -846,9 +946,6 @@ impl Instance {
// it's better for instantiation performance if we don't have to track
// "is-initialized" state at all!
- let func = &self.env_module().functions[index];
- let type_index = func.signature.unwrap_engine_type_index();
-
let module_with_code = ModuleWithCode::in_store(
registry,
self.runtime_module()
@@ -856,19 +953,13 @@ impl Instance {
)
.expect("module not in store");
- let array_call = VmPtr::from(
- NonNull::from(
- module_with_code
- .array_to_wasm_trampoline(def_index)
- .expect("should have array-to-Wasm trampoline for escaping function"),
- )
- .cast(),
- );
+ let array_call =
+ VmPtr::from(NonNull::from(module_with_code.function(array_to_wasm_key)).cast());
let wasm_call = Some(VmPtr::from(
NonNull::new(
module_with_code
- .finished_function(def_index)
+ .function(wasm_key)
.as_ptr()
.cast::<VMWasmCallFunction>()
.cast_mut(),
@@ -880,9 +971,7 @@ impl Instance {
// SAFETY: the offset calculated here should be correct with
// `self.offsets`
- let func_ref_ptr = unsafe {
- self.vmctx_plus_offset_raw::<VMFuncRef>(self.offsets().vmctx_func_ref(func.func_ref))
- };
+ let func_ref_ptr = unsafe { self.vmctx_plus_offset_raw::<VMFuncRef>(vmctx_offset) };
// SAFETY: the `func_ref_ptr` should be valid as it's within our
// `VMContext` area.
@@ -899,15 +988,16 @@ impl Instance {
}
/// Get the passive elements segment at the given index.
- pub(crate) fn passive_element_segment(&self, passive: PassiveElemIndex) -> &[ValRaw] {
- self.passive_elements[passive.index()].elements()
+ pub(crate) fn passive_element_segment(
+ self: Pin<&mut Self>,
+ passive: PassiveElemIndex,
+ ) -> &mut [ValRaw] {
+ self.passive_elements_mut()[passive.index()].elements_mut()
}
- pub(crate) fn passive_elements_mut(
- self: Pin<&mut Self>,
- ) -> Pin<&mut TryVec<PassiveElementSegment>> {
+ pub(crate) fn passive_elements_mut(self: Pin<&mut Self>) -> &mut TryVec<PassiveElementSegment> {
// SAFETY: Not moving data out of `self`.
- Pin::new(&mut unsafe { self.get_unchecked_mut() }.passive_elements)
+ &mut unsafe { self.get_unchecked_mut() }.passive_elements
}
/// Drop an element.
@@ -946,7 +1036,7 @@ impl Instance {
&self.runtime_info.wasm_data()[start..end]
}
- /// Returns the data for the passive segment identified by `index`
+ /// Returns the data for the runtime segment identified by `index`
///
/// Does not take into account the dynamic size of the data pointed to by
/// `index`, always returns the raw data from the module itself.
@@ -954,8 +1044,8 @@ impl Instance {
/// # Panics
///
/// Panics if `index` is out-of-bounds.
- fn passive_data(&self, index: PassiveDataIndex) -> &[u8] {
- let range = self.env_module().passive_data[index].clone();
+ fn runtime_data(&self, index: RuntimeDataIndex) -> &[u8] {
+ let range = self.env_module().runtime_data[index].clone();
self.wasm_data(range)
}
@@ -1013,10 +1103,7 @@ impl Instance {
// that `i` may be outside the limits of the static
// initialization so it's a fallible `get` instead of an index.
let module = self.env_module();
- let precomputed = match &module.table_initialization.initial_values[idx] {
- TableInitialValue::Null { precomputed } => precomputed,
- TableInitialValue::Expr(_) => unreachable!(),
- };
+ let precomputed = &module.table_initialization[idx];
// Panicking here helps catch bugs rather than silently truncating by accident.
let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned();
let func_ref = func_index
@@ -1031,13 +1118,6 @@ impl Instance {
self.get_defined_table(idx)
}
- /// Get a table by index regardless of whether it is locally-defined or an
- /// imported, foreign table.
- pub(crate) fn get_table(self: Pin<&mut Self>, table_index: TableIndex) -> &mut Table {
- let (idx, instance) = self.defined_table_index_and_instance(table_index);
- instance.get_defined_table(idx)
- }
-
/// Get a locally-defined table.
pub(crate) fn get_defined_table(self: Pin<&mut Self>, index: DefinedTableIndex) -> &mut Table {
&mut self.tables_mut()[index].1
@@ -1237,9 +1317,21 @@ impl Instance {
// after this, but if it's read then it'd hopefully crash faster than
// leaving this undefined.
unsafe {
- for (index, _init) in module.global_initializers.iter() {
+ for i in 0..module.num_defined_globals() {
+ let index = DefinedGlobalIndex::new(i);
instance.global_ptr(index).write(VMGlobalDefinition::new());
}
+ for (index, val) in module.global_initializers.iter() {
+ let mut def = VMGlobalDefinition::new();
+ match val {
+ GlobalConstValue::I32(i) => *def.as_i32_mut() = *i,
+ GlobalConstValue::I64(i) => *def.as_i64_mut() = *i,
+ GlobalConstValue::F32(i) => *def.as_f32_bits_mut() = *i,
+ GlobalConstValue::F64(i) => *def.as_f64_bits_mut() = *i,
+ GlobalConstValue::V128(i) => def.set_u128(*i),
+ }
+ instance.global_ptr(*index).write(def);
+ }
}
// Initialize the defined tags
@@ -1261,7 +1353,7 @@ impl Instance {
}
}
- // Initialize the lengths of passive data segments.
+ // Initialize the lengths of runtime data segments.
//
// SAFETY: it's safe to initialize these lengths during initialization
// here and the various types of pointers and such here should all be
@@ -1269,17 +1361,48 @@ impl Instance {
unsafe {
let offsets = instance.runtime_info.offsets();
let mut lengths =
- instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_lengths_begin());
+ instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin());
let mut bases =
- instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_bases_begin());
- for i in module.passive_data.keys() {
- let data = instance.passive_data(i);
+ instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin());
+ for i in module.runtime_data.keys() {
+ let data = instance.runtime_data(i);
lengths.write(u32::try_from(data.len()).unwrap());
lengths = lengths.add(1);
bases.write(VmPtr::from(NonNull::from(data).cast::<u8>()));
bases = bases.add(1);
}
}
+
+ // This is the half of the strategy of implementing memory-init-cow data
+ // segments. Notably the compiled startup function, if present, will
+ // skip data segments that have a null pointer. Here each linear memory
+ // is tested to see if it needs initialization. If it does, then the
+ // data segment is left in-place (and the startup function will
+ // initialize linear memory). Otherwise the data segment is null'd out.
+ // If the startup function runs (e.g. something else in the module needs
+ // it), then the corresponding data segment's initialization will be
+ // skipped.
+ if let MemoryInitialization::Static { map } = &module.memory_initialization {
+ for (memory, init) in map {
+ let Some(memory) = module.defined_memory_index(memory) else {
+ continue;
+ };
+ if instance.memories[memory].1.needs_init() {
+ continue;
+ }
+ if let Some((_offset, data)) = init {
+ let offsets = instance.runtime_info.offsets();
+ unsafe {
+ instance
+ .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data))
+ .write(0u32);
+ instance
+ .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data))
+ .write(0usize);
+ }
+ }
+ }
+ }
}
/// Attempts to convert from the host `addr` specified to a WebAssembly
@@ -1397,6 +1520,17 @@ impl Instance {
// SAFETY: see `store_mut` above.
unsafe { &mut self.get_unchecked_mut().wmemcheck_state }
}
+
+ pub(crate) fn needs_startup(&self) -> bool {
+ match self.env_module().startup {
+ ModuleStartup::None => false,
+ ModuleStartup::Always(_) => true,
+ ModuleStartup::IfMemoriesNeedInit(_) => self
+ .memories
+ .iter()
+ .any(|(_, (_, memory))| memory.needs_init()),
+ }
+ }
}
// SAFETY: `layout` should describe this accurately and `OwnedVMContext` is the
@@ -1721,39 +1855,14 @@ pub(crate) struct PassiveElementSegment {
impl PassiveElementSegment {
/// Create a new passive element segment with the given capacity.
pub(crate) fn new(ty: WasmRefType, capacity: usize) -> Result<Self, OutOfMemory> {
+ let mut elements = TryVec::with_capacity(capacity)?;
+ elements.resize_with(capacity, || ValRaw::null())?;
Ok(Self {
needs_gc_rooting: ty.is_vmgcref_type_and_not_i31(),
- elements: TryVec::with_capacity(capacity)?,
+ elements,
})
}
- /// Push a value onto this passive element segment.
- ///
- /// NB: Does not type check the value, relies on callers to ensure the
- /// value is of the correct type (generally, due to validation).
- pub(crate) fn push(&mut self, store: &mut StoreOpaque, val: Val) -> Result<()> {
- let mut val = {
- let mut store = AutoAssertNoGc::new(store);
- val.to_raw_(&mut store)?
- };
- if self.needs_gc_rooting {
- // Note that `anyref` accessors and constructors are used here
- // without actually checking the type of this segment or value. The
- // representation and handling of all three is the same which means
- // that this should work out.
- let gc_ref = val.get_anyref();
- debug_assert_eq!(gc_ref, val.get_exnref());
- debug_assert_eq!(gc_ref, val.get_externref());
- if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) {
- if let Some(gc_store) = store.optional_gc_store_mut() {
- val = ValRaw::anyref(gc_store.clone_gc_ref(&gc_ref).as_raw_u32());
- }
- }
- }
- self.elements.push(val)?;
- Ok(())
- }
-
/// Clear this segment's elements.
pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) {
let elements = mem::take(&mut self.elements);
@@ -1776,12 +1885,6 @@ impl PassiveElementSegment {
}
/// The elements of this segment.
- pub(crate) fn elements(&self) -> &[ValRaw] {
- &self.elements
- }
-
- /// The elements of this segment.
- #[cfg(feature = "gc")]
pub(crate) fn elements_mut(&mut self) -> &mut [ValRaw] {
&mut self.elements
}
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
index e2f6ae0824..47e301ede3 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
@@ -1,21 +1,16 @@
use crate::prelude::*;
-use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator};
use crate::runtime::vm::imports::Imports;
use crate::runtime::vm::instance::{Instance, InstanceHandle};
use crate::runtime::vm::memory::Memory;
use crate::runtime::vm::mpk::ProtectionKey;
use crate::runtime::vm::table::Table;
use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo};
-use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter};
-use crate::vm::instance::PassiveElementSegment;
-use crate::{OpaqueRootScope, Val};
+use crate::store::{InstanceId, StoreOpaque, StoreResourceLimiter};
use core::future::Future;
+use core::mem;
use core::pin::Pin;
-use core::{mem, ptr};
use wasmtime_environ::{
- DefinedMemoryIndex, DefinedTableIndex, EntityRef, HostPtr, InitMemory, MemoryInitialization,
- MemoryInitializer, MemoryKind, Module, SizeOverflow, TableInitialValue, TableSegmentElements,
- Trap, VMOffsets, WasmRefType,
+ DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryKind, Module, VMOffsets,
};
#[cfg(feature = "gc")]
@@ -499,445 +494,6 @@ impl dyn InstanceAllocator + '_ {
}
}
-fn check_table_init_bounds(
- store: &mut StoreOpaque,
- instance: InstanceId,
- module: &Module,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
-) -> Result<()> {
- let mut store = OpaqueRootScope::new(store);
-
- for segment in module.table_initialization.segments.iter() {
- let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?;
- let start = get_index(start, module.tables[segment.table_index].idx_type);
- let end = start.checked_add(segment.elements.len());
-
- let table = store.instance_mut(instance).get_table(segment.table_index);
- match end {
- Some(end) if end <= u64::try_from(table.size())? => {
- // Initializer is in bounds
- }
- _ => {
- bail!(Trap::TableOutOfBounds);
- }
- }
- }
-
- Ok(())
-}
-
-async fn initialize_tables(
- store: &mut StoreOpaque,
- mut limiter: Option<&mut StoreResourceLimiter<'_>>,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
- module: &Module,
-) -> Result<()> {
- let mut store = OpaqueRootScope::new(store);
- for (table, init) in module.table_initialization.initial_values.iter() {
- match init {
- // Tables are always initially null-initialized at this time
- TableInitialValue::Null { precomputed: _ } => {}
-
- TableInitialValue::Expr(expr) => {
- let init = const_evaluator
- .eval(&mut store, limiter.as_deref_mut(), context, expr)
- .await?;
- let idx = module.table_index(table);
- let id = store.id();
- let table = store
- .instance_mut(context.instance)
- .get_exported_table(id, idx);
- let size = table.size_(&store);
- table._fill(&mut store, 0, init.ref_().unwrap(), size)?;
- }
- }
- }
-
- // Note: if the module's table initializer state is in
- // FuncTable mode, we will lazily initialize tables based on
- // any statically-precomputed image of FuncIndexes, but there
- // may still be "leftover segments" that could not be
- // incorporated. So we have a unified handler here that
- // iterates over all segments (Segments mode) or leftover
- // segments (FuncTable mode) to initialize.
- for segment in module.table_initialization.segments.iter() {
- let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?;
- let start = get_index(start, module.tables[segment.table_index].idx_type);
-
- let end = start
- .checked_add(segment.elements.len())
- .ok_or_else(|| Trap::TableOutOfBounds)?;
-
- let store_id = store.id();
- let table = {
- let instance = store.instance(context.instance);
- instance.get_exported_table(store_id, segment.table_index)
- };
-
- if end > table.size_(&store) {
- return Err(Trap::TableOutOfBounds.into());
- }
-
- let positions = start..end;
-
- match &segment.elements {
- TableSegmentElements::Functions(funcs) => {
- for (i, func_idx) in positions.zip(funcs) {
- let func = {
- let (instance, registry) =
- store.instance_and_module_registry_mut(context.instance);
- // SAFETY: the `store_id` passed to `get_exported_func` is
- // indeed the store that owns the function.
- unsafe { instance.get_exported_func(registry, store_id, *func_idx) }
- };
- table.set_(&mut store, i, func.into())?;
- }
- }
- TableSegmentElements::Expressions { exprs, ty: _ } => {
- for (i, expr) in positions.zip(exprs) {
- let val = const_evaluator
- .eval(&mut store, limiter.as_deref_mut(), context, expr)
- .await?;
- table.set_(&mut store, i, val.ref_().unwrap())?;
- }
- }
- }
- }
-
- Ok(())
-}
-
-fn get_index(val: &Val, ty: wasmtime_environ::IndexType) -> u64 {
- match ty {
- wasmtime_environ::IndexType::I32 => val.unwrap_i32().cast_unsigned().into(),
- wasmtime_environ::IndexType::I64 => val.unwrap_i64().cast_unsigned(),
- }
-}
-
-fn get_memory_init_start(
- store: &mut StoreOpaque,
- init: &MemoryInitializer,
- instance: InstanceId,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
-) -> Result<u64> {
- let mut store = OpaqueRootScope::new(store);
- const_evaluator
- .eval_int(&mut store, context, &init.offset)
- .map(|v| {
- get_index(
- v,
- store.instance(instance).env_module().memories[init.memory_index].idx_type,
- )
- })
-}
-
-fn check_memory_init_bounds(
- store: &mut StoreOpaque,
- instance: InstanceId,
- initializers: &[MemoryInitializer],
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
-) -> Result<()> {
- for init in initializers {
- let memory = store.instance_mut(instance).get_memory(init.memory_index);
- let start = get_memory_init_start(store, init, instance, context, const_evaluator)?;
- let end = usize::try_from(start)
- .ok()
- .and_then(|start| start.checked_add(init.data.len()));
-
- match end {
- Some(end) if end <= memory.current_length() => {
- // Initializer is in bounds
- }
- _ => {
- bail!(Trap::MemoryOutOfBounds);
- }
- }
- }
-
- Ok(())
-}
-
-fn initialize_memories(
- store: &mut StoreOpaque,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
- module: &Module,
-) -> Result<()> {
- // Delegates to the `init_memory` method which is sort of a duplicate of
- // `instance.memory_init_segment` but is used at compile-time in other
- // contexts so is shared here to have only one method of memory
- // initialization.
- //
- // This call to `init_memory` notably implements all the bells and whistles
- // so errors only happen if an out-of-bounds segment is found, in which case
- // a trap is returned.
-
- struct InitMemoryAtInstantiation<'a> {
- module: &'a Module,
- store: &'a mut StoreOpaque,
- context: &'a mut ConstEvalContext,
- const_evaluator: &'a mut ConstExprEvaluator,
- }
-
- impl InitMemory for InitMemoryAtInstantiation<'_> {
- fn memory_size_in_bytes(
- &mut self,
- memory: wasmtime_environ::MemoryIndex,
- ) -> Result<u64, SizeOverflow> {
- let len = self
- .store
- .instance(self.context.instance)
- .get_memory(memory)
- .current_length();
- let len = u64::try_from(len).unwrap();
- Ok(len)
- }
-
- fn eval_offset(
- &mut self,
- memory: wasmtime_environ::MemoryIndex,
- expr: &wasmtime_environ::ConstExpr,
- ) -> Option<u64> {
- let mut store = OpaqueRootScope::new(&mut *self.store);
- let val = self
- .const_evaluator
- .eval_int(&mut store, self.context, expr)
- .ok()?;
- Some(get_index(
- val,
- store.instance(self.context.instance).env_module().memories[memory].idx_type,
- ))
- }
-
- fn write(
- &mut self,
- memory_index: wasmtime_environ::MemoryIndex,
- init: &wasmtime_environ::StaticMemoryInitializer,
- ) -> bool {
- // If this initializer applies to a defined memory but that memory
- // doesn't need initialization, due to something like copy-on-write
- // pre-initializing it via mmap magic, then this initializer can be
- // skipped entirely.
- let instance = self.store.instance_mut(self.context.instance);
- if let Some(memory_index) = self.module.defined_memory_index(memory_index) {
- if !instance.memories[memory_index].1.needs_init() {
- return true;
- }
- }
- let memory = instance.get_memory(memory_index);
-
- unsafe {
- let src = instance.wasm_data(init.data.clone());
- let offset = usize::try_from(init.offset).unwrap();
- let dst = memory.base.as_ptr().add(offset);
-
- assert!(offset + src.len() <= memory.current_length());
-
- // FIXME audit whether this is safe in the presence of shared
- // memory
- // (https://github.com/bytecodealliance/wasmtime/issues/4203).
- ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len())
- }
- true
- }
- }
-
- let ok = module
- .memory_initialization
- .init_memory(&mut InitMemoryAtInstantiation {
- module,
- store,
- context,
- const_evaluator,
- });
- if !ok {
- return Err(Trap::MemoryOutOfBounds.into());
- }
-
- Ok(())
-}
-
-fn check_init_bounds(
- store: &mut StoreOpaque,
- instance: InstanceId,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
- module: &Module,
-) -> Result<()> {
- check_table_init_bounds(store, instance, module, context, const_evaluator)?;
-
- match &module.memory_initialization {
- MemoryInitialization::Segmented(initializers) => {
- check_memory_init_bounds(store, instance, initializers, context, const_evaluator)?;
- }
- // Statically validated already to have everything in-bounds.
- MemoryInitialization::Static { .. } => {}
- }
-
- Ok(())
-}
-
-async fn initialize_globals(
- store: &mut StoreOpaque,
- mut limiter: Option<&mut StoreResourceLimiter<'_>>,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
- module: &Module,
-) -> Result<()> {
- assert!(core::ptr::eq(
- &**store.instance(context.instance).env_module(),
- module
- ));
-
- let mut store = OpaqueRootScope::new(store);
-
- for (index, init) in module.global_initializers.iter() {
- // Attempt a simple, synchronous evaluation before hitting the
- // general-purpose `.await` point below. This benchmarks ~15% faster in
- // instantiation vs just falling through to `.await` below.
- let val = if let Some(val) = const_evaluator.try_simple(init) {
- val
- } else {
- const_evaluator
- .eval(&mut store, limiter.as_deref_mut(), context, init)
- .await?
- };
-
- let id = store.id();
- let index = module.global_index(index);
- let mut instance = store.instance_mut(context.instance);
-
- #[cfg(feature = "wmemcheck")]
- if index.as_u32() == 0
- && module.globals[index].wasm_ty == wasmtime_environ::WasmValType::I32
- {
- if let Some(wmemcheck) = instance.as_mut().wmemcheck_state_mut() {
- let size = usize::try_from(val.unwrap_i32()).unwrap();
- wmemcheck.set_stack_size(size);
- }
- }
-
- let global = instance.as_mut().get_exported_global(id, index);
-
- // Note that mutability is bypassed here because this is, by definition,
- // initialization of globals meaning that if it's an immutable global
- // this is the one and only write.
- //
- // SAFETY: this is a valid module so `val` should have the correct type
- // for this global, and it's safe to write to a global for the first
- // time as-is happening here.
- unsafe {
- global.set_unchecked(&mut store, &val)?;
- }
- }
- Ok(())
-}
-
-async fn initialize_passive_elements(
- store: &mut StoreOpaque,
- mut limiter: Option<&mut StoreResourceLimiter<'_>>,
- context: &mut ConstEvalContext,
- const_evaluator: &mut ConstExprEvaluator,
- module: &Module,
-) -> Result<()> {
- let store_id = store.id();
-
- let instance = store.instance_mut(context.instance);
- debug_assert!(instance.passive_elements.is_empty());
- instance
- .passive_elements_mut()
- .reserve(module.passive_elements.len())?;
-
- for (idx, segment) in &module.passive_elements {
- match segment {
- TableSegmentElements::Functions(func_indices) => {
- let mut segment =
- PassiveElementSegment::new(WasmRefType::FUNCREF, func_indices.len())?;
- for func_idx in func_indices {
- let (instance, registry) =
- store.instance_and_module_registry_mut(context.instance);
- // SAFETY: `store_id` is for the store that owns this instance.
- let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) };
- segment.push(store, func.into())?;
- }
- let instance = store.instance_mut(context.instance);
- debug_assert_eq!(instance.passive_elements.len(), idx.index());
- instance.passive_elements_mut().push(segment)?;
- }
- TableSegmentElements::Expressions { ty, exprs } => {
- let mut segment = PassiveElementSegment::new(*ty, exprs.len())?;
- for expr in exprs {
- let mut store = OpaqueRootScope::new(&mut *store);
- let val = const_evaluator
- .eval(&mut store, limiter.as_deref_mut(), context, expr)
- .await?;
- segment.push(&mut store, *val)?;
- }
- let instance = store.instance_mut(context.instance);
- debug_assert_eq!(instance.passive_elements.len(), idx.index());
- instance.passive_elements_mut().push(segment)?;
- }
- }
- }
-
- debug_assert_eq!(
- module.passive_elements.len(),
- store.instance(context.instance).passive_elements.len()
- );
- Ok(())
-}
-
-pub async fn initialize_instance(
- store: &mut StoreOpaque,
- mut limiter: Option<&mut StoreResourceLimiter<'_>>,
- instance: InstanceId,
- module: &Module,
- is_bulk_memory: bool,
- asyncness: Asyncness,
-) -> Result<()> {
- let mut context = ConstEvalContext::new(instance, asyncness);
- let mut const_evaluator = ConstExprEvaluator::default();
-
- // If bulk memory is not enabled, bounds check the data and element segments before
- // making any changes. With bulk memory enabled, initializers are processed
- // in-order and side effects are observed up to the point of an out-of-bounds
- // initializer, so the early checking is not desired.
- if !is_bulk_memory {
- check_init_bounds(store, instance, &mut context, &mut const_evaluator, module)?;
- }
-
- initialize_globals(
- store,
- limiter.as_deref_mut(),
- &mut context,
- &mut const_evaluator,
- module,
- )
- .await?;
-
- initialize_tables(
- store,
- limiter.as_deref_mut(),
- &mut context,
- &mut const_evaluator,
- module,
- )
- .await?;
-
- initialize_memories(store, &mut context, &mut const_evaluator, &module)?;
-
- if is_bulk_memory {
- initialize_passive_elements(store, limiter, &mut context, &mut const_evaluator, &module)
- .await?;
- }
-
- Ok(())
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 241057fa71..f8641c87d6 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -341,8 +341,7 @@ fn passive_elem_segment_base(
store
.instance_mut(instance)
.passive_element_segment(elem_index)
- .as_ptr()
- .cast_mut()
+ .as_mut_ptr()
.cast()
}
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index ab097b978e..83844fb4ee 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -4,7 +4,6 @@
mod vm_host_func_context;
pub use self::vm_host_func_context::VMArrayCallHostFuncContext;
-use crate::bail_bug;
use crate::prelude::*;
use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16};
use crate::store::StoreOpaque;
@@ -20,7 +19,6 @@ use core::sync::atomic::{AtomicUsize, Ordering};
use wasmtime_environ::{
BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
DefinedTagIndex, NUM_COMPONENT_CONTEXT_SLOTS, VMCONTEXT_MAGIC, VMSharedTypeIndex,
- WasmHeapTopType, WasmValType,
};
/// A function pointer that exposes the array calling convention.
@@ -600,86 +598,6 @@ impl VMGlobalDefinition {
Self { storage: [0; 16] }
}
- /// Create a `VMGlobalDefinition` from a `ValRaw`.
- ///
- /// # Unsafety
- ///
- /// This raw value's type must match the given `WasmValType`.
- pub unsafe fn from_val_raw(
- store: &mut StoreOpaque,
- wasm_ty: WasmValType,
- raw: ValRaw,
- ) -> Result<Self> {
- let mut global = Self::new();
- unsafe {
- match wasm_ty {
- WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(),
- WasmValType::I64 => *global.as_i64_mut() = raw.get_i64(),
- WasmValType::F32 => *global.as_f32_bits_mut() = raw.get_f32(),
- WasmValType::F64 => *global.as_f64_bits_mut() = raw.get_f64(),
- WasmValType::V128 => global.set_u128(raw.get_v128()),
- WasmValType::Ref(r) => match r.heap_type.top() {
- WasmHeapTopType::Extern => {
- let r = VMGcRef::from_raw_u32(raw.get_externref());
- global.init_gc_ref(store, r.as_ref())?
- }
- WasmHeapTopType::Any => {
- let r = VMGcRef::from_raw_u32(raw.get_anyref());
- global.init_gc_ref(store, r.as_ref())?
- }
- WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),
- WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack.
- WasmHeapTopType::Exn => {
- let r = VMGcRef::from_raw_u32(raw.get_exnref());
- global.init_gc_ref(store, r.as_ref())?
- }
- },
- }
- }
- Ok(global)
- }
-
- /// Get this global's value as a `ValRaw`.
- ///
- /// # Unsafety
- ///
- /// This global's value's type must match the given `WasmValType`.
- pub unsafe fn to_val_raw(
- &self,
- store: &mut StoreOpaque,
- wasm_ty: WasmValType,
- ) -> Result<ValRaw> {
- unsafe {
- Ok(match wasm_ty {
- WasmValType::I32 => ValRaw::i32(*self.as_i32()),
- WasmValType::I64 => ValRaw::i64(*self.as_i64()),
- WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()),
- WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()),
- WasmValType::V128 => ValRaw::v128(self.get_u128()),
- WasmValType::Ref(r) => match r.heap_type.top() {
- WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() {
- Some(r) => store.clone_gc_ref(r).as_raw_u32(),
- None => 0,
- }),
- WasmHeapTopType::Any => ValRaw::anyref({
- match self.as_gc_ref() {
- Some(r) => store.clone_gc_ref(r).as_raw_u32(),
- None => 0,
- }
- }),
- WasmHeapTopType::Exn => ValRaw::exnref({
- match self.as_gc_ref() {
- Some(r) => store.clone_gc_ref(r).as_raw_u32(),
- None => 0,
- }
- }),
- WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),
- WasmHeapTopType::Cont => bail_bug!("unimplemented"), // FIXME: #10248 stack switching support.
- },
- })
- }
- }
-
/// Return a reference to the value as an i32.
pub unsafe fn as_i32(&self) -> &i32 {
unsafe { &*(self.storage.as_ref().as_ptr().cast::<i32>()) }
diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs
index 3939c26ff7..6a9a7f2982 100644
--- a/crates/winch/src/compiler.rs
+++ b/crates/winch/src/compiler.rs
@@ -167,25 +167,15 @@ impl wasmtime_environ::Compiler for Compiler {
})
}
- fn compile_array_to_wasm_trampoline(
- &self,
- translation: &ModuleTranslation<'_>,
- types: &ModuleTypesBuilder,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- self.trampolines
- .compile_array_to_wasm_trampoline(translation, types, key, symbol)
- }
-
- fn compile_wasm_to_array_trampoline(
+ fn compile_trampoline(
&self,
- wasm_func_ty: &wasmtime_environ::WasmFuncType,
+ translation: Option<&ModuleTranslation<'_>>,
key: FuncKey,
+ types: &ModuleTypesBuilder,
symbol: &str,
) -> Result<CompiledFunctionBody, CompileError> {
self.trampolines
- .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)
+ .compile_trampoline(translation, key, types, symbol)
}
fn append_code(
@@ -236,14 +226,6 @@ impl wasmtime_environ::Compiler for Compiler {
self.isa.create_systemv_cie()
}
- fn compile_wasm_to_builtin(
- &self,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- self.trampolines.compile_wasm_to_builtin(key, symbol)
- }
-
fn compiled_function_relocation_targets<'a>(
&'a self,
func: &'a dyn Any,
@@ -280,45 +262,14 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {
Ok(body)
}
- fn compile_array_to_wasm_trampoline(
- &self,
- translation: &ModuleTranslation<'_>,
- types: &ModuleTypesBuilder,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- let mut body = self
- .0
- .compile_array_to_wasm_trampoline(translation, types, key, symbol)?;
- if let Some(c) = self.0.inlining_compiler() {
- c.finish_compiling(&mut body, None, symbol)
- .map_err(|e| CompileError::Codegen(e.to_string()))?;
- }
- Ok(body)
- }
-
- fn compile_wasm_to_array_trampoline(
- &self,
- wasm_func_ty: &wasmtime_environ::WasmFuncType,
- key: FuncKey,
- symbol: &str,
- ) -> Result<CompiledFunctionBody, CompileError> {
- let mut body = self
- .0
- .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)?;
- if let Some(c) = self.0.inlining_compiler() {
- c.finish_compiling(&mut body, None, symbol)
- .map_err(|e| CompileError::Codegen(e.to_string()))?;
- }
- Ok(body)
- }
-
- fn compile_wasm_to_builtin(
+ fn compile_trampoline(
&self,
+ translation: Option<&ModuleTranslation<'_>>,
key: FuncKey,
+ types: &ModuleTypesBuilder,
symbol: &str,
) -> Result<CompiledFunctionBody, CompileError> {
- let mut body = self.0.compile_wasm_to_builtin(key, symbol)?;
+ let mut body = self.0.compile_trampoline(translation, key, types, symbol)?;
if let Some(c) = self.0.inlining_compiler() {
c.finish_compiling(&mut body, None, symbol)
.map_err(|e| CompileError::Codegen(e.to_string()))?;
@@ -381,7 +332,7 @@ impl wasmtime_environ::Compiler for NoInlineCompiler {
#[cfg(feature = "component-model")]
impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
- fn compile_trampoline(
+ fn compile_component_trampoline(
&self,
component: &wasmtime_environ::component::ComponentTranslation,
types: &wasmtime_environ::component::ComponentTypesBuilder,
@@ -393,7 +344,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
let mut body = self
.0
.component_compiler()
- .compile_trampoline(component, types, key, abi, tunables, symbol)?;
+ .compile_component_trampoline(component, types, key, abi, tunables, symbol)?;
if let Some(c) = self.0.inlining_compiler() {
c.finish_compiling(&mut body, None, symbol)
.map_err(|e| CompileError::Codegen(e.to_string()))?;
diff --git a/src/commands/objdump.rs b/src/commands/objdump.rs
index a81dc2a8be..c85607b634 100644
--- a/src/commands/objdump.rs
+++ b/src/commands/objdump.rs
@@ -205,6 +205,7 @@ impl ObjdumpCommand {
|| name.ends_with("_array_call")
|| name.ends_with("_wasm_call")
|| name.contains("unsafe-intrinsics-")
+ || name.contains("module_start")
{
Func::Trampoline
} else if name.contains("libcall") || name.starts_with("component") {
diff --git a/tests/all/arrays.rs b/tests/all/arrays.rs
index 31f68b306e..b42a38855d 100644
--- a/tests/all/arrays.rs
+++ b/tests/all/arrays.rs
@@ -1002,6 +1002,7 @@ fn issue_13034_array_layout_overflow() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn host_arrayref_has_trace_info_for_gc() -> Result<()> {
for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {
println!("Using GC collector: {collector:?}");
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
index 96d45dcebd..a7f34fb7fc 100644
--- a/tests/all/cli_tests.rs
+++ b/tests/all/cli_tests.rs
@@ -3051,11 +3051,11 @@ fn profile_guest() -> Result<()> {
None,
)?;
- assert!(output.status.success());
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
println!("> stdout:\n{stdout}");
println!("> stderr:\n{stderr}");
+ assert!(output.status.success());
assert!(!stderr.contains("Error"));
let out_json = std::fs::read_to_string(format!("{dir}/out.json")).unwrap();
println!("> out.json:\n{out_json}");
diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rs
index e4f326877a..cde1dec742 100644
--- a/tests/all/exceptions.rs
+++ b/tests/all/exceptions.rs
@@ -216,6 +216,7 @@ fn exception_across_no_wasm(config: &mut Config) -> Result<()> {
}
#[wasmtime_test(wasm_features(gc, exceptions))]
+#[cfg_attr(miri, ignore)]
fn gc_with_exnref_global(config: &mut Config) -> Result<()> {
let engine = Engine::new(config)?;
let mut store = Store::new(&engine, ());
diff --git a/tests/all/exnrefs.rs b/tests/all/exnrefs.rs
index 6251d01ff6..de0b73efbb 100644
--- a/tests/all/exnrefs.rs
+++ b/tests/all/exnrefs.rs
@@ -85,6 +85,7 @@ fn exn_objects() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn host_exnref_has_trace_info_for_gc() -> Result<()> {
for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {
println!("Using GC collector: {collector:?}");
diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast
index 7196df646a..f918eb94ac 100644
--- a/tests/all/fuel.wast
+++ b/tests/all/fuel.wast
@@ -1,11 +1,11 @@
(assert_fuel 0 (module))
-(assert_fuel 1
+(assert_fuel 3
(module
(func $f)
(start $f)))
-(assert_fuel 2
+(assert_fuel 4
(module
(func $f
i32.const 0
@@ -13,7 +13,7 @@
)
(start $f)))
-(assert_fuel 1
+(assert_fuel 3
(module
(func $f
block
@@ -21,14 +21,14 @@
)
(start $f)))
-(assert_fuel 1
+(assert_fuel 3
(module
(func $f
unreachable
)
(start $f)))
-(assert_fuel 7
+(assert_fuel 9
(module
(func $f
i32.const 0
@@ -41,7 +41,7 @@
)
(start $f)))
-(assert_fuel 1
+(assert_fuel 3
(module
(func $f
return
@@ -55,7 +55,7 @@
)
(start $f)))
-(assert_fuel 3
+(assert_fuel 5
(module
(func $f
i32.const 0
@@ -65,7 +65,7 @@
)
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
i32.const 1
@@ -76,7 +76,7 @@
)
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
i32.const 1
@@ -89,7 +89,7 @@
)
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
i32.const 0
@@ -102,7 +102,7 @@
)
(start $f)))
-(assert_fuel 3
+(assert_fuel 5
(module
(func $f
block
@@ -114,7 +114,7 @@
)
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
block
@@ -127,7 +127,7 @@
(start $f)))
;; count code before unreachable
-(assert_fuel 2
+(assert_fuel 4
(module
(func $f
i32.const 0
@@ -136,7 +136,7 @@
(start $f)))
;; count code before return
-(assert_fuel 2
+(assert_fuel 4
(module
(func $f
i32.const 0
@@ -145,14 +145,14 @@
(start $f)))
;; cross-function fuel works
-(assert_fuel 3
+(assert_fuel 5
(module
(func $f
call $other
)
(func $other)
(start $f)))
-(assert_fuel 5
+(assert_fuel 7
(module
(func $f
i32.const 0
@@ -162,7 +162,7 @@
)
(func $other (param i32))
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
call $other
@@ -172,7 +172,7 @@
i32.const 0
)
(start $f)))
-(assert_fuel 4
+(assert_fuel 6
(module
(func $f
i32.const 0
@@ -183,14 +183,14 @@
(start $f)))
;; loops!
-(assert_fuel 1
+(assert_fuel 3
(module
(func $f
loop
end
)
(start $f)))
-(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func
+(assert_fuel 55 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func
(module
(func $f
(local i32)
@@ -207,7 +207,7 @@
)
(start $f)))
-(assert_fuel 105
+(assert_fuel 107
(module
(memory 1)
(func $f
@@ -218,7 +218,7 @@
)
(start $f)))
-(assert_fuel 105
+(assert_fuel 107
(module
(memory 1)
(func $f
@@ -229,7 +229,7 @@
)
(start $f)))
-(assert_fuel 25
+(assert_fuel 27
(module
(memory 1)
(func $f
@@ -241,7 +241,7 @@
(start $f)
(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
-(assert_fuel 105
+(assert_fuel 107
(module
(table 100 funcref)
(func $f
@@ -252,7 +252,7 @@
)
(start $f)))
-(assert_fuel 105
+(assert_fuel 107
(module
(table 100 funcref)
(func $f
@@ -263,7 +263,7 @@
)
(start $f)))
-(assert_fuel 104
+(assert_fuel 106
(module
(table 0 funcref)
(func $f
@@ -274,7 +274,7 @@
)
(start $f)))
-(assert_fuel 25
+(assert_fuel 27
(module
(table 20 funcref)
(func $f
@@ -286,7 +286,7 @@
(start $f)
(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
-(assert_fuel 107
+(assert_fuel 211
(module
(type $a (array (mut i8)))
(global $a (ref $a) (array.new_default $a (i32.const 100)))
@@ -300,7 +300,7 @@
)
(start $f)))
-(assert_fuel 106
+(assert_fuel 210
(module
(type $a (array (mut i8)))
(global $a (ref $a) (array.new_default $a (i32.const 100)))
@@ -313,7 +313,7 @@
)
(start $f)))
-(assert_fuel 24
+(assert_fuel 26
(module
(type $a (array (mut i8)))
(func $f
@@ -325,7 +325,7 @@
(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
(start $f)))
-(assert_fuel 26
+(assert_fuel 130
(module
(type $a (array (mut i8)))
(global $a (ref $a) (array.new_default $a (i32.const 100)))
@@ -339,7 +339,7 @@
(data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
(start $f)))
-(assert_fuel 24
+(assert_fuel 26
(module
(type $a (array (mut funcref)))
(func $f
@@ -351,7 +351,7 @@
(start $f)
(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
-(assert_fuel 26
+(assert_fuel 130
(module
(type $a (array (mut funcref)))
(global $a (ref $a) (array.new_default $a (i32.const 100)))
@@ -365,7 +365,7 @@
(start $f)
(elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
-(assert_fuel 403 ;; 400 bytes set + 1 func + 2 instructions (drop is 0)
+(assert_fuel 105
(module
(type $a (array (mut funcref)))
(func $f
@@ -375,7 +375,7 @@
)
(start $f)))
-(assert_fuel 404 ;; 400 bytes set + 1 func + 3 instructions (drop is 0)
+(assert_fuel 106
(module
(type $a (array (mut funcref)))
(func $f
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index 7301fe6bf2..b86a73f0b4 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -373,6 +373,7 @@ fn table_drops_externref() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn global_init_no_leak() -> Result<()> {
let (mut store, module) = ref_types_module(
false,
@@ -1157,6 +1158,7 @@ fn issue_9669() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn drc_transitive_drop_cons_list() -> Result<()> {
let _ = env_logger::try_init();
@@ -3315,6 +3317,8 @@ fn typed_option_noextern() -> Result<()> {
/// A test that performs a GC without actually compiling or running any Wasm
/// functions so we can run this test under MIRI.
#[test]
+// FIXME: need to fold this into other miri testing which precompiles
+#[cfg_attr(miri, ignore)]
fn miri_gc_smoke_test() -> Result<()> {
for collector in [
Collector::Copying,
diff --git a/tests/all/memory.rs b/tests/all/memory.rs
index 8897ad081d..f321f5d98c 100644
--- a/tests/all/memory.rs
+++ b/tests/all/memory.rs
@@ -510,6 +510,7 @@ fn dynamic_extra_growth_unchanged_pointer(config: &mut Config) -> Result<()> {
// determining this failure we shouldn't hit any overflows or anything like that
// (checked via debug-mode tests).
#[wasmtime_test]
+#[cfg_attr(miri, ignore)]
fn memory64_maximum_minimum(config: &mut Config) -> Result<()> {
config.wasm_memory64(true);
let engine = Engine::new(&config)?;
diff --git a/tests/all/module.rs b/tests/all/module.rs
index da99f7442b..6a4daff1a7 100644
--- a/tests/all/module.rs
+++ b/tests/all/module.rs
@@ -141,6 +141,7 @@ fn serialize_deterministic() {
// into an initialization image doesn't unnecessarily create a massive module by
// accident with a very large initialization image in it.
#[test]
+#[cfg_attr(miri, ignore)]
fn serialize_not_overly_massive() -> Result<()> {
let mut config = Config::new();
config.memory_guaranteed_dense_image_size(1 << 20);
diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs
index c14a6dbbc5..ffb7482003 100644
--- a/tests/all/pooling_allocator.rs
+++ b/tests/all/pooling_allocator.rs
@@ -98,6 +98,7 @@ fn memory_limit() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn memory_init() -> Result<()> {
let mut pool = crate::small_pool_config();
pool.max_memory_size(2 << 16).table_elements(0);
@@ -441,6 +442,7 @@ fn total_core_instances_limit() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn preserve_data_segments() -> Result<()> {
let mut pool = crate::small_pool_config();
pool.total_memories(2);
@@ -516,6 +518,7 @@ fn multi_memory_with_imported_memories() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn drop_externref_global_during_module_init() -> Result<()> {
struct Limiter;
@@ -1248,6 +1251,7 @@ fn decommit_batching() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn tricky_empty_table_with_empty_virtual_memory_alloc() -> Result<()> {
// Configure the pooling allocator to have no access to virtual memory, e.g.
// no table elements but a single table. This should technically support a
@@ -1458,6 +1462,7 @@ fn pooling_reuse_resets() -> Result<()> {
// asserts that the previous image is indeed not available any more as that
// would otherwise mean data was leaked between modules.
#[test]
+#[cfg_attr(miri, ignore)]
fn memory_reset_if_instantiation_fails() -> Result<()> {
struct Limiter;
diff --git a/tests/all/structs.rs b/tests/all/structs.rs
index 59e14ae087..652f028f31 100644
--- a/tests/all/structs.rs
+++ b/tests/all/structs.rs
@@ -912,6 +912,7 @@ fn issue_9714(config: &mut Config) -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn host_structref_has_trace_info_for_gc() -> Result<()> {
for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {
println!("Using GC collector: {collector:?}");
diff --git a/tests/all/table.rs b/tests/all/table.rs
index b4375c479a..6d8d9bd843 100644
--- a/tests/all/table.rs
+++ b/tests/all/table.rs
@@ -368,6 +368,7 @@ fn host_table_keep_type_registration() -> Result<()> {
}
#[test]
+#[cfg_attr(miri, ignore)]
fn gc_store_with_table_initializers() -> Result<()> {
let mut config = Config::new();
config.wasm_gc(true);
diff --git a/tests/disas/array-fill-anyref.wat b/tests/disas/array-fill-anyref.wat
index 52bc524188..428cb320f9 100644
--- a/tests/disas/array-fill-anyref.wat
+++ b/tests/disas/array-fill-anyref.wat
@@ -81,14 +81,12 @@
;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
-;; sig0 = (i64 vmctx, i64, i32, i64) tail
-;; fn0 = colocated u805306368:2 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
;; @003e trapz v2, user16
-;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8
-;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32
+;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8
+;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32
;; @003e v6 = uextend.i64 v2
;; @003e v8 = iadd v7, v6
;; @003e v9 = iconst.i64 16
@@ -100,19 +98,33 @@
;; @003e v12 = uextend.i64 v11
;; @003e v17 = icmp ugt v16, v12
;; @003e trapnz v17, user17
-;; @003e v28 = load.i64 notrap aligned v44+40
-;; v40 = iconst.i64 20
-;; @003e v21 = iadd v8, v40 ; v40 = 20
-;; v48 = iconst.i64 2
-;; v49 = ishl v13, v48 ; v48 = 2
-;; @003e v24 = iadd v21, v49
-;; v51 = ishl v14, v48 ; v48 = 2
-;; @003e v30 = uadd_overflow_trap v24, v51, user2
+;; @003e v28 = load.i64 notrap aligned v49+40
+;; v45 = iconst.i64 20
+;; @003e v21 = iadd v8, v45 ; v45 = 20
+;; v53 = iconst.i64 2
+;; v54 = ishl v13, v53 ; v53 = 2
+;; @003e v24 = iadd v21, v54
+;; v56 = ishl v14, v53 ; v53 = 2
+;; @003e v30 = uadd_overflow_trap v24, v56, user2
;; @003e v29 = iadd v7, v28
;; @003e v31 = icmp ugt v30, v29
;; @003e trapnz v31, user2
+;; v51 = iconst.i64 0
+;; @003e v33 = icmp eq v14, v51 ; v51 = 0
;; @003a v5 = iconst.i32 0
-;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0
+;; v44 = iconst.i64 4
+;; @003e v32 = iadd v24, v56
+;; @003e brif v33, block3, block2(v24)
+;;
+;; block2(v34: i64):
+;; v58 = iconst.i32 0
+;; @003e store user2 little v58, v34 ; v58 = 0
+;; v59 = iconst.i64 4
+;; v60 = iadd v34, v59 ; v59 = 4
+;; @003e v36 = icmp eq v60, v32
+;; @003e brif v36, block3, block2(v60)
+;;
+;; block3:
;; @0041 jump block1
;;
;; block1:
diff --git a/tests/disas/array-fill-externref.wat b/tests/disas/array-fill-externref.wat
index a327322ede..542857bb45 100644
--- a/tests/disas/array-fill-externref.wat
+++ b/tests/disas/array-fill-externref.wat
@@ -77,14 +77,12 @@
;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
-;; sig0 = (i64 vmctx, i64, i32, i64) tail
-;; fn0 = colocated u805306368:2 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
;; @003d trapz v2, user16
-;; @003d v44 = load.i64 notrap aligned readonly can_move v0+8
-;; @003d v7 = load.i64 notrap aligned readonly can_move v44+32
+;; @003d v49 = load.i64 notrap aligned readonly can_move v0+8
+;; @003d v7 = load.i64 notrap aligned readonly can_move v49+32
;; @003d v6 = uextend.i64 v2
;; @003d v8 = iadd v7, v6
;; @003d v9 = iconst.i64 16
@@ -96,19 +94,33 @@
;; @003d v12 = uextend.i64 v11
;; @003d v17 = icmp ugt v16, v12
;; @003d trapnz v17, user17
-;; @003d v28 = load.i64 notrap aligned v44+40
-;; v40 = iconst.i64 20
-;; @003d v21 = iadd v8, v40 ; v40 = 20
-;; v48 = iconst.i64 2
-;; v49 = ishl v13, v48 ; v48 = 2
-;; @003d v24 = iadd v21, v49
-;; v51 = ishl v14, v48 ; v48 = 2
-;; @003d v30 = uadd_overflow_trap v24, v51, user2
+;; @003d v28 = load.i64 notrap aligned v49+40
+;; v45 = iconst.i64 20
+;; @003d v21 = iadd v8, v45 ; v45 = 20
+;; v53 = iconst.i64 2
+;; v54 = ishl v13, v53 ; v53 = 2
+;; @003d v24 = iadd v21, v54
+;; v56 = ishl v14, v53 ; v53 = 2
+;; @003d v30 = uadd_overflow_trap v24, v56, user2
;; @003d v29 = iadd v7, v28
;; @003d v31 = icmp ugt v30, v29
;; @003d trapnz v31, user2
+;; v51 = iconst.i64 0
+;; @003d v33 = icmp eq v14, v51 ; v51 = 0
;; @0039 v5 = iconst.i32 0
-;; @003d call fn0(v0, v24, v5, v51) ; v5 = 0
+;; v44 = iconst.i64 4
+;; @003d v32 = iadd v24, v56
+;; @003d brif v33, block3, block2(v24)
+;;
+;; block2(v34: i64):
+;; v58 = iconst.i32 0
+;; @003d store user2 little v58, v34 ; v58 = 0
+;; v59 = iconst.i64 4
+;; v60 = iadd v34, v59 ; v59 = 4
+;; @003d v36 = icmp eq v60, v32
+;; @003d brif v36, block3, block2(v60)
+;;
+;; block3:
;; @0040 jump block1
;;
;; block1:
diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.wat
index 4f594096bb..026b754942 100644
--- a/tests/disas/array-fill-funcref.wat
+++ b/tests/disas/array-fill-funcref.wat
@@ -88,14 +88,14 @@
;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
-;; sig0 = (i64 vmctx, i64, i32, i64) tail
-;; fn0 = colocated u805306368:2 sig0
+;; sig0 = (i64 vmctx, i64) -> i64 tail
+;; fn0 = colocated u805306368:25 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
;; @0049 trapz v2, user16
-;; @0049 v44 = load.i64 notrap aligned readonly can_move v0+8
-;; @0049 v7 = load.i64 notrap aligned readonly can_move v44+32
+;; @0049 v52 = load.i64 notrap aligned readonly can_move v0+8
+;; @0049 v7 = load.i64 notrap aligned readonly can_move v52+32
;; @0049 v6 = uextend.i64 v2
;; @0049 v8 = iadd v7, v6
;; @0049 v9 = iconst.i64 16
@@ -107,19 +107,34 @@
;; @0049 v12 = uextend.i64 v11
;; @0049 v17 = icmp ugt v16, v12
;; @0049 trapnz v17, user17
-;; @0049 v28 = load.i64 notrap aligned v44+40
-;; v40 = iconst.i64 20
-;; @0049 v21 = iadd v8, v40 ; v40 = 20
-;; v47 = iconst.i64 2
-;; v48 = ishl v13, v47 ; v47 = 2
-;; @0049 v24 = iadd v21, v48
-;; v50 = ishl v14, v47 ; v47 = 2
-;; @0049 v30 = uadd_overflow_trap v24, v50, user2
+;; @0049 v28 = load.i64 notrap aligned v52+40
+;; v48 = iconst.i64 20
+;; @0049 v21 = iadd v8, v48 ; v48 = 20
+;; v55 = iconst.i64 2
+;; v56 = ishl v13, v55 ; v55 = 2
+;; @0049 v24 = iadd v21, v56
+;; v58 = ishl v14, v55 ; v55 = 2
+;; @0049 v30 = uadd_overflow_trap v24, v58, user2
;; @0049 v29 = iadd v7, v28
;; @0049 v31 = icmp ugt v30, v29
;; @0049 trapnz v31, user2
-;; @0049 v32 = iconst.i32 0
-;; @0049 call fn0(v0, v24, v32, v50) ; v32 = 0
+;; @0045 v5 = iconst.i64 0
+;; @0049 v33 = icmp eq v14, v5 ; v5 = 0
+;; v47 = iconst.i64 4
+;; @0049 v32 = iadd v24, v58
+;; @0049 brif v33, block3, block2(v24)
+;;
+;; block2(v34: i64):
+;; v60 = iconst.i64 0
+;; @0049 v36 = call fn0(v0, v60) ; v60 = 0
+;; @0049 v37 = ireduce.i32 v36
+;; @0049 store user2 little v37, v34
+;; v61 = iconst.i64 4
+;; v62 = iadd v34, v61 ; v61 = 4
+;; @0049 v39 = icmp eq v62, v32
+;; @0049 brif v39, block3, block2(v62)
+;;
+;; block3:
;; @004c jump block1
;;
;; block1:
diff --git a/tests/disas/array-fill-i31ref.wat b/tests/disas/array-fill-i31ref.wat
index 6723ac376d..b094bdde3f 100644
--- a/tests/disas/array-fill-i31ref.wat
+++ b/tests/disas/array-fill-i31ref.wat
@@ -81,14 +81,12 @@
;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
-;; sig0 = (i64 vmctx, i64, i32, i64) tail
-;; fn0 = colocated u805306368:2 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
;; @003e trapz v2, user16
-;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8
-;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32
+;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8
+;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32
;; @003e v6 = uextend.i64 v2
;; @003e v8 = iadd v7, v6
;; @003e v9 = iconst.i64 16
@@ -100,19 +98,33 @@
;; @003e v12 = uextend.i64 v11
;; @003e v17 = icmp ugt v16, v12
;; @003e trapnz v17, user17
-;; @003e v28 = load.i64 notrap aligned v44+40
-;; v40 = iconst.i64 20
-;; @003e v21 = iadd v8, v40 ; v40 = 20
-;; v48 = iconst.i64 2
-;; v49 = ishl v13, v48 ; v48 = 2
-;; @003e v24 = iadd v21, v49
-;; v51 = ishl v14, v48 ; v48 = 2
-;; @003e v30 = uadd_overflow_trap v24, v51, user2
+;; @003e v28 = load.i64 notrap aligned v49+40
+;; v45 = iconst.i64 20
+;; @003e v21 = iadd v8, v45 ; v45 = 20
+;; v53 = iconst.i64 2
+;; v54 = ishl v13, v53 ; v53 = 2
+;; @003e v24 = iadd v21, v54
+;; v56 = ishl v14, v53 ; v53 = 2
+;; @003e v30 = uadd_overflow_trap v24, v56, user2
;; @003e v29 = iadd v7, v28
;; @003e v31 = icmp ugt v30, v29
;; @003e trapnz v31, user2
+;; v51 = iconst.i64 0
+;; @003e v33 = icmp eq v14, v51 ; v51 = 0
;; @003a v5 = iconst.i32 0
-;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0
+;; v44 = iconst.i64 4
+;; @003e v32 = iadd v24, v56
+;; @003e brif v33, block3, block2(v24)
+;;
+;; block2(v34: i64):
+;; v58 = iconst.i32 0
+;; @003e store user2 little v58, v34 ; v58 = 0
+;; v59 = iconst.i64 4
+;; v60 = iadd v34, v59 ; v59 = 4
+;; @003e v36 = icmp eq v60, v32
+;; @003e brif v36, block3, block2(v60)
+;;
+;; block3:
;; @0041 jump block1
;;
;; block1:
diff --git a/tests/disas/gc/array-init-data.wat b/tests/disas/gc/array-init-data.wat
index 39ff89c798..c44bdab873 100644
--- a/tests/disas/gc/array-init-data.wat
+++ b/tests/disas/gc/array-init-data.wat
@@ -27,8 +27,8 @@
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
;; @002a trapz v2, user16
-;; @002a v57 = load.i64 notrap aligned readonly can_move v0+8
-;; @002a v7 = load.i64 notrap aligned readonly can_move v57+32
+;; @002a v58 = load.i64 notrap aligned readonly can_move v0+8
+;; @002a v7 = load.i64 notrap aligned readonly can_move v58+32
;; @002a v6 = uextend.i64 v2
;; @002a v8 = iadd v7, v6
;; @002a v9 = iconst.i64 16
@@ -40,22 +40,23 @@
;; @002a v12 = uextend.i64 v11
;; @002a v17 = icmp ugt v16, v12
;; @002a trapnz v17, user17
-;; @002a v26 = uload32 notrap aligned v0+56
-;; @002a v27 = uextend.i64 v4
-;; @002a v30 = iadd v27, v14
-;; @002a v31 = icmp ugt v30, v26
-;; @002a trapnz v31, heap_oob
-;; @002a v33 = load.i64 notrap aligned v0+48
-;; @002a v40 = load.i64 notrap aligned v57+40
-;; v53 = iconst.i64 20
-;; @002a v21 = iadd v8, v53 ; v53 = 20
+;; @002a v26 = load.i32 notrap aligned v0+56
+;; @002a v28 = uextend.i64 v4
+;; @002a v31 = iadd v28, v14
+;; @002a v27 = uextend.i64 v26
+;; @002a v32 = icmp ugt v31, v27
+;; @002a trapnz v32, heap_oob
+;; @002a v34 = load.i64 notrap aligned v0+48
+;; @002a v41 = load.i64 notrap aligned v58+40
+;; v54 = iconst.i64 20
+;; @002a v21 = iadd v8, v54 ; v54 = 20
;; @002a v24 = iadd v21, v13
-;; @002a v42 = uadd_overflow_trap v24, v14, user2
-;; @002a v41 = iadd v7, v40
-;; @002a v43 = icmp ugt v42, v41
-;; @002a trapnz v43, user2
-;; @002a v35 = iadd v33, v27
-;; @002a call fn0(v0, v24, v35, v14)
+;; @002a v43 = uadd_overflow_trap v24, v14, user2
+;; @002a v42 = iadd v7, v41
+;; @002a v44 = icmp ugt v43, v42
+;; @002a trapnz v44, user2
+;; @002a v36 = iadd v34, v28
+;; @002a call fn0(v0, v24, v36, v14)
;; @002e jump block1
;;
;; block1:
diff --git a/tests/disas/gc/array-new-data.wat b/tests/disas/gc/array-new-data.wat
index 2ff85b8dfb..29e109b03e 100644
--- a/tests/disas/gc/array-new-data.wat
+++ b/tests/disas/gc/array-new-data.wat
@@ -92,96 +92,98 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
-;; @0025 v6 = uload32 notrap aligned v0+56
-;; @0025 v7 = uextend.i64 v2
-;; @0025 v8 = uextend.i64 v3
-;; @0025 v10 = iadd v7, v8
-;; @0025 v11 = icmp ugt v10, v6
-;; @0025 trapnz v11, heap_oob
-;; @0025 v13 = load.i64 notrap aligned v0+48
-;; v124 = iconst.i64 32
-;; @0025 v19 = ushr v8, v124 ; v124 = 32
-;; @0025 trapnz v19, user18
-;; @0025 v16 = iconst.i32 20
-;; @0025 v21 = uadd_overflow_trap v16, v3, user18 ; v16 = 20
-;; @0025 v23 = load.i64 notrap aligned readonly can_move v0+32
-;; @0025 v24 = load.i32 notrap aligned v23
-;; @0025 v25 = load.i32 notrap aligned v23+4
-;; @0025 v31 = uextend.i64 v24
-;; @0025 v26 = uextend.i64 v21
-;; @0025 v27 = iconst.i64 15
-;; @0025 v29 = iadd v26, v27 ; v27 = 15
-;; @0025 v28 = iconst.i64 -16
-;; @0025 v30 = band v29, v28 ; v28 = -16
-;; @0025 v32 = iadd v31, v30
-;; @0025 v33 = uextend.i64 v25
-;; @0025 v34 = icmp ule v32, v33
-;; @0025 brif v34, block2, block3
+;; @0025 v6 = load.i32 notrap aligned v0+56
+;; @0025 v8 = uextend.i64 v2
+;; @0025 v9 = uextend.i64 v3
+;; @0025 v11 = iadd v8, v9
+;; @0025 v7 = uextend.i64 v6
+;; @0025 v12 = icmp ugt v11, v7
+;; @0025 trapnz v12, heap_oob
+;; @0025 v14 = load.i64 notrap aligned v0+48
+;; v126 = iconst.i64 32
+;; @0025 v20 = ushr v9, v126 ; v126 = 32
+;; @0025 trapnz v20, user18
+;; @0025 v17 = iconst.i32 20
+;; @0025 v22 = uadd_overflow_trap v17, v3, user18 ; v17 = 20
+;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+32
+;; @0025 v25 = load.i32 notrap aligned v24
+;; @0025 v26 = load.i32 notrap aligned v24+4
+;; @0025 v32 = uextend.i64 v25
+;; @0025 v27 = uextend.i64 v22
+;; @0025 v28 = iconst.i64 15
+;; @0025 v30 = iadd v27, v28 ; v28 = 15
+;; @0025 v29 = iconst.i64 -16
+;; @0025 v31 = band v30, v29 ; v29 = -16
+;; @0025 v33 = iadd v32, v31
+;; @0025 v34 = uextend.i64 v26
+;; @0025 v35 = icmp ule v33, v34
+;; @0025 brif v35, block2, block3
;;
;; block2:
-;; v134 = iconst.i32 15
-;; v135 = iadd.i32 v21, v134 ; v134 = 15
-;; v138 = iconst.i32 -16
-;; v139 = band v135, v138 ; v138 = -16
-;; v141 = iadd.i32 v24, v139
-;; @0025 store notrap aligned region0 v141, v23
-;; v155 = iconst.i32 -1476395002
-;; v156 = load.i64 notrap aligned readonly can_move v0+8
-;; v157 = load.i64 notrap aligned readonly can_move v156+32
-;; @0025 v48 = iadd v157, v31
-;; @0025 store notrap aligned v155, v48 ; v155 = -1476395002
-;; v158 = load.i64 notrap aligned readonly can_move v0+40
-;; v159 = load.i32 notrap aligned readonly can_move v158
-;; @0025 store notrap aligned v159, v48+4
-;; v160 = band.i64 v29, v28 ; v28 = -16
-;; @0025 istore32 notrap aligned v160, v48+8
-;; @0025 jump block4(v24, v48)
+;; v136 = iconst.i32 15
+;; v137 = iadd.i32 v22, v136 ; v136 = 15
+;; v140 = iconst.i32 -16
+;; v141 = band v137, v140 ; v140 = -16
+;; v143 = iadd.i32 v25, v141
+;; @0025 store notrap aligned region0 v143, v24
+;; v157 = iconst.i32 -1476395002
+;; v158 = load.i64 notrap aligned readonly can_move v0+8
+;; v159 = load.i64 notrap aligned readonly can_move v158+32
+;; @0025 v49 = iadd v159, v32
+;; @0025 store notrap aligned v157, v49 ; v157 = -1476395002
+;; v160 = load.i64 notrap aligned readonly can_move v0+40
+;; v161 = load.i32 notrap aligned readonly can_move v160
+;; @0025 store notrap aligned v161, v49+4
+;; v162 = band.i64 v30, v29 ; v29 = -16
+;; @0025 istore32 notrap aligned v162, v49+8
+;; @0025 jump block4(v25, v49)
;;
;; block3 cold:
-;; @0025 v36 = iconst.i32 -1476395002
-;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+40
-;; @0025 v39 = load.i32 notrap aligned readonly can_move v38
-;; @0025 v40 = iconst.i32 16
-;; @0025 v41 = call fn0(v0, v36, v39, v21, v40) ; v36 = -1476395002, v40 = 16
-;; @0025 v120 = load.i64 notrap aligned readonly can_move v0+8
-;; @0025 v42 = load.i64 notrap aligned readonly can_move v120+32
-;; @0025 v43 = uextend.i64 v41
-;; @0025 v44 = iadd v42, v43
-;; @0025 jump block4(v41, v44)
+;; @0025 v37 = iconst.i32 -1476395002
+;; @0025 v39 = load.i64 notrap aligned readonly can_move v0+40
+;; @0025 v40 = load.i32 notrap aligned readonly can_move v39
+;; @0025 v41 = iconst.i32 16
+;; @0025 v42 = call fn0(v0, v37, v40, v22, v41) ; v37 = -1476395002, v41 = 16
+;; @0025 v122 = load.i64 notrap aligned readonly can_move v0+8
+;; @0025 v43 = load.i64 notrap aligned readonly can_move v122+32
+;; @0025 v44 = uextend.i64 v42
+;; @0025 v45 = iadd v43, v44
+;; @0025 jump block4(v42, v45)
;;
-;; block4(v53: i32, v54: i64):
-;; v119 = stack_addr.i64 ss0
-;; store notrap v53, v119
-;; v118 = iconst.i64 16
-;; @0025 v55 = iadd v54, v118 ; v118 = 16
-;; @0025 store.i32 user2 v3, v55
-;; v99 = load.i32 notrap v119
-;; @0025 trapz v99, user16
-;; v161 = load.i64 notrap aligned readonly can_move v0+8
-;; v162 = load.i64 notrap aligned readonly can_move v161+32
-;; @0025 v57 = uextend.i64 v99
-;; @0025 v59 = iadd v162, v57
-;; @0025 v61 = iadd v59, v118 ; v118 = 16
-;; @0025 v62 = load.i32 user2 readonly v61
-;; @0025 v63 = uextend.i64 v62
-;; @0025 v68 = icmp.i64 ugt v8, v63
-;; @0025 trapnz v68, user17
-;; @0025 v77 = uload32.i64 notrap aligned v0+56
-;; @0025 v82 = icmp.i64 ugt v10, v77
-;; @0025 trapnz v82, heap_oob
-;; @0025 v84 = load.i64 notrap aligned v0+48
-;; @0025 v91 = load.i64 notrap aligned v161+40
-;; v109 = iconst.i64 20
-;; @0025 v72 = iadd v59, v109 ; v109 = 20
-;; @0025 v93 = uadd_overflow_trap v72, v8, user2
-;; @0025 v92 = iadd v162, v91
-;; @0025 v94 = icmp ugt v93, v92
-;; @0025 trapnz v94, user2
-;; @0025 v86 = iadd v84, v7
-;; @0025 call fn1(v0, v72, v86, v8), stack_map=[i32 @ ss0+0]
-;; v96 = load.i32 notrap v119
+;; block4(v54: i32, v55: i64):
+;; v121 = stack_addr.i64 ss0
+;; store notrap v54, v121
+;; v120 = iconst.i64 16
+;; @0025 v56 = iadd v55, v120 ; v120 = 16
+;; @0025 store.i32 user2 v3, v56
+;; v101 = load.i32 notrap v121
+;; @0025 trapz v101, user16
+;; v163 = load.i64 notrap aligned readonly can_move v0+8
+;; v164 = load.i64 notrap aligned readonly can_move v163+32
+;; @0025 v58 = uextend.i64 v101
+;; @0025 v60 = iadd v164, v58
+;; @0025 v62 = iadd v60, v120 ; v120 = 16
+;; @0025 v63 = load.i32 user2 readonly v62
+;; @0025 v64 = uextend.i64 v63
+;; @0025 v69 = icmp.i64 ugt v9, v64
+;; @0025 trapnz v69, user17
+;; @0025 v78 = load.i32 notrap aligned v0+56
+;; @0025 v79 = uextend.i64 v78
+;; @0025 v84 = icmp.i64 ugt v11, v79
+;; @0025 trapnz v84, heap_oob
+;; @0025 v86 = load.i64 notrap aligned v0+48
+;; @0025 v93 = load.i64 notrap aligned v163+40
+;; v111 = iconst.i64 20
+;; @0025 v73 = iadd v60, v111 ; v111 = 20
+;; @0025 v95 = uadd_overflow_trap v73, v9, user2
+;; @0025 v94 = iadd v164, v93
+;; @0025 v96 = icmp ugt v95, v94
+;; @0025 trapnz v96, user2
+;; @0025 v88 = iadd v86, v8
+;; @0025 call fn1(v0, v73, v88, v9), stack_map=[i32 @ ss0+0]
+;; v98 = load.i32 notrap v121
;; @0029 jump block1
;;
;; block1:
-;; @0029 return v96
+;; @0029 return v98
;; }
diff --git a/tests/disas/gc/array-new-default-anyref.wat b/tests/disas/gc/array-new-default-anyref.wat
index 82015ac769..c80b3f3925 100644
--- a/tests/disas/gc/array-new-default-anyref.wat
+++ b/tests/disas/gc/array-new-default-anyref.wat
@@ -10,7 +10,6 @@
)
)
;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
-;; ss0 = explicit_slot 4, align = 4
;; region0 = 2 "vmctx"
;; gv0 = vmctx
;; gv1 = load.i64 notrap aligned readonly gv0+8
@@ -20,22 +19,20 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail
-;; sig1 = (i64 vmctx, i64, i32, i64) tail
;; fn0 = colocated u805306368:24 sig0
-;; fn1 = colocated u805306368:2 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
;; @001f v5 = uextend.i64 v2
-;; v102 = iconst.i64 2
-;; v103 = ishl v5, v102 ; v102 = 2
-;; v100 = iconst.i64 32
-;; @001f v7 = ushr v103, v100 ; v100 = 32
+;; v98 = iconst.i64 2
+;; v99 = ishl v5, v98 ; v98 = 2
+;; v96 = iconst.i64 32
+;; @001f v7 = ushr v99, v96 ; v96 = 32
;; @001f trapnz v7, user18
;; @001f v4 = iconst.i32 20
-;; v109 = iconst.i32 2
-;; v110 = ishl v2, v109 ; v109 = 2
-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20
+;; v105 = iconst.i32 2
+;; v106 = ishl v2, v105 ; v105 = 2
+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20
;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32
;; @001f v12 = load.i32 notrap aligned v11
;; @001f v13 = load.i32 notrap aligned v11+4
@@ -51,22 +48,22 @@
;; @001f brif v22, block2, block3
;;
;; block2:
-;; v118 = iconst.i32 15
-;; v119 = iadd.i32 v9, v118 ; v118 = 15
-;; v122 = iconst.i32 -16
-;; v123 = band v119, v122 ; v122 = -16
-;; v125 = iadd.i32 v12, v123
-;; @001f store notrap aligned region0 v125, v11
-;; v141 = iconst.i32 -1476394994
-;; v142 = load.i64 notrap aligned readonly can_move v0+8
-;; v143 = load.i64 notrap aligned readonly can_move v142+32
-;; @001f v36 = iadd v143, v19
-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994
-;; v144 = load.i64 notrap aligned readonly can_move v0+40
-;; v145 = load.i32 notrap aligned readonly can_move v144
-;; @001f store notrap aligned v145, v36+4
-;; v146 = band.i64 v17, v16 ; v16 = -16
-;; @001f istore32 notrap aligned v146, v36+8
+;; v114 = iconst.i32 15
+;; v115 = iadd.i32 v9, v114 ; v114 = 15
+;; v118 = iconst.i32 -16
+;; v119 = band v115, v118 ; v118 = -16
+;; v121 = iadd.i32 v12, v119
+;; @001f store notrap aligned region0 v121, v11
+;; v137 = iconst.i32 -1476394994
+;; v138 = load.i64 notrap aligned readonly can_move v0+8
+;; v139 = load.i64 notrap aligned readonly can_move v138+32
+;; @001f v36 = iadd v139, v19
+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994
+;; v140 = load.i64 notrap aligned readonly can_move v0+40
+;; v141 = load.i32 notrap aligned readonly can_move v140
+;; @001f store notrap aligned v141, v36+4
+;; v142 = band.i64 v17, v16 ; v16 = -16
+;; @001f istore32 notrap aligned v142, v36+8
;; @001f jump block4(v12, v36)
;;
;; block3 cold:
@@ -75,41 +72,51 @@
;; @001f v27 = load.i32 notrap aligned readonly can_move v26
;; @001f v28 = iconst.i32 16
;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16
-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8
-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32
+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8
+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32
;; @001f v31 = uextend.i64 v29
;; @001f v32 = iadd v30, v31
;; @001f jump block4(v29, v32)
;;
;; block4(v41: i32, v42: i64):
-;; v95 = stack_addr.i64 ss0
-;; store notrap v41, v95
-;; v94 = iconst.i64 16
-;; @001f v43 = iadd v42, v94 ; v94 = 16
+;; v91 = iconst.i64 16
+;; @001f v43 = iadd v42, v91 ; v91 = 16
;; @001f store.i32 user2 v2, v43
-;; v77 = load.i32 notrap v95
-;; @001f trapz v77, user16
-;; v147 = load.i64 notrap aligned readonly can_move v0+8
-;; v148 = load.i64 notrap aligned readonly can_move v147+32
-;; @001f v46 = uextend.i64 v77
-;; @001f v48 = iadd v148, v46
-;; @001f v50 = iadd v48, v94 ; v94 = 16
+;; @001f trapz v41, user16
+;; v143 = load.i64 notrap aligned readonly can_move v0+8
+;; v144 = load.i64 notrap aligned readonly can_move v143+32
+;; @001f v46 = uextend.i64 v41
+;; @001f v48 = iadd v144, v46
+;; @001f v50 = iadd v48, v91 ; v91 = 16
;; @001f v51 = load.i32 user2 readonly v50
;; @001f v52 = uextend.i64 v51
;; @001f v57 = icmp.i64 ugt v5, v52
;; @001f trapnz v57, user17
-;; @001f v68 = load.i64 notrap aligned v147+40
+;; @001f v68 = load.i64 notrap aligned v143+40
;; v85 = iconst.i64 20
;; @001f v61 = iadd v48, v85 ; v85 = 20
-;; @001f v70 = uadd_overflow_trap v61, v103, user2
-;; @001f v69 = iadd v148, v68
+;; @001f v70 = uadd_overflow_trap v61, v99, user2
+;; @001f v69 = iadd v144, v68
;; @001f v71 = icmp ugt v70, v69
;; @001f trapnz v71, user2
+;; v123 = iconst.i64 0
+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0
;; @001f v44 = iconst.i32 0
-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0
-;; v74 = load.i32 notrap v95
-;; @0022 jump block1
+;; v97 = iconst.i64 4
+;; @001f v72 = iadd v61, v99
+;; @001f brif v73, block6, block5(v61)
;;
-;; block1:
-;; @0022 return v74
+;; block5(v74: i64):
+;; v145 = iconst.i32 0
+;; @001f store user2 little v145, v74 ; v145 = 0
+;; v146 = iconst.i64 4
+;; v147 = iadd v74, v146 ; v146 = 4
+;; @001f v76 = icmp eq v147, v72
+;; @001f brif v76, block6, block5(v147)
+;;
+;; block6:
+;; @0022 jump block1(v41)
+;;
+;; block1(v3: i32):
+;; @0022 return v3
;; }
diff --git a/tests/disas/gc/array-new-default-exnref.wat b/tests/disas/gc/array-new-default-exnref.wat
index 910d21579e..10e6a6d2a3 100644
--- a/tests/disas/gc/array-new-default-exnref.wat
+++ b/tests/disas/gc/array-new-default-exnref.wat
@@ -10,7 +10,6 @@
)
)
;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
-;; ss0 = explicit_slot 4, align = 4
;; region0 = 2 "vmctx"
;; gv0 = vmctx
;; gv1 = load.i64 notrap aligned readonly gv0+8
@@ -20,22 +19,20 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail
-;; sig1 = (i64 vmctx, i64, i32, i64) tail
;; fn0 = colocated u805306368:24 sig0
-;; fn1 = colocated u805306368:2 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
;; @001f v5 = uextend.i64 v2
-;; v102 = iconst.i64 2
-;; v103 = ishl v5, v102 ; v102 = 2
-;; v100 = iconst.i64 32
-;; @001f v7 = ushr v103, v100 ; v100 = 32
+;; v98 = iconst.i64 2
+;; v99 = ishl v5, v98 ; v98 = 2
+;; v96 = iconst.i64 32
+;; @001f v7 = ushr v99, v96 ; v96 = 32
;; @001f trapnz v7, user18
;; @001f v4 = iconst.i32 20
-;; v109 = iconst.i32 2
-;; v110 = ishl v2, v109 ; v109 = 2
-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20
+;; v105 = iconst.i32 2
+;; v106 = ishl v2, v105 ; v105 = 2
+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20
;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32
;; @001f v12 = load.i32 notrap aligned v11
;; @001f v13 = load.i32 notrap aligned v11+4
@@ -51,22 +48,22 @@
;; @001f brif v22, block2, block3
;;
;; block2:
-;; v118 = iconst.i32 15
-;; v119 = iadd.i32 v9, v118 ; v118 = 15
-;; v122 = iconst.i32 -16
-;; v123 = band v119, v122 ; v122 = -16
-;; v125 = iadd.i32 v12, v123
-;; @001f store notrap aligned region0 v125, v11
-;; v141 = iconst.i32 -1476394994
-;; v142 = load.i64 notrap aligned readonly can_move v0+8
-;; v143 = load.i64 notrap aligned readonly can_move v142+32
-;; @001f v36 = iadd v143, v19
-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994
-;; v144 = load.i64 notrap aligned readonly can_move v0+40
-;; v145 = load.i32 notrap aligned readonly can_move v144
-;; @001f store notrap aligned v145, v36+4
-;; v146 = band.i64 v17, v16 ; v16 = -16
-;; @001f istore32 notrap aligned v146, v36+8
+;; v114 = iconst.i32 15
+;; v115 = iadd.i32 v9, v114 ; v114 = 15
+;; v118 = iconst.i32 -16
+;; v119 = band v115, v118 ; v118 = -16
+;; v121 = iadd.i32 v12, v119
+;; @001f store notrap aligned region0 v121, v11
+;; v137 = iconst.i32 -1476394994
+;; v138 = load.i64 notrap aligned readonly can_move v0+8
+;; v139 = load.i64 notrap aligned readonly can_move v138+32
+;; @001f v36 = iadd v139, v19
+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994
+;; v140 = load.i64 notrap aligned readonly can_move v0+40
+;; v141 = load.i32 notrap aligned readonly can_move v140
+;; @001f store notrap aligned v141, v36+4
+;; v142 = band.i64 v17, v16 ; v16 = -16
+;; @001f istore32 notrap aligned v142, v36+8
;; @001f jump block4(v12, v36)
;;
;; block3 cold:
@@ -75,41 +72,51 @@
;; @001f v27 = load.i32 notrap aligned readonly can_move v26
;; @001f v28 = iconst.i32 16
;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16
-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8
-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32
+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8
+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32
;; @001f v31 = uextend.i64 v29
;; @001f v32 = iadd v30, v31
;; @001f jump block4(v29, v32)
;;
;; block4(v41: i32, v42: i64):
-;; v95 = stack_addr.i64 ss0
-;; store notrap v41, v95
-;; v94 = iconst.i64 16
-;; @001f v43 = iadd v42, v94 ; v94 = 16
+;; v91 = iconst.i64 16
+;; @001f v43 = iadd v42, v91 ; v91 = 16
;; @001f store.i32 user2 v2, v43
-;; v77 = load.i32 notrap v95
-;; @001f trapz v77, user16
-;; v147 = load.i64 notrap aligned readonly can_move v0+8
-;; v148 = load.i64 notrap aligned readonly can_move v147+32
-;; @001f v46 = uextend.i64 v77
-;; @001f v48 = iadd v148, v46
-;; @001f v50 = iadd v48, v94 ; v94 = 16
+;; @001f trapz v41, user16
+;; v143 = load.i64 notrap aligned readonly can_move v0+8
+;; v144 = load.i64 notrap aligned readonly can_move v143+32
+;; @001f v46 = uextend.i64 v41
+;; @001f v48 = iadd v144, v46
+;; @001f v50 = iadd v48, v91 ; v91 = 16
;; @001f v51 = load.i32 user2 readonly v50
;; @001f v52 = uextend.i64 v51
;; @001f v57 = icmp.i64 ugt v5, v52
;; @001f trapnz v57, user17
-;; @001f v68 = load.i64 notrap aligned v147+40
+;; @001f v68 = load.i64 notrap aligned v143+40
;; v85 = iconst.i64 20
;; @001f v61 = iadd v48, v85 ; v85 = 20
-;; @001f v70 = uadd_overflow_trap v61, v103, user2
-;; @001f v69 = iadd v148, v68
+;; @001f v70 = uadd_overflow_trap v61, v99, user2
+;; @001f v69 = iadd v144, v68
;; @001f v71 = icmp ugt v70, v69
;; @001f trapnz v71, user2
+;; v123 = iconst.i64 0
+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0
;; @001f v44 = iconst.i32 0
-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0
-;; v74 = load.i32 notrap v95
-;; @0022 jump block1
+;; v97 = iconst.i64 4
+;; @001f v72 = iadd v61, v99
+;; @001f brif v73, block6, block5(v61)
;;
-;; block1:
-;; @0022 return v74
+;; block5(v74: i64):
+;; v145 = iconst.i32 0
+;; @001f store user2 little v145, v74 ; v145 = 0
+;; v146 = iconst.i64 4
+;; v147 = iadd v74, v146 ; v146 = 4
+;; @001f v76 = icmp eq v147, v72
+;; @001f brif v76, block6, block5(v147)
+;;
+;; block6:
+;; @0022 jump block1(v41)
+;;
+;; block1(v3: i32):
+;; @0022 return v3
;; }
diff --git a/tests/disas/gc/array-new-default-externref.wat b/tests/disas/gc/array-new-default-externref.wat
index 4a0771b172..bb5b2a4e56 100644
--- a/tests/disas/gc/array-new-default-externref.wat
+++ b/tests/disas/gc/array-new-default-externref.wat
@@ -10,7 +10,6 @@
)
)
;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
-;; ss0 = explicit_slot 4, align = 4
;; region0 = 2 "vmctx"
;; gv0 = vmctx
;; gv1 = load.i64 notrap aligned readonly gv0+8
@@ -20,22 +19,20 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail
-;; sig1 = (i64 vmctx, i64, i32, i64) tail
;; fn0 = colocated u805306368:24 sig0
-;; fn1 = colocated u805306368:2 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
;; @001f v5 = uextend.i64 v2
-;; v102 = iconst.i64 2
-;; v103 = ishl v5, v102 ; v102 = 2
-;; v100 = iconst.i64 32
-;; @001f v7 = ushr v103, v100 ; v100 = 32
+;; v98 = iconst.i64 2
+;; v99 = ishl v5, v98 ; v98 = 2
+;; v96 = iconst.i64 32
+;; @001f v7 = ushr v99, v96 ; v96 = 32
;; @001f trapnz v7, user18
;; @001f v4 = iconst.i32 20
-;; v109 = iconst.i32 2
-;; v110 = ishl v2, v109 ; v109 = 2
-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20
+;; v105 = iconst.i32 2
+;; v106 = ishl v2, v105 ; v105 = 2
+;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20
;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32
;; @001f v12 = load.i32 notrap aligned v11
;; @001f v13 = load.i32 notrap aligned v11+4
@@ -51,22 +48,22 @@
;; @001f brif v22, block2, block3
;;
;; block2:
-;; v118 = iconst.i32 15
-;; v119 = iadd.i32 v9, v118 ; v118 = 15
-;; v122 = iconst.i32 -16
-;; v123 = band v119, v122 ; v122 = -16
-;; v125 = iadd.i32 v12, v123
-;; @001f store notrap aligned region0 v125, v11
-;; v141 = iconst.i32 -1476394994
-;; v142 = load.i64 notrap aligned readonly can_move v0+8
-;; v143 = load.i64 notrap aligned readonly can_move v142+32
-;; @001f v36 = iadd v143, v19
-;; @001f store notrap aligned v141, v36 ; v141 = -1476394994
-;; v144 = load.i64 notrap aligned readonly can_move v0+40
-;; v145 = load.i32 notrap aligned readonly can_move v144
-;; @001f store notrap aligned v145, v36+4
-;; v146 = band.i64 v17, v16 ; v16 = -16
-;; @001f istore32 notrap aligned v146, v36+8
+;; v114 = iconst.i32 15
+;; v115 = iadd.i32 v9, v114 ; v114 = 15
+;; v118 = iconst.i32 -16
+;; v119 = band v115, v118 ; v118 = -16
+;; v121 = iadd.i32 v12, v119
+;; @001f store notrap aligned region0 v121, v11
+;; v137 = iconst.i32 -1476394994
+;; v138 = load.i64 notrap aligned readonly can_move v0+8
+;; v139 = load.i64 notrap aligned readonly can_move v138+32
+;; @001f v36 = iadd v139, v19
+;; @001f store notrap aligned v137, v36 ; v137 = -1476394994
+;; v140 = load.i64 notrap aligned readonly can_move v0+40
+;; v141 = load.i32 notrap aligned readonly can_move v140
+;; @001f store notrap aligned v141, v36+4
+;; v142 = band.i64 v17, v16 ; v16 = -16
+;; @001f istore32 notrap aligned v142, v36+8
;; @001f jump block4(v12, v36)
;;
;; block3 cold:
@@ -75,41 +72,51 @@
;; @001f v27 = load.i32 notrap aligned readonly can_move v26
;; @001f v28 = iconst.i32 16
;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16
-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8
-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32
+;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8
+;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32
;; @001f v31 = uextend.i64 v29
;; @001f v32 = iadd v30, v31
;; @001f jump block4(v29, v32)
;;
;; block4(v41: i32, v42: i64):
-;; v95 = stack_addr.i64 ss0
-;; store notrap v41, v95
-;; v94 = iconst.i64 16
-;; @001f v43 = iadd v42, v94 ; v94 = 16
+;; v91 = iconst.i64 16
+;; @001f v43 = iadd v42, v91 ; v91 = 16
;; @001f store.i32 user2 v2, v43
-;; v77 = load.i32 notrap v95
-;; @001f trapz v77, user16
-;; v147 = load.i64 notrap aligned readonly can_move v0+8
-;; v148 = load.i64 notrap aligned readonly can_move v147+32
-;; @001f v46 = uextend.i64 v77
-;; @001f v48 = iadd v148, v46
-;; @001f v50 = iadd v48, v94 ; v94 = 16
+;; @001f trapz v41, user16
+;; v143 = load.i64 notrap aligned readonly can_move v0+8
+;; v144 = load.i64 notrap aligned readonly can_move v143+32
+;; @001f v46 = uextend.i64 v41
+;; @001f v48 = iadd v144, v46
+;; @001f v50 = iadd v48, v91 ; v91 = 16
;; @001f v51 = load.i32 user2 readonly v50
;; @001f v52 = uextend.i64 v51
;; @001f v57 = icmp.i64 ugt v5, v52
;; @001f trapnz v57, user17
-;; @001f v68 = load.i64 notrap aligned v147+40
+;; @001f v68 = load.i64 notrap aligned v143+40
;; v85 = iconst.i64 20
;; @001f v61 = iadd v48, v85 ; v85 = 20
-;; @001f v70 = uadd_overflow_trap v61, v103, user2
-;; @001f v69 = iadd v148, v68
+;; @001f v70 = uadd_overflow_trap v61, v99, user2
+;; @001f v69 = iadd v144, v68
;; @001f v71 = icmp ugt v70, v69
;; @001f trapnz v71, user2
+;; v123 = iconst.i64 0
+;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0
;; @001f v44 = iconst.i32 0
-;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0
-;; v74 = load.i32 notrap v95
-;; @0022 jump block1
+;; v97 = iconst.i64 4
+;; @001f v72 = iadd v61, v99
+;; @001f brif v73, block6, block5(v61)
;;
-;; block1:
-;; @0022 return v74
+;; block5(v74: i64):
+;; v145 = iconst.i32 0
+;; @001f store user2 little v145, v74 ; v145 = 0
+;; v146 = iconst.i64 4
+;; v147 = iadd v74, v146 ; v146 = 4
+;; @001f v76 = icmp eq v147, v72
+;; @001f brif v76, block6, block5(v147)
+;;
+;; block6:
+;; @0022 jump block1(v41)
+;;
+;; block1(v3: i32):
+;; @0022 return v3
;; }
diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.wat
index baa936d1dd..45901103e3 100644
--- a/tests/disas/gc/array-new-default-funcref.wat
+++ b/tests/disas/gc/array-new-default-funcref.wat
@@ -20,22 +20,22 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail
-;; sig1 = (i64 vmctx, i64, i32, i64) tail
+;; sig1 = (i64 vmctx, i64) -> i64 tail
;; fn0 = colocated u805306368:24 sig0
-;; fn1 = colocated u805306368:2 sig1
+;; fn1 = colocated u805306368:25 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
;; @001f v5 = uextend.i64 v2
-;; v102 = iconst.i64 2
-;; v103 = ishl v5, v102 ; v102 = 2
-;; v100 = iconst.i64 32
-;; @001f v7 = ushr v103, v100 ; v100 = 32
+;; v110 = iconst.i64 2
+;; v111 = ishl v5, v110 ; v110 = 2
+;; v108 = iconst.i64 32
+;; @001f v7 = ushr v111, v108 ; v108 = 32
;; @001f trapnz v7, user18
;; @001f v4 = iconst.i32 20
-;; v109 = iconst.i32 2
-;; v110 = ishl v2, v109 ; v109 = 2
-;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20
+;; v117 = iconst.i32 2
+;; v118 = ishl v2, v117 ; v117 = 2
+;; @001f v9 = uadd_overflow_trap v4, v118, user18 ; v4 = 20
;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32
;; @001f v12 = load.i32 notrap aligned v11
;; @001f v13 = load.i32 notrap aligned v11+4
@@ -51,22 +51,22 @@
;; @001f brif v22, block2, block3
;;
;; block2:
-;; v118 = iconst.i32 15
-;; v119 = iadd.i32 v9, v118 ; v118 = 15
-;; v122 = iconst.i32 -16
-;; v123 = band v119, v122 ; v122 = -16
-;; v125 = iadd.i32 v12, v123
-;; @001f store notrap aligned region0 v125, v11
-;; v140 = iconst.i32 -1476395002
-;; v141 = load.i64 notrap aligned readonly can_move v0+8
-;; v142 = load.i64 notrap aligned readonly can_move v141+32
-;; @001f v36 = iadd v142, v19
-;; @001f store notrap aligned v140, v36 ; v140 = -1476395002
-;; v143 = load.i64 notrap aligned readonly can_move v0+40
-;; v144 = load.i32 notrap aligned readonly can_move v143
-;; @001f store notrap aligned v144, v36+4
-;; v145 = band.i64 v17, v16 ; v16 = -16
-;; @001f istore32 notrap aligned v145, v36+8
+;; v126 = iconst.i32 15
+;; v127 = iadd.i32 v9, v126 ; v126 = 15
+;; v130 = iconst.i32 -16
+;; v131 = band v127, v130 ; v130 = -16
+;; v133 = iadd.i32 v12, v131
+;; @001f store notrap aligned region0 v133, v11
+;; v148 = iconst.i32 -1476395002
+;; v149 = load.i64 notrap aligned readonly can_move v0+8
+;; v150 = load.i64 notrap aligned readonly can_move v149+32
+;; @001f v36 = iadd v150, v19
+;; @001f store notrap aligned v148, v36 ; v148 = -1476395002
+;; v151 = load.i64 notrap aligned readonly can_move v0+40
+;; v152 = load.i32 notrap aligned readonly can_move v151
+;; @001f store notrap aligned v152, v36+4
+;; v153 = band.i64 v17, v16 ; v16 = -16
+;; @001f istore32 notrap aligned v153, v36+8
;; @001f jump block4(v12, v36)
;;
;; block3 cold:
@@ -75,41 +75,56 @@
;; @001f v27 = load.i32 notrap aligned readonly can_move v26
;; @001f v28 = iconst.i32 16
;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476395002, v28 = 16
-;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8
-;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32
+;; @001f v104 = load.i64 notrap aligned readonly can_move v0+8
+;; @001f v30 = load.i64 notrap aligned readonly can_move v104+32
;; @001f v31 = uextend.i64 v29
;; @001f v32 = iadd v30, v31
;; @001f jump block4(v29, v32)
;;
;; block4(v41: i32, v42: i64):
-;; v95 = stack_addr.i64 ss0
-;; store notrap v41, v95
-;; v94 = iconst.i64 16
-;; @001f v43 = iadd v42, v94 ; v94 = 16
+;; v103 = stack_addr.i64 ss0
+;; store notrap v41, v103
+;; v102 = iconst.i64 16
+;; @001f v43 = iadd v42, v102 ; v102 = 16
;; @001f store.i32 user2 v2, v43
-;; v77 = load.i32 notrap v95
-;; @001f trapz v77, user16
-;; v146 = load.i64 notrap aligned readonly can_move v0+8
-;; v147 = load.i64 notrap aligned readonly can_move v146+32
-;; @001f v46 = uextend.i64 v77
-;; @001f v48 = iadd v147, v46
-;; @001f v50 = iadd v48, v94 ; v94 = 16
+;; v83 = load.i32 notrap v103
+;; @001f trapz v83, user16
+;; v154 = load.i64 notrap aligned readonly can_move v0+8
+;; v155 = load.i64 notrap aligned readonly can_move v154+32
+;; @001f v46 = uextend.i64 v83
+;; @001f v48 = iadd v155, v46
+;; @001f v50 = iadd v48, v102 ; v102 = 16
;; @001f v51 = load.i32 user2 readonly v50
;; @001f v52 = uextend.i64 v51
;; @001f v57 = icmp.i64 ugt v5, v52
;; @001f trapnz v57, user17
-;; @001f v68 = load.i64 notrap aligned v146+40
-;; v85 = iconst.i64 20
-;; @001f v61 = iadd v48, v85 ; v85 = 20
-;; @001f v70 = uadd_overflow_trap v61, v103, user2
-;; @001f v69 = iadd v147, v68
+;; @001f v68 = load.i64 notrap aligned v154+40
+;; v93 = iconst.i64 20
+;; @001f v61 = iadd v48, v93 ; v93 = 20
+;; @001f v70 = uadd_overflow_trap v61, v111, user2
+;; @001f v69 = iadd v155, v68
;; @001f v71 = icmp ugt v70, v69
;; @001f trapnz v71, user2
-;; @001f v45 = iconst.i32 0
-;; @001f call fn1(v0, v61, v45, v103), stack_map=[i32 @ ss0+0] ; v45 = 0
-;; v74 = load.i32 notrap v95
+;; @001f v44 = iconst.i64 0
+;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0
+;; v109 = iconst.i64 4
+;; @001f v72 = iadd v61, v111
+;; @001f brif v73, block6, block5(v61)
+;;
+;; block5(v74: i64):
+;; v156 = iconst.i64 0
+;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0
+;; @001f v77 = ireduce.i32 v76
+;; @001f store user2 little v77, v74
+;; v157 = iconst.i64 4
+;; v158 = iadd v74, v157 ; v157 = 4
+;; @001f v79 = icmp eq v158, v72
+;; @001f brif v79, block6, block5(v158)
+;;
+;; block6:
+;; v80 = load.i32 notrap v103
;; @0022 jump block1
;;
;; block1:
-;; @0022 return v74
+;; @0022 return v80
;; }
diff --git a/tests/disas/passive-data.wat b/tests/disas/passive-data.wat
index 0fbe3b45fc..0d8605e9c4 100644
--- a/tests/disas/passive-data.wat
+++ b/tests/disas/passive-data.wat
@@ -28,29 +28,30 @@
;; @003d v6 = load.i64 notrap aligned v0+64
;; @003d v7 = uextend.i64 v2
;; @003d v8 = uextend.i64 v4
-;; v32 = iconst.i64 1
-;; @003d v9 = imul v8, v32 ; v32 = 1
+;; v33 = iconst.i64 1
+;; @003d v9 = imul v8, v33 ; v33 = 1
;; @003d v10 = iadd v7, v9
;; @003d v11 = icmp ugt v10, v6
;; @003d trapnz v11, heap_oob
;; @003d v12 = load.i64 notrap aligned readonly can_move v0+56
;; @003d v13 = uextend.i64 v2
-;; v30 = iconst.i64 1
-;; @003d v14 = imul v13, v30 ; v30 = 1
+;; v31 = iconst.i64 1
+;; @003d v14 = imul v13, v31 ; v31 = 1
;; @003d v15 = iadd v12, v14
-;; @003d v17 = uload32 notrap aligned v0+152
-;; @003d v18 = uextend.i64 v3
-;; @003d v19 = uextend.i64 v4
-;; v29 = iconst.i64 1
-;; @003d v20 = imul v19, v29 ; v29 = 1
-;; @003d v21 = iadd v18, v20
-;; @003d v22 = icmp ugt v21, v17
-;; @003d trapnz v22, heap_oob
-;; @003d v24 = load.i64 notrap aligned v0+144
-;; @003d v25 = uextend.i64 v3
-;; @003d v26 = iadd v24, v25
-;; @003d v27 = uextend.i64 v4
-;; @003d call fn0(v0, v15, v26, v27)
+;; @003d v17 = load.i32 notrap aligned v0+152
+;; @003d v18 = uextend.i64 v17
+;; @003d v19 = uextend.i64 v3
+;; @003d v20 = uextend.i64 v4
+;; v30 = iconst.i64 1
+;; @003d v21 = imul v20, v30 ; v30 = 1
+;; @003d v22 = iadd v19, v21
+;; @003d v23 = icmp ugt v22, v18
+;; @003d trapnz v23, heap_oob
+;; @003d v25 = load.i64 notrap aligned v0+144
+;; @003d v26 = uextend.i64 v3
+;; @003d v27 = iadd v25, v26
+;; @003d v28 = uextend.i64 v4
+;; @003d call fn0(v0, v15, v27, v28)
;; @0041 jump block1
;;
;; block1:
diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.wat
new file mode 100644
index 0000000000..95e2e5b187
--- /dev/null
+++ b/tests/disas/startup-data-active.wat
@@ -0,0 +1,62 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+;;! flags = '-Wgc -Wfunction-references'
+
+(module
+ (memory 1)
+
+ (data (i32.const 1) "hi")
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned gv0+64
+;; gv2 = load.i64 notrap aligned readonly can_move gv0+56
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:1 sig0
+;;
+;; block0(v0: i64, v1: i64):
+;; v3 = load.i64 notrap aligned v0+112
+;; v4 = iconst.i64 0
+;; v5 = icmp eq v3, v4 ; v4 = 0
+;; brif v5, block2, block1
+;;
+;; block1:
+;; v8 = load.i32 notrap aligned v0+120
+;; v11 = load.i64 notrap aligned v0+64
+;; v13 = uextend.i64 v8
+;; v16 = icmp ugt v13, v11
+;; trapnz v16, heap_oob
+;; v17 = load.i64 notrap aligned readonly can_move v0+56
+;; call fn0(v0, v17, v3, v13)
+;; jump block2
+;;
+;; block2:
+;; return
+;; }
diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat
new file mode 100644
index 0000000000..10e6af2848
--- /dev/null
+++ b/tests/disas/startup-elem-active.wat
@@ -0,0 +1,77 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+;;! flags = '-Wgc -Wfunction-references'
+
+(module
+ (table 10 anyref)
+
+ (elem (i32.const 1) (ref i31)
+ (item (ref.i31 (i32.const 10)))
+ (item (ref.i31 (i32.const 11)))
+ (item (ref.i31 (i32.const 12)))
+ )
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; region0 = 1 "table"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned gv0+48
+;; gv2 = load.i64 notrap aligned gv0+56
+;;
+;; block0(v0: i64, v1: i64):
+;; v4 = load.i64 notrap aligned v0+56
+;; v5 = ireduce.i32 v4
+;; v6 = uextend.i64 v5
+;; v86 = iconst.i64 4
+;; v92 = icmp ult v6, v86 ; v86 = 4
+;; trapnz v92, user6
+;; v12 = load.i64 notrap aligned v0+48
+;; v103 = iconst.i32 21
+;; v2 = iconst.i32 1
+;; v114 = icmp ule v5, v2 ; v2 = 1
+;; v79 = iconst.i64 0
+;; v15 = iadd v12, v86 ; v86 = 4
+;; v28 = select_spectre_guard v114, v79, v15 ; v79 = 0
+;; store user6 aligned region0 v103, v28 ; v103 = 21
+;; v117 = iconst.i32 23
+;; v123 = iconst.i32 2
+;; v129 = icmp ule v5, v123 ; v123 = 2
+;; v131 = iconst.i64 8
+;; v39 = iadd v12, v131 ; v131 = 8
+;; v41 = select_spectre_guard v129, v79, v39 ; v79 = 0
+;; store user6 aligned region0 v117, v41 ; v117 = 23
+;; v133 = iconst.i32 25
+;; v3 = iconst.i32 3
+;; v144 = icmp ule v5, v3 ; v3 = 3
+;; v146 = iconst.i64 12
+;; v52 = iadd v12, v146 ; v146 = 12
+;; v54 = select_spectre_guard v144, v79, v52 ; v79 = 0
+;; store user6 aligned region0 v133, v54 ; v133 = 25
+;; return
+;; }
diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.wat
new file mode 100644
index 0000000000..49da0905df
--- /dev/null
+++ b/tests/disas/startup-global.wat
@@ -0,0 +1,42 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+
+(module
+ (global i64 i64.const 0 i64.const 0 i64.add)
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; region0 = 1 "table"
+;; gv0 = vmctx
+;;
+;; block0(v0: i64, v1: i64):
+;; v2 = iconst.i64 0
+;; store notrap aligned region0 v2, v0+48 ; v2 = 0
+;; return
+;; }
diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.wat
new file mode 100644
index 0000000000..eaec68a403
--- /dev/null
+++ b/tests/disas/startup-passive-segment.wat
@@ -0,0 +1,50 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+;;! flags = '-Wgc'
+
+(module
+ (elem (ref i31) (item (ref.i31 (i32.const 0))) (item (ref.i31 (i32.const 1))))
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; gv0 = vmctx
+;; sig0 = (i64 vmctx, i32) -> i64 tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i64, v1: i64):
+;; v3 = iconst.i32 0
+;; v4 = call fn0(v0, v3) ; v3 = 0
+;; v18 = iconst.i32 1
+;; store user2 little v18, v4 ; v18 = 1
+;; v25 = iconst.i32 3
+;; v13 = iconst.i64 16
+;; v12 = iadd v4, v13 ; v13 = 16
+;; store user2 little v25, v12 ; v25 = 3
+;; return
+;; }
diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.wat
new file mode 100644
index 0000000000..aac06f2446
--- /dev/null
+++ b/tests/disas/startup-start.wat
@@ -0,0 +1,42 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+
+(module
+ (func $f)
+ (start $f)
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u0:0 sig0
+;;
+;; block0(v0: i64, v1: i64):
+;; call fn0(v0, v0)
+;; return
+;; }
diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat
new file mode 100644
index 0000000000..657a42ecbb
--- /dev/null
+++ b/tests/disas/startup-table-initial-value.wat
@@ -0,0 +1,65 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! filter = 'module_start'
+;;! flags = '-Wgc -Wfunction-references'
+
+(module
+ (table 10 (ref i31) (ref.i31 (i32.const 0)))
+)
+;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; sig0 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2415919104:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i64 notrap aligned v0+8
+;; v5 = get_frame_pointer.i64
+;; store notrap aligned v5, v4+72
+;; v6 = get_stack_pointer.i64
+;; store notrap aligned v6, v4+64
+;; v7 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned v7, v4+80
+;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
+;;
+;; block2:
+;; v8 = iconst.i8 1
+;; return v8 ; v8 = 1
+;;
+;; block3:
+;; v9 = iconst.i8 0
+;; return v9 ; v9 = 0
+;; }
+;;
+;; function u2415919104:0(i64 vmctx, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned gv0+48
+;; gv2 = load.i64 notrap aligned gv0+56
+;;
+;; block0(v0: i64, v1: i64):
+;; v7 = load.i64 notrap aligned v0+56
+;; v8 = ireduce.i32 v7
+;; v9 = uextend.i64 v8
+;; v41 = iconst.i64 10
+;; v53 = icmp ult v9, v41 ; v41 = 10
+;; trapnz v53, user6
+;; v15 = load.i64 notrap aligned v0+48
+;; v34 = iconst.i32 1
+;; v83 = iconst.i64 36
+;; v85 = iadd v15, v83 ; v83 = 36
+;; v29 = iconst.i64 4
+;; jump block1(v15)
+;;
+;; block1(v23: i64):
+;; v88 = iconst.i32 1
+;; store notrap aligned v88, v23 ; v88 = 1
+;; v89 = iadd.i64 v15, v83 ; v83 = 36
+;; v90 = icmp eq v23, v89
+;; v91 = iconst.i64 4
+;; v92 = iadd v23, v91 ; v91 = 4
+;; brif v90, block2, block1(v92)
+;;
+;; block2:
+;; return
+;; }
diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat
index ae2e8eaf43..e9ccc89ad7 100644
--- a/tests/disas/winch/x64/table/init_copy_drop.wat
+++ b/tests/disas/winch/x64/table/init_copy_drop.wat
@@ -303,7 +303,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x114d
+;; callq 0x1178
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -388,7 +388,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x114d
+;; callq 0x1178
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -473,7 +473,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x114d
+;; callq 0x1178
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -558,7 +558,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x114d
+;; callq 0x1178
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -643,7 +643,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x114d
+;; callq 0x1178
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -754,7 +754,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x114d
+;; callq 0x1178
;; addq $0xc, %rsp
;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
diff --git a/tests/misc_testsuite/gc/array-fill-null-funcref.wast b/tests/misc_testsuite/gc/array-fill-null-funcref.wast
new file mode 100644
index 0000000000..2e74c3384d
--- /dev/null
+++ b/tests/misc_testsuite/gc/array-fill-null-funcref.wast
@@ -0,0 +1,10 @@
+;;! gc = true
+
+(module
+ (type $a (array funcref))
+ (func (export "hi") (result funcref)
+ (array.get $a (array.new_default $a (i32.const 10)) (i32.const 3))
+ )
+)
+
+(assert_return (invoke "hi") (ref.null func))
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index be5eaefc49..ffa64484d0 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -1722,7 +1722,7 @@ where
self.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?;
self.context.free_reg(dst);
- let passive_data_index = match self.env.translation.passive_data_map[segment] {
+ let runtime_data_index = match self.env.translation.runtime_data_map[segment] {
Some(i) => i,
// Active data segments always have length zero, so this is only
@@ -1747,7 +1747,7 @@ where
let data_segment_length_offset = self
.env
.vmoffsets
- .vmctx_passive_data_length(passive_data_index);
+ .vmctx_runtime_data_length(runtime_data_index);
let tmp1 = self.context.any_gpr(self.masm)?;
let tmp2 = self.context.any_gpr(self.masm)?;
self.masm.load(
@@ -1774,7 +1774,7 @@ where
let data_segment_base_offset = self
.env
.vmoffsets
- .vmctx_passive_data_base(passive_data_index);
+ .vmctx_runtime_data_base(runtime_data_index);
self.masm.load(
self.masm.address_at_vmctx(data_segment_base_offset)?,
writable!(tmp1),
@@ -1804,7 +1804,7 @@ where
}
pub fn emit_data_drop(&mut self, data_index: DataIndex) -> Result<()> {
- let passive_data_index = match self.env.translation.passive_data_map[data_index] {
+ let runtime_data_index = match self.env.translation.runtime_data_map[data_index] {
Some(idx) => idx,
// Active data segments do nothing when dropped, so this is a noop.
None => return Ok(()),
@@ -1812,7 +1812,7 @@ where
let data_segment_offset = self
.env
.vmoffsets
- .vmctx_passive_data_length(passive_data_index);
+ .vmctx_runtime_data_length(runtime_data_index);
let len_addr = self.masm.address_at_vmctx(data_segment_offset)?;
self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32)
}