Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

75.1 KB
Raw Blame
//! Alias regions for the memory that compiled Wasm code accesses.
//!
//! Every load and store Wasmtime emits carries a
//! `cranelift_codegen::ir::AliasRegion` naming the memory it touches, so that
//! Cranelift's alias analysis can better eliminate redundant loads, dead
//! stores, etc. This module, and primarily its `AliasRegions` type, parcels
//! those regions out.
//!
//! `AliasRegions` accessors come from two places:
//!
//! 1. Fields of the `VM*` types and of the vmctx types get one region per
//!    field, and an accessor per field is generated from our
//!    `for_each_vm[ctx]_type!` macros. An `enum` gets only two: one for its
//!    tag, one shared by all variants' fields, which overlap in memory.
//!
//! 2. There are a handful of hand-written methods for memory regions that are
//!    not part of the vmctx or a `VM*` type at all (linear memories, the GC
//!    heap, stack slots, etc...).
//!
//! We strive to avoid adding new hand-written methods as much as possible.

use crate::translate::Load;
use cranelift_codegen::{
    cursor::FuncCursor,
    ir::{self, InstBuilder as _},
};
use std::collections::HashMap;
use std::hash::{Hash as _, Hasher};
use wasmtime_environ::{
    BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,
    MemoryIndex, ModuleInternedTypeIndex, ModuleTypesBuilder, NUM_COMPONENT_CONTEXT_SLOTS,
    PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets, VmctxArrayIndex as _,
    WasmCompositeInnerType,
    component::{
        ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,
        RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,
        RuntimeReallocIndex, VMComponentOffsets,
    },
};

#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
enum VmType {
    VMContext,
    VMStoreContext,
    VMMemoryDefinition,
    VMTableDefinition,
    #[allow(
        dead_code,
        reason = "generated uniformly for all VM types via `for_each_vm_type!`"
    )]
    VMTagDefinition,
    VMComponentContext,
    VMDrcHeapData,
    VMCopyingHeapData,
    VMNullHeapData,
    VMDeferredThread,
    VMStackLimits,
    #[allow(
        dead_code,
        reason = "generated uniformly for all VM types via `for_each_vm_type!`"
    )]
    VMLazyThread,
    VMContRef,
    VMCommonStackInformation,
    VMHostArray,
    ContinuationStackMemory,
    VMFunctionImport,
    VMMemoryImport,
    VMTableImport,
    VMTagImport,
    VMGlobalImport,
    VMFuncRef,
    TypeIdsArray,
    EpochCounter,
    BuiltinFunctionsArray,
    ComponentBuiltinFunctionsArray,
    HostValRaw,
    VMPayloads,
    VMRawContObj,
    VMGcHeader,
    VMDrcHeader,
    VMCopyingHeader,
    #[allow(
        dead_code,
        reason = "generated uniformly for all VM types via `for_each_vm_type!`"
    )]
    VMAsanStackSupport,
    VMResourceIndex,
    VMHandleTable,
    VMHandleTableSlot,
    VMTypedResource,
    #[allow(
        dead_code,
        reason = "generated uniformly for all VM types via `for_each_vm_type!`"
    )]
    VMScope,
    #[allow(
        dead_code,
        reason = "generated uniformly for all VM types via `for_each_vm_type!`"
    )]
    VMTransmitLocalState,
}

/// A key that uniquely identifies an alias region across an entire compilation.
///
/// This is used to assign stable `user_id`s to `AliasRegionData` entries so
/// that alias regions can be deduplicated during inlining.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum AliasRegionKey {
    /// An access of a field within a VM data structure of type `ty`.
    Vm {
        /// The type of VM data structure being accessed.
        ty: VmType,
        /// The offset of the accessed field *within* the `ty` structure (or
        /// the base offset of the array, for array fields, or the payload
        /// offset, for an enum's variants' fields).
        offset: u32,
    },

    /// An access of a memory that crosses a module boundary and whose
    /// definition we do not statically know (shared across all such memories).
    PublicMemory,

    /// A defined memory access.
    DefinedMemory {
        /// The static module index.
        module: StaticModuleIndex,
        /// The defined memory index within the module.
        index: DefinedMemoryIndex,
    },

    /// An access of a table that crosses a module boundary and whose definition
    /// we do not statically know (shared across all such tables).
    PublicTable,

    /// A defined table access.
    DefinedTable {
        /// The static module index.
        module: StaticModuleIndex,
        /// The defined table index within the module.
        index: DefinedTableIndex,
    },

    /// An access of a global that crosses a module boundary and whose definition
    /// we do not statically know (shared across all such globals).
    PublicGlobal,

    /// A defined global access.
    DefinedGlobal {
        /// The static module index.
        module: StaticModuleIndex,
        /// The defined global index within the module.
        index: DefinedGlobalIndex,
    },

    /// An access of a GC array's `length` word.
    GcArrayLength,

    /// An access of a GC struct's field.
    GcStructField {
        /// The first supertype that introduced this field.
        ty: ModuleInternedTypeIndex,
        /// The index of the field being accessed.
        field: u32,
    },

    /// An access of a GC array's elements, which all share one region.
    GcArrayElements {
        /// The first supertype that introduced these elements.
        ty: ModuleInternedTypeIndex,
    },

    /// An access of an exception object's payload field.
    GcExnPayload {
        /// The exception type.
        exn_ty: ModuleInternedTypeIndex,
        /// The payload field being accessed.
        field: u32,
    },

    /// A stack slot access.
    Stack {
        /// The stack slot being accessed.
        slot: ir::StackSlot,
    },

    /// All unsafe intrinsics share a single alias region.
    ///
    /// By contract, unsafe intrinsics cannot access our internal `VM*` types or
    /// linear memories or anything else that has its own dedicated alias
    /// region. However, we cannot guarantee anything about the (lack of)
    /// aliasing of the embedder's data structures that the unsafe intrinsics do
    /// access, so all their accesses are lumped together into the same region.
    UnsafeIntrinsicMemory,

    /// An access of a `ValRaw` inside a passive element segment.
    ElementSegment,

    /// An access of the bytes inside a data segment.
    DataSegment,

    /// An access of an exception object's tag-instance word.
    GcExnTagInstance,

    /// An access of an exception object's tag-defined word.
    GcExnTagDefined,

    /// An access of the shadow bytes of a linear memory, for watchpoints.
    ShadowMemory,
}

impl AliasRegionKey {
    /// Encode this key into a raw `u32` suitable for use as an
    /// `AliasRegionData::user_id`.
    ///
    /// We lossily encode the key into a `user_id` via a single-byte hash, so
    /// there are at most 256 alias regions. This avoids compile-time blowups
    /// associated with many alias regions.
    ///
    /// Note that mapping multiple alias regions onto the same `user_id` is
    /// always okay: by collapsing two logical regions into one actual region,
    /// we are letting Cranelift believe they *might* alias even when they
    /// actually cannot. At worst this prevents some optimization. The opposite,
    /// however -- if we were to place accesses that *could* alias in two
    /// different regions -- is not sound. That would let Cranelift optimize
    /// based on a false premise, leading to misoptimizations and symptoms like
    /// memory writes "disappearing".
    ///
    /// Simultaneously, this mapping is independent of the particular set of
    /// `AliasRegionKey`s used in a particular Wasm-to-CLIF translation, which
    /// is necessary for inlining. If we assigned `user_id = 1` to the first
    /// `AliasRegionKey`, `user_id = 2` to the second, etc... then inlining one
    /// function into another would mean that the `user_id`s for the same
    /// `AliasRegionKey` could be inconsistent with each other, which also leads
    /// to the miscompilation scenario described above.
    pub(crate) fn into_raw(self) -> u32 {
        /// A version of `rustc_hash::FxHasher` that is guaranteed to provide
        /// deterministic hashing across Wasmtime builds and processes.
        struct AliasRegionHasher {
            hash: u64,
        }

        impl AliasRegionHasher {
            fn new(seed: u64) -> Self {
                AliasRegionHasher { hash: seed }
            }

            fn add(&mut self, i: u64) {
                const K: u64 = 0x517c_c1b7_2722_0a95;
                self.hash = (self.hash.rotate_left(5) ^ i).wrapping_mul(K);
            }
        }

        impl Hasher for AliasRegionHasher {
            fn finish(&self) -> u64 {
                self.hash
            }

            fn write(&mut self, bytes: &[u8]) {
                for b in bytes {
                    self.add((*b).into());
                }
            }

            fn write_u8(&mut self, i: u8) {
                self.add(i.into());
            }

            fn write_u16(&mut self, i: u16) {
                self.add(i.into());
            }

            fn write_u32(&mut self, i: u32) {
                self.add(i.into());
            }

            fn write_u64(&mut self, i: u64) {
                self.add(i);
            }

            fn write_u128(&mut self, i: u128) {
                self.add(u64::try_from(i & u128::from(u64::MAX)).unwrap());
                self.add(u64::try_from(i >> 64).unwrap());
            }

            fn write_usize(&mut self, i: usize) {
                self.add(u64::try_from(i).unwrap());
            }
        }

        /// The `AliasRegionHasher` seed, chosen to shift which keys collide
        /// with which.
        ///
        /// Collisions are inevitable because we are lossily mapping our
        /// ~"infinite" keys onto only 256 actual alias regions. However, not
        /// all collisions are equal. The cost of a pair of globals colliding is
        /// marginal. A collision between `VMContext::store_context` and the
        /// first defined memory's base pointer, two of the most-frequently used
        /// regions in essentially every function, would be expensive.
        ///
        /// This value was picked by brute-force search as one for which none
        /// of the following collide with each other: the fixed, non-indexed
        /// keys appearing anywhere in `tests/disas`, and the first five defined
        /// memories, tables, and globals of the first module. If a new `VM*`
        /// field or `AliasRegionKey` variant introduces such a collision later,
        /// the `tests/disas` expectations will show it as two regions merging
        /// into one, and a new seed can be searched for in the same way.
        const SEED: u64 = 15912981;

        let mut hasher = AliasRegionHasher::new(SEED);
        self.hash(&mut hasher);
        let hash = hasher.finish();

        // `xor`-fold the hash down to a single byte.
        hash.to_le_bytes().into_iter().fold(0, |a, b| a ^ b).into()
    }
}

/// Which logical part of a GC object an access targets.
#[derive(Clone, Copy, Debug)]
pub enum GcAccess {
    /// An access of a struct's field, by the type at the access site.
    StructField {
        ty: ModuleInternedTypeIndex,
        field: u32,
    },

    /// An access of one of an array's elements, by the type at the access site.
    ArrayElements { ty: ModuleInternedTypeIndex },

    /// An access of an exception object's payload field.
    ExnPayload {
        exn_ty: ModuleInternedTypeIndex,
        field: u32,
    },

    /// An access of `VMGcHeader::kind` or the reserved bits packed into it.
    HeaderKind,

    /// An access of `VMGcHeader::ty`.
    HeaderTypeIndex,

    /// An access of `VMDrcHeader::ref_count`.
    DrcRefCount,

    /// An access of `VMDrcHeader::next_over_approximated_stack_root`.
    DrcNextOverApproximatedStackRoot,

    /// An access of `VMCopyingHeader::object_size`.
    CopyingObjectSize,

    /// An access of an array's `length` word.
    ArrayLength,

    /// An access of an exception object's tag-instance word.
    ExnTagInstance,

    /// An access of an exception object's tag-defined word.
    ExnTagDefined,
}

/// Alias region bookkeeping and load/store helper type.
///
/// This is scoped to a single function: it hands out `ir::AliasRegion`s, which
/// are indices into one function's `ir::AliasRegionSet`, so a given
/// `AliasRegions` must not be reused across functions.
pub struct AliasRegions<Offsets> {
    pointer_type: ir::Type,
    offsets: Offsets,

    /// Cache for `Self::gc_introducer`, keyed on the access site's type and the
    /// field index, if any.
    gc_introducer_cache: HashMap<(ModuleInternedTypeIndex, u32), ModuleInternedTypeIndex>,
}

impl<Offsets> AliasRegions<Offsets> {
    /// Get the offsets this `AliasRegions` computes its field offsets from.
    pub fn offsets(&self) -> &Offsets {
        &self.offsets
    }

    /// Make the alias region for a stack map.
    pub fn stack_map_region(
        regions: &mut ir::AliasRegionSet,
        _ty: ir::Type,
        slot: ir::StackSlot,
        _offset: u32,
    ) -> Option<ir::AliasRegion> {
        Some(Self::insert_region(regions, AliasRegionKey::Stack { slot }))
    }

    /// Get the alias region for a stack slot.
    pub fn stack_slot_region(
        &mut self,
        func: &mut ir::Function,
        slot: ir::StackSlot,
    ) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::Stack { slot })
    }

    /// Get the alias region for the given key.
    fn region(&mut self, func: &mut ir::Function, key: AliasRegionKey) -> ir::AliasRegion {
        Self::insert_region(&mut func.dfg.alias_regions, key)
    }

    /// Get or create the alias region for the given key.
    fn insert_region(regions: &mut ir::AliasRegionSet, key: AliasRegionKey) -> ir::AliasRegion {
        let user_id = key.into_raw();
        let region = regions.insert(ir::AliasRegionData {
            user_id,
            description: "".into(),
        });
        log::trace!("alias region: {key:?} => {region} (user_id = {user_id})");
        region
    }
}

/// A single field within one of Wasmtime's `VM*` types, along with everything
/// needed to emit a correctly alias-analyzed load or store of that field.
///
/// A `Field` is obtained from the `Field`-returning accessors generated by
/// [`define_vm_type_alias_region_helpers!`], for example
/// `alias_regions.vm_memory_definition().base()`.
///
/// This is the evolved form of the old `Load` type: in addition to producing a
/// deferred [`Load`] descriptor (via [`Field::to_deferred_load`]) for use in a
/// `VmctxLoadChain`, a `Field` can directly emit loads and stores relative to a
/// pointer to the containing `VM*` structure.
pub struct Field<'a, Offsets> {
    /// The alias-region cache used to get-or-create this field's alias region.
    regions: &'a mut AliasRegions<Offsets>,
    /// The key identifying this field's alias region.
    ///
    /// This is fixed when the `Field` is created, derived from the field's
    /// offset *within* its containing `VM*` type, and is deliberately
    /// independent of [`Field::relative_to`]: rebasing the load/store
    /// (e.g. to make the access relative to the `vmctx` rather than to a pointer
    /// to the containing structure) must not change which alias region the
    /// access belongs to.
    key: AliasRegionKey,
    /// The offset added to the base value when emitting a load or store of this
    /// field.
    ///
    /// Initially the field's offset *within* its containing `VM*` type; callers
    /// may rebase it via [`Field::relative_to`].
    offset: u32,
    /// The base memory flags for accesses of this field, before this field's
    /// alias region is mixed in.
    flags: ir::MemFlagsData,
    /// The Cranelift type of this field.
    ty: ir::Type,
}

impl<'a, Offsets> Field<'a, Offsets> {
    /// Create a new `Field` for the given `offset` within `vm_type`, loaded or
    /// stored with the given base `flags` and Cranelift type `ty`.
    fn new(
        regions: &'a mut AliasRegions<Offsets>,
        vm_type: VmType,
        offset: u32,
        flags: ir::MemFlagsData,
        ty: ir::Type,
    ) -> Self {
        Self::new_in_region(regions, vm_type, offset, offset, flags, ty)
    }

    /// As `new`, but keyed on `region_offset` rather than the field's own
    /// `offset`, so that fields overlapping in memory (an enum's variants'
    /// fields) share one region instead of claiming they cannot alias.
    fn new_in_region(
        regions: &'a mut AliasRegions<Offsets>,
        vm_type: VmType,
        region_offset: u32,
        offset: u32,
        flags: ir::MemFlagsData,
        ty: ir::Type,
    ) -> Self {
        Field {
            regions,
            key: AliasRegionKey::Vm {
                ty: vm_type,
                offset: region_offset,
            },
            offset,
            flags,
            ty,
        }
    }

    /// Mark accesses of this field as `readonly`.
    ///
    /// Whether a field is `readonly` often depends on dynamic properties of the
    /// module being compiled (e.g. whether a memory can be relocated) rather
    /// than being a static property of the field; this method allows callers
    /// to mark the load as `readonly` in these cases.
    pub fn readonly(mut self) -> Self {
        self.flags.set_readonly(true);
        self
    }

    /// Mark accesses of this field as `readonly` if and only if `readonly` is
    /// `true`.
    ///
    /// See the note on [`Field::readonly`].
    pub fn readonly_if(self, readonly: bool) -> Self {
        if readonly { self.readonly() } else { self }
    }

    /// Mark accesses of this field as `can_move`.
    ///
    /// See the note on [`Field::readonly`].
    pub fn can_move(mut self) -> Self {
        self.flags = self.flags.with_can_move(true);
        self
    }

    /// Mark accesses of this field as `can_move` if and only if `can_move` is
    /// `true`.
    ///
    /// See the note on [`Field::readonly`].
    pub fn can_move_if(self, can_move: bool) -> Self {
        if can_move { self.can_move() } else { self }
    }

    /// Set the trap code for accesses of this field.
    ///
    /// A `Field`'s accesses do not trap by default.
    ///
    /// Note that when signals-based traps are disabled, callers must use the
    /// explicit call-to-host trapping codegen instead.
    pub fn trap_code(mut self, code: Option<ir::TrapCode>) -> Self {
        self.flags = self.flags.with_trap_code(code);
        self
    }

    /// Cast this field to the given type.
    ///
    /// This can be used, for example, to load a field as a narrower or
    /// differently-typed value than its declared Cranelift type.
    pub fn cast(mut self, ty: ir::Type) -> Self {
        self.ty = ty;
        self
    }

    /// Rebase this field's load or store to be relative to a new base.
    ///
    /// The `struct_offset` parameter is the offset of this field's containing
    /// `VM*` structure within the new base.
    ///
    /// A `Field` starts out relative to a pointer to its containing `VM*`
    /// structure. When that structure is inlined directly into a larger one (as
    /// an owned memory's `VMMemoryDefinition` is inlined into the `vmctx`), use
    /// this method to fold the structure's own offset within the larger
    /// structure into this `Field`, so that the resulting access is relative to
    /// the larger structure directly.
    pub fn relative_to(mut self, struct_offset: u32) -> Self {
        self.offset += struct_offset;
        self
    }

    /// Get-or-create this field's alias region.
    pub fn region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.regions.region(func, self.key)
    }

    /// Get-or-create this field's alias region and mix it into this field's
    /// base flags.
    fn flags_with_region(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {
        let region = self.region(func);
        self.flags.with_alias_region(Some(region))
    }

    /// Get a deferred [`Load`] descriptor for this field, for use in a
    /// `VmctxLoadChain`.
    ///
    /// The load is emitted at the field's current offset relative to its base
    /// value. By default that is the field's offset within its containing `VM*`
    /// structure, so the load is relative to a pointer to that structure; to
    /// make it relative to something else (e.g. the `vmctx`, when the structure
    /// is inlined into the `vmctx`), call [`Field::relative_to`] first.
    pub fn to_deferred_load(&mut self, func: &mut ir::Function) -> Load {
        let flags = self.flags_with_region(func);
        Load {
            offset: self.offset,
            flags,
            ty: self.ty,
        }
    }

    /// Emit a load of this field relative to `ptr`, a pointer to the containing
    /// `VM*` structure.
    pub fn load(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
        let load = self.to_deferred_load(cursor.func);
        load.emit(cursor, ptr)
    }

    /// Emit an atomic load of this field relative to `ptr`, a pointer to the
    /// containing `VM*` structure.
    pub fn load_atomic(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
        let flags = self.flags_with_region(cursor.func);
        let ty = self.ty;
        let pointer_type = self.regions.pointer_type;
        let offset = cursor.ins().iconst(pointer_type, i64::from(self.offset));
        let addr = cursor.ins().iadd(ptr, offset);
        cursor.ins().atomic_load(ty, flags, addr)
    }

    /// Emit a store of `value` to this field relative to `ptr`, a pointer to
    /// the containing `VM*` structure.
    pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {
        let flags = self.flags_with_region(cursor.func);
        cursor
            .ins()
            .store(flags, value, ptr, i32::try_from(self.offset).unwrap());
    }

    /// Like [`Self::store`], except removes the `readonly` flag for this region
    /// if one would otherwise be specified.
    ///
    /// This is useful for `#[readonly]` fields that are initialized precisely
    /// once before use, such as `VMFuncRef` fields.
    pub fn store_without_readonly(
        &mut self,
        cursor: &mut FuncCursor<'_>,
        ptr: ir::Value,
        value: ir::Value,
    ) {
        let flags = self.flags_with_region(cursor.func).with_readonly(false);
        cursor
            .ins()
            .store(flags, value, ptr, i32::try_from(self.offset).unwrap());
    }
}

/// Define, for each `VM*` type, an `AliasRegions` accessor that returns a
/// wrapper exposing a [`Field`]-returning method per field of that type.
///
/// For example, given
///
/// ```ignore
/// struct VMMemoryDefinition {
///     base: VmPtr<u8>,
///     current_length: AtomicUsize,
/// }
/// ```
///
/// this macro generates:
///
/// ```ignore
/// impl<Offsets> AliasRegions<Offsets> {
///     fn vm_memory_definition(&mut self) -> VMMemoryDefinition<'_, Offsets> { ... }
/// }
///
/// struct VMMemoryDefinition<'a, Offsets> { ... }
///
/// impl<'a, Offsets: GetPtrSize> VMMemoryDefinition<'a, Offsets> {
///     fn base(self) -> Field<'a, Offsets> { ... }
///     fn current_length(self) -> Field<'a, Offsets> { ... }
/// }
/// ```
///
/// A field marked `#[aggregate]` gets no accessor, because it has no single
/// Cranelift type. A field marked `#[indexed]` is a fixed-size array of a
/// scalar type; its accessor takes the element index, and each element gets its
/// own alias region.
#[allow(
    unused_macro_rules,
    reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \
              uniformly but not exercised until a VM type uses those markers"
)]
macro_rules! define_vm_type_alias_region_helpers {
    // `UnsafeCell<T>` is `repr(transparent)` and is accessed exactly as its `T`
    // would be; delegate to the inner type.
    (@field_ty $pt:expr, UnsafeCell < $inner:tt >) => {
        define_vm_type_alias_region_helpers!(@field_ty $pt, $inner)
    };

    // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the
    // target pointer type as an `ir::Type`).
    //
    // Note that there is deliberately no arm for a composite field (a nested
    // struct, an array, or a range): such a field has no single Cranelift type,
    // and is marked `#[aggregate]` in `for_each_vm_type!` so that no accessor is
    // generated for it at all.
    (@field_ty $pt:expr, VmPtr < $g:ty >) => { $pt };
    (@field_ty $pt:expr, VmPtr < VmPtr < $g:ty >>) => { $pt };
    (@field_ty $pt:expr, Option < VmPtr < $g:ty >>) => { $pt };
    (@field_ty $pt:expr, AtomicUsize) => { $pt };
    (@field_ty $pt:expr, usize) => { $pt };
    (@field_ty $pt:expr, * mut $g:ty) => { $pt };
    (@field_ty $pt:expr, i64) => { ir::types::I64 };
    (@field_ty $pt:expr, u64) => { ir::types::I64 };
    (@field_ty $pt:expr, u32) => { ir::types::I32 };
    (@field_ty $pt:expr, u16) => { ir::types::I16 };
    (@field_ty $pt:expr, u8) => { ir::types::I8 };
    (@field_ty $pt:expr, NonZeroU32) => { ir::types::I32 };
    (@field_ty $pt:expr, Option < VMGcRef >) => { ir::types::I32 };
    // `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in
    // `for_each_vm_type!`.
    (@field_ty $pt:expr, VMLazyThread) => { $pt };
    (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
    (@field_ty $pt:expr, DefinedTableIndex) => { ir::types::I32 };
    (@field_ty $pt:expr, DefinedMemoryIndex) => { ir::types::I32 };
    (@field_ty $pt:expr, DefinedTagIndex) => { ir::types::I32 };
    (@field_ty $pt:expr, VMGlobalKind) => { ir::types::I64 };
    (@field_ty $pt:expr, VMStackState) => { ir::types::I32 };

    // Classify an `#[indexed]` field's array type to
    // `(element ir::Type, element size in bytes, element count)`.
    (@array_info $pt:expr, UnsafeCell < $inner:tt >) => {
        define_vm_type_alias_region_helpers!(@array_info $pt, $inner)
    };
    (@array_info $pt:expr, [u32; $n:expr]) => { (ir::types::I32, 4u32, $n) };

    // Apply a field attribute to its access flags. The `#[readonly]` and
    // `#[can_move]` markers map to the corresponding `MemFlagsData` builder
    // methods; doc comments have no effect on flags; any other attribute is a
    // compile error.
    (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly(true) };
    (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move(true) };
    (@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };

    // Emit the accessor method for a single ordinary (scalar) field.
    (@method $Name:ident $snake:ident plain $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
        #[doc = concat!(
            "Get the [`Field`] for the `", stringify!($fname),
            "` field of `", stringify!($Name), "`."
        )]
        pub fn $fname(self) -> Field<'a, Offsets> {
            let offset = u32::from(
                self.regions.offsets.get_ptr_size().$snake().$fname()
            );
            let flags = ir::MemFlagsData::trusted();
            $(
                let flags = define_vm_type_alias_region_helpers!(
                    @apply_attr flags, $fattr
                );
            )*
            let ty = define_vm_type_alias_region_helpers!(
                @field_ty self.regions.pointer_type, $($fty)*
            );
            Field::new(self.regions, VmType::$Name, offset, flags, ty)
        }
    };

    // Emit the accessor method for an `#[indexed]` field: a fixed-size array of
    // a scalar type, indexed by a compile-time constant.
    (@method $Name:ident $snake:ident indexed $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
        #[doc = concat!(
            "Get the [`Field`] for the `index`th element of the `",
            stringify!($fname), "` field of `", stringify!($Name), "`."
        )]
        pub fn $fname(self, index: u8) -> Field<'a, Offsets> {
            let (ty, elem_size, len) = define_vm_type_alias_region_helpers!(
                @array_info self.regions.pointer_type, $($fty)*
            );
            assert!(
                usize::from(index) < len,
                concat!(
                    "index out of bounds for `", stringify!($Name), "::",
                    stringify!($fname), "`"
                ),
            );
            let base = u32::from(
                self.regions.offsets.get_ptr_size().$snake().$fname()
            );
            let offset = base + u32::from(index) * elem_size;
            let flags = ir::MemFlagsData::trusted();
            $(
                let flags = define_vm_type_alias_region_helpers!(
                    @apply_attr flags, $fattr
                );
            )*
            Field::new(self.regions, VmType::$Name, offset, flags, ty)
        }
    };

    // Emit the accessor `struct` and per-field methods for one `VM*` type.
    //
    // Fields arrive pre-split into `{ $kind $fname [ $attrs... ] [ $fty... ] }`
    // groups (see `@munch` below); the terminal arm consumes those groups once
    // the struct body has been fully peeled apart.
    (@emit $Name:ident $snake:ident {
        $( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
    }) => {
        #[doc = concat!(
            "An [`AliasRegions`] accessor for the fields of a `",
            stringify!($Name), "`."
        )]
        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        pub struct $Name<'a, Offsets> {
            regions: &'a mut AliasRegions<Offsets>,
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<Offsets> AliasRegions<Offsets> {
            #[doc = concat!(
                "Get an accessor for the fields of a `", stringify!($Name), "`."
            )]
            pub fn $snake(&mut self) -> $Name<'_, Offsets> {
                $Name { regions: self }
            }
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<'a, Offsets> $Name<'a, Offsets>
        where
            Offsets: GetPtrSize,
        {
            $(
                define_vm_type_alias_region_helpers!(
                    @method $Name $snake $fkind $fname [ $($fattr)* ] [ $($fty)* ]
                );
            )*
        }
    };

    // Peel fields off the raw struct body one at a time, accumulating completed
    // `{ $kind $fname [ $attrs... ] [ $fty... ] }` groups plus the pending
    // attributes for the field currently being parsed. `$kind` is `plain` or
    // `indexed`, and selects which `@method` arm emits the field's accessor.
    //
    // Splitting the body by hand (rather than matching `$fty:tt $(< $fgen:ty
    // >)?` within a repetition) is what lets each field's type reach the
    // `@field_ty` classifier as raw tokens, so it can require `Option`s to
    // specifically be `Option<VmPtr<_>>`.
    (@munch $Name:ident $snake:ident { $($groups:tt)* }) => {
        define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });
    };
    // A field marked `#[aggregate]` is a composite (a nested struct or array)
    // so there is no single Cranelift type for such fields, so no accessors are
    // generated for them; interior accesses are computed from the field's
    // offset instead, which the generated `offsets::*` methods still provide.
    (@munch $Name:ident $snake:ident { $($groups:tt)* }
        $(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*
    ) => {
        define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);
    };
    // A field marked `#[indexed]` is a fixed-size array of a scalar type whose
    // elements are accessed individually by a compile-time constant index. It
    // has no single Cranelift type either, but its elements do, so it gets an
    // index-taking accessor.
    (@munch $Name:ident $snake:ident { $($groups:tt)* }
        $(#[doc = $fdoc:literal])* #[indexed] $fvis:vis $fname:ident : $($rest:tt)*
    ) => {
        define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } indexed $fname [] [] $($rest)*);
    };
    // Consume one field's attributes, visibility, and name, then collect its
    // type tokens.
    (@munch $Name:ident $snake:ident { $($groups:tt)* }
        $(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*
    ) => {
        define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } plain $fname [ $($fattr)* ] [] $($rest)*);
    };
    // Accumulate one field's type tokens up to its terminating comma, then
    // append the completed group.
    (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
        define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fkind $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);
    };
    (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
        define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fkind $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);
    };

    // Emit the accessor `struct` and per-field methods for one enum variant,
    // so that `alias_regions.vm_foo().variant_a().field()` mirrors the
    // generated offset accessors.
    (@emit_variant $Name:ident $snake:ident $VariantAccess:ident $vsnake:ident {
        $( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
    }) => {
        #[doc = concat!(
            "An [`AliasRegions`] accessor for the fields of the `",
            stringify!($Name), "::", stringify!($VariantAccess), "` variant."
        )]
        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        pub struct $VariantAccess<'a, Offsets> {
            regions: &'a mut AliasRegions<Offsets>,
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<'a, Offsets> $Name<'a, Offsets> {
            #[doc = concat!(
                "Get an accessor for the fields of the `", stringify!($Name),
                "::", stringify!($VariantAccess), "` variant."
            )]
            pub fn $vsnake(self) -> $VariantAccess<'a, Offsets> {
                $VariantAccess { regions: self.regions }
            }
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<'a, Offsets> $VariantAccess<'a, Offsets>
        where
            Offsets: GetPtrSize,
        {
            $(
                define_vm_type_alias_region_helpers!(
                    @variant_method $Name $snake $vsnake $fkind $fname
                    [ $($fattr)* ] [ $($fty)* ]
                );
            )*
        }
    };

    // As `@method` above, but the offset comes from the variant's own offsets
    // and the region is the enum's shared payload one.
    (@variant_method $Name:ident $snake:ident $vsnake:ident plain $fname:ident
        [ $($fattr:tt)* ] [ $($fty:tt)* ]
    ) => {
        #[doc = concat!(
            "Get the [`Field`] for the `", stringify!($fname),
            "` field of this variant of `", stringify!($Name), "`.\n\n\
             All variants' fields share one alias region."
        )]
        pub fn $fname(self) -> Field<'a, Offsets> {
            let ptr_size = self.regions.offsets.get_ptr_size();
            let payload = u32::from(ptr_size.$snake().payload());
            let offset = u32::from(ptr_size.$snake().$vsnake().$fname());
            let flags = ir::MemFlagsData::trusted();
            $(
                let flags = define_vm_type_alias_region_helpers!(
                    @apply_attr flags, $fattr
                );
            )*
            let ty = define_vm_type_alias_region_helpers!(
                @field_ty self.regions.pointer_type, $($fty)*
            );
            Field::new_in_region(self.regions, VmType::$Name, payload, offset, flags, ty)
        }
    };

    // Emit the accessor `struct` for a `#[repr(C, u32)]` enum: its `u32` tag,
    // plus one nested accessor type per variant.
    //
    // An enum gets two regions, not one per field: variants' fields occupy the
    // same payload bytes, so they can alias both across and within a variant.
    (@emit_enum $Name:ident $snake:ident $tag:ident {
        $( { $VariantAccess:ident $vsnake:ident { $($vgroups:tt)* } } )*
    }) => {
        #[doc = concat!(
            "An [`AliasRegions`] accessor for the tag and variants of the `",
            stringify!($Name), "` enum."
        )]
        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        pub struct $Name<'a, Offsets> {
            regions: &'a mut AliasRegions<Offsets>,
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<Offsets> AliasRegions<Offsets> {
            #[doc = concat!(
                "Get an accessor for the tag and variants of a `",
                stringify!($Name), "`."
            )]
            pub fn $snake(&mut self) -> $Name<'_, Offsets> {
                $Name { regions: self }
            }
        }

        #[allow(
            dead_code,
            reason = "generated uniformly for all VM types; not all accessors are used yet"
        )]
        impl<'a, Offsets> $Name<'a, Offsets>
        where
            Offsets: GetPtrSize,
        {
            #[doc = concat!(
                "Get the [`Field`] for the `", stringify!($tag), "` tag of `",
                stringify!($Name), "`.\n\n\
                 The tag has its own alias region, distinct from the variants'."
            )]
            pub fn tag(self) -> Field<'a, Offsets> {
                let offset = u32::from(
                    self.regions.offsets.get_ptr_size().$snake().tag()
                );
                let flags = ir::MemFlagsData::trusted();
                let ty = define_vm_type_alias_region_helpers!(
                    @field_ty self.regions.pointer_type, $tag
                );
                Field::new(self.regions, VmType::$Name, offset, flags, ty)
            }
        }

        $(
            define_vm_type_alias_region_helpers!(
                @emit_variant $Name $snake $VariantAccess $vsnake { $($vgroups)* }
            );
        )*
    };

    // Peel variants off an enum's body one at a time; `@munch_variant` handles
    // each variant's own body and hands the remainder back.
    (@munch_enum $Name:ident $snake:ident $tag:ident { $($vs:tt)* }) => {
        define_vm_type_alias_region_helpers!(@emit_enum $Name $snake $tag { $($vs)* });
    };
    (@munch_enum $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $(#[doc = $vdoc:literal])*
        #[snake_name = $vsnake:ident]
        $(#[$vattr:meta])*
        $Variant:ident { $($vbody:tt)* } = $disc:literal ,
        $($rest:tt)*
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake {}
            [ $($vbody)* ] [ $($rest)* ]
        );
    };

    // As `@munch` for a struct body, resuming `@munch_enum` on the remainder.
    (@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $Variant:ident $vsnake:ident { $($g:tt)* } [] [ $($rest:tt)* ]
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_enum $Name $snake $tag
            { $($vs)* { $Variant $vsnake { $($g)* } } }
            $($rest)*
        );
    };
    // As in `@munch`, an `#[aggregate]` field gets no accessor.
    (@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $Variant:ident $vsnake:ident { $($g:tt)* }
        [ $(#[doc = $fdoc:literal])* #[aggregate] $fname:ident : $fty:ty , $($vrest:tt)* ] $rest:tt
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
            [ $($vrest)* ] $rest
        );
    };
    (@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $Variant:ident $vsnake:ident { $($g:tt)* }
        [ $(# $fattr:tt)* $fname:ident : $($vrest:tt)* ] $rest:tt
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_variant_ty $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
            $fname [ $($fattr)* ] [] [ $($vrest)* ] $rest
        );
    };
    (@munch_variant_ty $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $Variant:ident $vsnake:ident { $($g:tt)* }
        $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] [ , $($vrest:tt)* ] $rest:tt
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake
            { $($g)* { plain $fname [ $($fattr)* ] [ $($fty)* ] } }
            [ $($vrest)* ] $rest
        );
    };
    (@munch_variant_ty $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
        $Variant:ident $vsnake:ident { $($g:tt)* }
        $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] [ $tok:tt $($vrest:tt)* ] $rest:tt
    ) => {
        define_vm_type_alias_region_helpers!(
            @munch_variant_ty $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
            $fname [ $($fattr)* ] [ $($fty)* $tok ] [ $($vrest)* ] $rest
        );
    };

    // Dispatch on `struct` vs `enum`.
    (@def struct $Name:ident $snake:ident $repr:tt { $($body:tt)* }) => {
        define_vm_type_alias_region_helpers!(@munch $Name $snake {} $($body)*);
    };
    // See the matching arm in `define_vm_type_offsets!`.
    (@def enum $Name:ident $snake:ident [ C, $tag:ident ] { $($body:tt)* }) => {
        define_vm_type_alias_region_helpers!(@munch_enum $Name $snake $tag {} $($body)*);
    };

    // Top-level entry: the list of `VM*` type definitions. Each is either a
    // `struct` or a `#[repr(C, u32)]` `enum`; `@def` dispatches on which.
    ( $(
        $(#[doc = $sdoc:literal])*
        $(#[cfg($($scfg:tt)*)])?
        $(#[derive($($d:ident),*)])?
        #[repr($($repr:tt)*)]
        #[snake_name = $snake:ident]
        $svis:vis $kind:ident $Name:ident {
            $($body:tt)*
        }
    )* ) => {
        $(
            define_vm_type_alias_region_helpers!(@def $kind $Name $snake [$($repr)*] { $($body)* });
        )*
    };
}
wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);

/// Define, for each of Wasmtime's vmctx types, an [`AliasRegions`] accessor
/// that returns a wrapper exposing a [`Field`]-returning method per field of
/// that vmctx.
///
/// For example, `alias_regions.vmctx().epoch_ptr()` is the `VMContext::epoch_ptr`
/// field, and `alias_regions.vmcomponent().callbacks(i)` is the `i`th element of
/// the `VMComponentContext`'s runtime-callbacks array.
///
/// A vmctx's `static` fields sit at offsets that depend only on the target
/// pointer size, so their accessors are available for any `Offsets: GetPtrSize`.
/// Its `dynamic` fields sit at offsets that additionally depend on the module or
/// component being compiled, so their accessors are only available when the
/// `AliasRegions` carries that vmctx's own fully-computed offsets. The exception
/// is a `dynamic` field marked `#[ptr_size_offset]`, whose offsets are derived
/// only from the pointer size; these live in the `Offsets: GetPtrSize` block as
/// well.
///
/// A field marked `#[aggregate]` gets no accessor, for the same reason it gets
/// none in [`define_vm_type_alias_region_helpers!`]: it has no single Cranelift
/// type. Instead, its interior is reached by rebasing a [`Field`] of the nested
/// `VM*` type onto the aggregate's own offset within the vmctx (see
/// [`Field::relative_to`]), which produces exactly the same alias region as
/// accessing that `VM*` type through a pointer would. For the few aggregates that
/// have no `VM*` type of their own, and hence no alias region of their own, there
/// are hand-written helpers below.
///
/// A field marked `#[pointee(..)]` additionally gets an accessor for what it
/// points *at*, which lives outside the vmctx. For example,
/// `alias_regions.vmctx().builtin_functions_array(i)` is the `i`th builtin in
/// the array that the `VMContext::builtin_functions` field points at.
#[allow(
    unused_macro_rules,
    reason = "entry shapes and marker attributes are handled uniformly for both \
              sections of both vmctx types, but not every combination occurs"
)]
macro_rules! define_vmctx_alias_region_helpers {
    // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the target
    // pointer type as an `ir::Type`).
    (@field_ty $pt:expr, u32) => { ir::types::I32 };
    (@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };
    (@field_ty $pt:expr, VmPtr < VmPtr < $g:ident >>) => { $pt };
    (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
    (@field_ty $pt:expr, AtomicU64) => { ir::types::I64 };
    (@field_ty $pt:expr, unsafe extern "C" fn) => { $pt };

    // Determine the Cranelift type a field is *accessed* as: the classification
    // of its `#[access_as = T]` type if it has one, and of its declared type
    // otherwise.
    (@access_ty $pt:expr, [ $($fty:tt)* ] []) => {
        define_vmctx_alias_region_helpers!(@field_ty $pt, $($fty)*)
    };
    (@access_ty $pt:expr, $fty:tt [ #[access_as = $($t:tt)*] $($rest:tt)* ]) => {
        define_vmctx_alias_region_helpers!(@field_ty $pt, $($t)*)
    };
    (@access_ty $pt:expr, $fty:tt [ # $skip:tt $($rest:tt)* ]) => {
        define_vmctx_alias_region_helpers!(@access_ty $pt, $fty [ $($rest)* ])
    };

    // Apply a field attribute to its access flags.
    (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly(true) };
    (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move(true) };
    (@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };
    (@apply_attr $flags:expr, [ptr_size_offset]) => { $flags };
    (@apply_attr $flags:expr, [pointee( $($t:tt)* )]) => { $flags };

    // Compute a field's access flags and Cranelift type from its declared type
    // and marker attributes, and build the `Field` for it at `$offset`.
    (@field $Name:ident ($self:expr, $offset:expr) [ $($fty:tt)* ] [ $(# $fattr:tt)* ]) => {{
        let this = $self;
        let flags = ir::MemFlagsData::trusted();
        $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $fattr); )*
        let ty = define_vmctx_alias_region_helpers!(
            @access_ty this.regions.pointer_type, [ $($fty)* ] [ $(# $fattr)* ]
        );
        let offset = $offset;
        Field::new(this.regions, VmType::$Name, offset, flags, ty)
    }};

    // Scan a field's attributes for a `#[pointee(..)]` and emit the accessor it
    // describes, if any.
    (@pointee_methods $Offsets:tt [ ]) => {};
    (@pointee_methods $Offsets:tt [ #[pointee( $($p:tt)* )] $($rest:tt)* ]) => {
        define_vmctx_alias_region_helpers!(@pointee $Offsets $($p)*);
    };
    (@pointee_methods $Offsets:tt [ # $skip:tt $($rest:tt)* ]) => {
        define_vmctx_alias_region_helpers!(@pointee_methods $Offsets [ $($rest)* ]);
    };

    // An array of pointees.
    (@pointee [ $($Offsets:tt)* ]
        $(# $pattr:tt)* $Region:ident as $pname:ident [ $Index:ident ] : $($pty:tt)*
    ) => {
        #[doc = concat!(
            "Get the [`Field`] for the `index`th element of the out-of-line `",
            stringify!($Region), "` array.\n\nThe returned [`Field`] is relative \
             to the array's base pointer, not to the vmctx."
        )]
        pub fn $pname(self, index: $Index) -> Field<'a, $($Offsets)*> {
            let ty = define_vmctx_alias_region_helpers!(
                @field_ty self.regions.pointer_type, $($pty)*
            );
            let offset = index.vmctx_array_index().checked_mul(ty.bytes()).unwrap();
            let flags = ir::MemFlagsData::trusted();
            $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
            Field::new(self.regions, VmType::$Region, offset, flags, ty)
        }
    };

    // A single pointee.
    (@pointee [ $($Offsets:tt)* ]
        $(# $pattr:tt)* $Region:ident as $pname:ident : $($pty:tt)*
    ) => {
        #[doc = concat!(
            "Get the [`Field`] for the out-of-line `", stringify!($Region),
            "`.\n\nThe returned [`Field`] is relative to the pointer to it, not \
             to the vmctx."
        )]
        pub fn $pname(self) -> Field<'a, $($Offsets)*> {
            let ty = define_vmctx_alias_region_helpers!(
                @field_ty self.regions.pointer_type, $($pty)*
            );
            let flags = ir::MemFlagsData::trusted();
            $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
            Field::new(self.regions, VmType::$Region, 0, flags, ty)
        }
    };

    // ### `static` Section Entries

    (@static_entry $Name:ident $snake:ident align { $al:tt }) => {};

    // Aggregates get no accessor.
    (@static_entry $Name:ident $snake:ident $kind:ident { #[aggregate] $($rest:tt)* }) => {};

    (@static_entry $Name:ident $snake:ident field {
        $(# $fattr:tt)* $fname:ident : $($fty:tt)*
    }) => {
        #[doc = concat!(
            "Get the [`Field`] for the `", stringify!($fname), "` field of `",
            stringify!($Name), "`."
        )]
        pub fn $fname(self) -> Field<'a, Offsets> {
            let offset = u32::from(self.regions.offsets.get_ptr_size().$snake().$fname());
            define_vmctx_alias_region_helpers!(
                @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
            )
        }

        define_vmctx_alias_region_helpers!(@pointee_methods [ Offsets ] [ $(# $fattr)* ]);
    };

    // ### `dynamic` Section Entries Marked `#[ptr_size_offset]`
    //
    // These get their offsets from the pointer-size-only `offsets::VMFoo<P>`
    // wrapper, and don't require a full `VMOffsets` parameterization.

    (@ptr_size_entry $Name:ident $snake:ident field {
        #[ptr_size_offset] $(# $fattr:tt)* $fname:ident : $($fty:tt)*
    }) => {
        #[doc = concat!(
            "Get the [`Field`] for the `", stringify!($fname), "` field of `",
            stringify!($Name), "`."
        )]
        pub fn $fname(self) -> Field<'a, Offsets> {
            let offset = self.regions.offsets.get_ptr_size().$snake().$fname();
            define_vmctx_alias_region_helpers!(
                @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
            )
        }
    };

    (@ptr_size_entry $Name:ident $snake:ident array {
        #[ptr_size_offset] $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
    }) => {
        #[doc = concat!(
            "Get the [`Field`] for the `index`th element of `", stringify!($Name),
            "`'s `", stringify!($fname), "` array.\n\nThis is not bounds checked: \
             the array's length depends on the module or component being compiled, \
             which is precisely what this accessor does not require knowing."
        )]
        pub fn $fname(self, index: $Index) -> Field<'a, Offsets> {
            let offset = self.regions.offsets.get_ptr_size().$snake().$fname(index);
            define_vmctx_alias_region_helpers!(
                @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
            )
        }
    };

    (@ptr_size_entry $Name:ident $snake:ident $kind:ident $entry:tt) => {};

    // ### `dynamic` Section Entries

    // Entries marked `#[ptr_size_offset]` were already handled above; emitting
    // them here too would be a duplicate definition.
    (@dynamic_entry $Name:ident $Offsets:tt $kind:ident {
        #[ptr_size_offset] $($rest:tt)*
    }) => {};

    (@dynamic_entry $Name:ident $Offsets:tt align { $al:tt }) => {};

    // Aggregates get no accessor.
    (@dynamic_entry $Name:ident $Offsets:tt $kind:ident { #[aggregate] $($rest:tt)* }) => {};

    (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] array {
        $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
    }) => {
        #[doc = concat!(
            "Get the [`Field`] for the `index`th element of `", stringify!($Name),
            "`'s `", stringify!($fname), "` array.\n\nPanics if `index` is out of \
             bounds for this vmctx."
        )]
        pub fn $fname(self, index: $Index) -> Field<'a, $($Offsets)*> {
            let offset = self.regions.offsets.$fname().at(index);
            define_vmctx_alias_region_helpers!(
                @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
            )
        }

        define_vmctx_alias_region_helpers!(
            @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
        );
    };

    // A single field, whether unconditionally present (`field`) or only
    // conditionally (`optional`).
    (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] $kind:ident {
        $(# $fattr:tt)* $fname:ident $([ if $flag:ident ])? : $($fty:tt)*
    }) => {
        #[doc = concat!(
            "Get the [`Field`] for the `", stringify!($fname), "` field of `",
            stringify!($Name), "`."
            $(, "\n\nPanics if `", stringify!($flag), "` is false, in which case \
                 this field is not present at all.")?
        )]
        pub fn $fname(self) -> Field<'a, $($Offsets)*> {
            let offset = self.regions.offsets.$fname();
            define_vmctx_alias_region_helpers!(
                @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
            )
        }

        define_vmctx_alias_region_helpers!(
            @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
        );
    };

    // Emit the `impl` block holding the accessors for the dynamically-positioned
    // fields.
    (@dynamic_impl $Name:ident [ $($Offsets:tt)* ] $OffsetsTt:tt {
        $($kind:ident $entry:tt)*
    }) => {
        #[allow(
            dead_code,
            reason = "generated uniformly for every field; not all fields are \
                      accessed by compiled code"
        )]
        impl<'a> $Name<'a, $($Offsets)*> {
            $(
                define_vmctx_alias_region_helpers!(
                    @dynamic_entry $Name $OffsetsTt $kind $entry
                );
            )*
        }
    };

    // Emit the accessor `struct` and both `impl` blocks for one vmctx type.
    (@emit $Name:ident $snake:ident $Offsets:tt
        static { $($skind:ident $sentry:tt)* }
        dynamic { $($dkind:ident $dentry:tt)* }
    ) => {
        #[doc = concat!(
            "An [`AliasRegions`] accessor for the fields of a `", stringify!($Name),
            "`."
        )]
        pub struct $Name<'a, Offsets> {
            regions: &'a mut AliasRegions<Offsets>,
        }

        impl<Offsets> AliasRegions<Offsets> {
            #[doc = concat!(
                "Get an accessor for the fields of a `", stringify!($Name), "`."
            )]
            pub fn $snake(&mut self) -> $Name<'_, Offsets> {
                $Name { regions: self }
            }
        }

        // A statically-positioned field's offset depends only on the target
        // pointer size, so these accessors work with any `Offsets`.
        #[allow(
            dead_code,
            reason = "generated uniformly for every field; not all fields are \
                      accessed by compiled code"
        )]
        impl<'a, Offsets> $Name<'a, Offsets>
        where
            Offsets: GetPtrSize,
        {
            $( define_vmctx_alias_region_helpers!(@static_entry $Name $snake $skind $sentry); )*
            $( define_vmctx_alias_region_helpers!(@ptr_size_entry $Name $snake $dkind $dentry); )*
        }

        // A dynamically-positioned field's offset depends on the module or
        // component being compiled, so these accessors require this vmctx's own
        // fully-computed offsets.
        define_vmctx_alias_region_helpers!(
            @dynamic_impl $Name $Offsets $Offsets { $($dkind $dentry)* }
        );
    };

    // Map each vmctx type to the offsets type that computes its
    // dynamically-positioned fields' offsets.
    (@one VMContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
        define_vmctx_alias_region_helpers!(@emit VMContext $snake [ VMOffsets<u8> ]
            static { $($stat)* } dynamic { $($dyn)* });
    };
    (@one VMComponentContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
        define_vmctx_alias_region_helpers!(
            @emit VMComponentContext $snake [ VMComponentOffsets<u8> ]
            static { $($stat)* } dynamic { $($dyn)* }
        );
    };

    // Top-level entry.
    ( $(
        {
            $Name:ident $snake:ident
            static { $($stat:tt)* }
            dynamic { $($dyn:tt)* }
        }
    )* ) => {
        $(
            define_vmctx_alias_region_helpers!(@one $Name $snake
                static { $($stat)* } dynamic { $($dyn)* });
        )*
    };
}
wasmtime_environ::for_each_vmctx_type!(define_vmctx_alias_region_helpers);

impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    /// Create a new `AliasRegions`.
    pub fn new(offsets: Offsets) -> Self {
        Self {
            pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())
                .unwrap(),
            offsets,
            gc_introducer_cache: HashMap::new(),
        }
    }

    /// The Cranelift type of a pointer on the target being compiled for.
    pub fn pointer_type(&self) -> ir::Type {
        self.pointer_type
    }

    /// The pointer size of the target being compiled for, used to compute the
    /// layout of Wasmtime's vmctx types.
    pub fn ptr_size(&self) -> &Offsets::Ptr {
        self.offsets.get_ptr_size()
    }

    /// Get the alias region for the given GC object access.
    pub fn gc_access_region(
        &mut self,
        func: &mut ir::Function,
        types: &ModuleTypesBuilder,
        access: GcAccess,
    ) -> ir::AliasRegion {
        let key = match access {
            GcAccess::HeaderKind => return self.vm_gc_header().kind().region(func),
            GcAccess::HeaderTypeIndex => return self.vm_gc_header().ty().region(func),
            GcAccess::DrcRefCount => return self.vm_drc_header().ref_count().region(func),
            GcAccess::DrcNextOverApproximatedStackRoot => {
                return self
                    .vm_drc_header()
                    .next_over_approximated_stack_root()
                    .region(func);
            }
            GcAccess::CopyingObjectSize => {
                return self.vm_copying_header().object_size().region(func);
            }

            GcAccess::ArrayLength => AliasRegionKey::GcArrayLength,
            GcAccess::ExnTagInstance => AliasRegionKey::GcExnTagInstance,
            GcAccess::ExnTagDefined => AliasRegionKey::GcExnTagDefined,

            GcAccess::StructField { ty, field } => AliasRegionKey::GcStructField {
                ty: self.gc_introducer(types, ty, field),
                field,
            },
            GcAccess::ArrayElements { ty } => AliasRegionKey::GcArrayElements {
                ty: self.gc_introducer(types, ty, 0),
            },
            GcAccess::ExnPayload { exn_ty, field } => {
                AliasRegionKey::GcExnPayload { exn_ty, field }
            }
        };
        self.region(func, key)
    }

    /// Find the type that introduced the given field (or, for arrays, the
    /// elements) into `ty`'s subtyping hierarchy, memoizing the answer.
    fn gc_introducer(
        &mut self,
        types: &ModuleTypesBuilder,
        ty: ModuleInternedTypeIndex,
        field: u32,
    ) -> ModuleInternedTypeIndex {
        if let Some(introducer) = self.gc_introducer_cache.get(&(ty, field)) {
            return *introducer;
        }

        let mut introducer = ty;
        while let Some(supertype) = types[introducer].supertype {
            let supertype = supertype.unwrap_module_type_index();

            let supertype_declares_field = match &types[supertype].composite_type.inner {
                // Subtyping a struct can only append new fields, so we need
                // only look at the field length.
                WasmCompositeInnerType::Struct(s) => field < u32::try_from(s.fields.len()).unwrap(),

                // Array elements are always introduced by the first type
                // without a supertype.
                WasmCompositeInnerType::Array(_) => true,

                inner @ WasmCompositeInnerType::Func(_)
                | inner @ WasmCompositeInnerType::Cont(_)
                | inner @ WasmCompositeInnerType::Exn(_) => {
                    unreachable!("not a GC type with subtyping: {inner:?}")
                }
            };

            if supertype_declares_field {
                introducer = supertype;
            } else {
                break;
            }
        }

        self.gc_introducer_cache.insert((ty, field), introducer);
        introducer
    }

    /// Get the alias region shared by all memories that cross a module boundary
    /// and whose definition we do not statically know.
    pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::PublicMemory)
    }

    /// Get the alias region for accessing a particular defined memory.
    pub fn defined_memory_region(
        &mut self,
        func: &mut ir::Function,
        module: StaticModuleIndex,
        index: DefinedMemoryIndex,
    ) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::DefinedMemory { module, index })
    }

    /// Get the alias region shared by all tables that cross a module boundary
    /// and whose definition we do not statically know.
    pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::PublicTable)
    }

    /// Get the alias region for accessing a particular defined table.
    pub fn defined_table_region(
        &mut self,
        func: &mut ir::Function,
        module: StaticModuleIndex,
        index: DefinedTableIndex,
    ) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::DefinedTable { module, index })
    }

    /// Get the alias region shared by all globals that cross a module boundary
    /// and whose definition we do not statically know.
    pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::PublicGlobal)
    }

    /// Get the alias region for accessing a particular defined global.
    pub fn defined_global_region(
        &mut self,
        func: &mut ir::Function,
        module: StaticModuleIndex,
        index: DefinedGlobalIndex,
    ) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::DefinedGlobal { module, index })
    }
}

/// `VMStoreContext`-related methods.
impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    fn vmstore_context_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
        self.region(
            func,
            AliasRegionKey::Vm {
                ty: VmType::VMStoreContext,
                offset,
            },
        )
    }

    /// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
    ///
    /// The caller supplies the base flags because whether the base pointer is
    /// `readonly`/`can_move` depends on the GC heap's tunables.
    pub fn vmstore_context_gc_heap_base_load(
        &mut self,
        func: &mut ir::Function,
        base_flags: ir::MemFlagsData,
    ) -> Load {
        let offset = self
            .offsets
            .get_ptr_size()
            .vm_store_context()
            .gc_heap_base()
            .into();
        let region = self.vmstore_context_region(func, offset);
        Load {
            offset,
            flags: base_flags.with_alias_region(Some(region)),
            ty: self.pointer_type,
        }
    }

    /// Load the GC heap base pointer (`VMStoreContext::gc_heap.base`).
    ///
    /// The caller supplies the base flags because whether the base pointer is
    /// `readonly`/`can_move` depends on the GC heap's tunables.
    pub fn vmstore_context_gc_heap_base(
        &mut self,
        cursor: &mut FuncCursor<'_>,
        base_flags: ir::MemFlagsData,
        vmstore_ctx: ir::Value,
    ) -> ir::Value {
        self.vmstore_context_gc_heap_base_load(cursor.func, base_flags)
            .emit(cursor, vmstore_ctx)
    }

    /// Get a `Load` of the GC heap bound (`VMStoreContext::gc_heap.current_length`).
    pub fn vmstore_context_gc_heap_current_length_load(&mut self, func: &mut ir::Function) -> Load {
        let offset = self
            .offsets
            .get_ptr_size()
            .vm_store_context()
            .gc_heap_current_length()
            .into();
        let region = self.vmstore_context_region(func, offset);
        Load {
            offset,
            flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
            ty: self.pointer_type,
        }
    }

    /// Load the GC heap bound (`VMStoreContext::gc_heap.current_length`).
    pub fn vmstore_context_gc_heap_current_length(
        &mut self,
        cursor: &mut FuncCursor<'_>,
        vmstore_ctx: ir::Value,
    ) -> ir::Value {
        self.vmstore_context_gc_heap_current_length_load(cursor.func)
            .emit(cursor, vmstore_ctx)
    }

    /// Get the alias region for the `VMStoreContext::stack_chain` field.
    ///
    /// The `VMStackChain` is two pointers wide and is emitted by the stack
    /// switching `VMStackChain` load/store helpers, which take a region
    /// argument; this provides that region.
    pub fn vmstore_context_stack_chain_region(
        &mut self,
        func: &mut ir::Function,
    ) -> ir::AliasRegion {
        let offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();
        self.vmstore_context_region(func, offset.into())
    }
}

/// Component-specific methods.
impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    /// Get the alias region for the `ValRaw` array used to marshal arguments
    /// and results across the array calling convention used by various
    /// trampolines.
    pub fn host_val_raw_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(
            func,
            AliasRegionKey::Vm {
                ty: VmType::HostValRaw,
                offset: 0,
            },
        )
    }
}

/// Passive data and element segment-related methods.
impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    /// Get the alias region for the runtime `ValRaw` storage of passive element
    /// segments (written by passive-element initialization and read by
    /// `table.init` and `array.{new,init}_elem`).
    pub fn element_segment_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::ElementSegment)
    }

    /// Get the alias region for the bytes of data segments (read by
    /// `memory.init` and `array.{new,init}_data`).
    pub fn data_segment_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::DataSegment)
    }

    /// Get the alias region for the shadow bytes of linear memories, which
    /// watchpoint checks read before stores.
    pub fn shadow_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(func, AliasRegionKey::ShadowMemory)
    }
}

/// Unsafe intrinsic-related methods.
impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    /// Perform an unsafe intrinsic's load.
    pub fn unsafe_intrinsic_load(
        &mut self,
        cursor: &mut FuncCursor<'_>,
        ty: ir::Type,
        base_flags: ir::MemFlagsData,
        addr: ir::Value,
    ) -> ir::Value {
        let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);
        cursor
            .ins()
            .load(ty, base_flags.with_alias_region(Some(region)), addr, 0)
    }

    /// Perform an unsafe intrinsic's store.
    pub fn unsafe_intrinsic_store(
        &mut self,
        cursor: &mut FuncCursor<'_>,
        base_flags: ir::MemFlagsData,
        val: ir::Value,
        addr: ir::Value,
    ) {
        let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);
        cursor
            .ins()
            .store(base_flags.with_alias_region(Some(region)), val, addr, 0);
    }
}

/// `VMComponentContext` fields that are not simply one of the layout's own
/// fields, and so are not generated by [`define_vmctx_alias_region_helpers!`].
impl AliasRegions<VMComponentOffsets<u8>> {
    /// Get the [`Field`] for the `callee` of the `index`th lowering in the
    /// `VMComponentContext`'s `lowerings` array.
    pub fn vmcomponent_lowering_callee(
        &mut self,
        index: LoweredIndex,
    ) -> Field<'_, VMComponentOffsets<u8>> {
        let offset = self.offsets.lowering_callee(index);
        self.vmlowering_field(offset)
    }

    /// Get the [`Field`] for the host data of the `index`th lowering in the
    /// `VMComponentContext`'s `lowerings` array.
    pub fn vmcomponent_lowering_data(
        &mut self,
        index: LoweredIndex,
    ) -> Field<'_, VMComponentOffsets<u8>> {
        let offset = self.offsets.lowering_data(index);
        self.vmlowering_field(offset)
    }

    /// Get the [`Field`] at `offset` within a `VMLowering` inlined into the
    /// `VMComponentContext`.
    ///
    /// Unlike the other aggregates inlined into a vmctx, `VMLowering` is not one
    /// of the types defined by `for_each_vm_type!`, and so has no alias region of
    /// its own to rebase onto the aggregate's offset. Its fields are therefore
    /// part of the containing `VMComponentContext`'s region, keyed by their
    /// offset within it.
    fn vmlowering_field(&mut self, offset: u32) -> Field<'_, VMComponentOffsets<u8>> {
        let ty = self.pointer_type;
        Field::new(
            self,
            VmType::VMComponentContext,
            offset,
            ir::MemFlagsData::trusted(),
            ty,
        )
    }
}

/// Stack-switching and continuation-object methods.
impl<Offsets> AliasRegions<Offsets>
where
    Offsets: GetPtrSize,
{
    /// Get the alias region for the `VMContRef::parent_chain` field.
    ///
    /// Like `VMStoreContext::stack_chain`, this field is a two-pointer-wide
    /// `VMStackChain`, which is not yet defined by `for_each_vm_type!` and so
    /// has no alias region of its own. Its bytes are therefore mapped to the
    /// containing `VMContRef`'s regions, keyed by the field's offset within it.
    pub fn vm_cont_ref_parent_chain_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        let offset = self.offsets.get_ptr_size().vm_cont_ref().parent_chain();
        self.region(
            func,
            AliasRegionKey::Vm {
                ty: VmType::VMContRef,
                offset: offset.into(),
            },
        )
    }

    /// Get the alias region for the top-of-stack pointer at the start of the
    /// `VMContRef::stack` field.
    ///
    /// `VMContinuationStack` is a platform-specific struct private to the
    /// runtime, is not one of the types defined by `for_each_vm_type!`, and so
    /// has no alias region of its own; as with `parent_chain` above, its bytes
    /// are mapped to the containing `VMContRef`'s regions.
    pub fn vm_cont_ref_top_of_stack_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        let offset = self.offsets.get_ptr_size().vm_cont_ref().stack();
        self.region(
            func,
            AliasRegionKey::Vm {
                ty: VmType::VMContRef,
                offset: offset.into(),
            },
        )
    }

    /// Region for a continuation's stack memory: its payload/handler data
    /// buffers and the control-context records stored on the continuation
    /// stack.
    ///
    /// These are distinct from the `VMContRef` object itself (which only holds
    /// pointers to them).
    pub fn continuation_stack_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
        self.region(
            func,
            AliasRegionKey::Vm {
                ty: VmType::ContinuationStackMemory,
                offset: 0,
            },
        )
    }
}

/// Debug-assert that every CLIF memory instruction in `func` carries an alias
/// region.
///
/// The Wasm-to-CLIF translation in `crates/cranelift/*` tags 100% of the memory
/// instructions it emits (loads, stores, and atomics) with an alias region.
/// This helper upholds that invariant; it is called at each
/// `FunctionBuilder::finalize` site, after finalization so that the stack-map
/// spills inserted by the safepoint pass (tagged via the
/// `FunctionBuilder::make_stack_map_alias_region` hook) are checked too.
pub(crate) fn debug_assert_all_mem_insts_have_alias_regions(func: &ir::Function) {
    if cfg!(debug_assertions) {
        for block in func.layout.blocks() {
            for inst in func.layout.block_insts(block) {
                // Only loads, stores, and atomics access memory. Some non-memory
                // instructions (e.g. `bitcast`) also carry `MemFlags` purely for
                // lane/byte-order and legitimately have no alias region, so they
                // are excluded here.
                let opcode = func.dfg.insts[inst].opcode();
                if !opcode.can_load() && !opcode.can_store() {
                    continue;
                }
                if let Some(flags) = func.dfg.insts[inst].memflags() {
                    debug_assert!(
                        func.dfg.mem_flags[flags].alias_region().is_some(),
                        "CLIF memory instruction emitted without an alias region: {}",
                        func.dfg.display_inst(inst),
                    );
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Collisions are sound and only cost an optimization, so this is not a
    /// correctness test: it exists so that anything reshuffling the key hashes
    /// says so here rather than as `tests/disas` diffs where two regions have
    /// quietly merged. If it fires, find a new `SEED` as described on
    /// `into_raw`, additionally covering the keys below.
    #[test]
    fn gc_keys_do_not_collide() {
        let ty = ModuleInternedTypeIndex::from_u32;

        let mut keys = vec![
            AliasRegionKey::GcArrayLength,
            AliasRegionKey::GcExnTagInstance,
            AliasRegionKey::GcExnTagDefined,
            AliasRegionKey::Vm {
                ty: VmType::VMGcHeader,
                offset: 0,
            },
            AliasRegionKey::Vm {
                ty: VmType::VMGcHeader,
                offset: 4,
            },
            AliasRegionKey::Vm {
                ty: VmType::VMDrcHeader,
                offset: 8,
            },
            AliasRegionKey::Vm {
                ty: VmType::VMDrcHeader,
                offset: 16,
            },
            AliasRegionKey::Vm {
                ty: VmType::VMCopyingHeader,
                offset: 8,
            },
        ];
        for t in 0..3 {
            for field in 0..2 {
                keys.push(AliasRegionKey::GcStructField { ty: ty(t), field });
            }
            keys.push(AliasRegionKey::GcArrayElements { ty: ty(t) });
        }

        for (i, a) in keys.iter().enumerate() {
            for b in &keys[i + 1..] {
                assert_ne!(
                    a.into_raw(),
                    b.into_raw(),
                    "alias region keys {a:?} and {b:?} collide on \
                     user_id {}; a new `SEED` is needed",
                    a.into_raw(),
                );
            }
        }
    }
}