A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: box a few fields in `MachBuffer`/`MachBufferFinalized` to avoid memmoves. (#14300)
Chris Fallin817c58787f43
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rsindex d8acfed3ab..5783fd7798 100644--- a/cranelift/codegen/src/ir/mod.rs+++ b/cranelift/codegen/src/ir/mod.rs@@ -61,6 +61,7 @@ pub use crate::ir::memflags::{MemFlagsSet,};pub use crate::ir::progpoint::ProgramPoint;+pub use crate::ir::sourceloc::MaybeRelSourceLoc;pub use crate::ir::sourceloc::RelSourceLoc;pub use crate::ir::sourceloc::SourceLoc;pub use crate::ir::stackslot::{diff --git a/cranelift/codegen/src/ir/sourceloc.rs b/cranelift/codegen/src/ir/sourceloc.rsindex 6704574d0d..95817d2e79 100644--- a/cranelift/codegen/src/ir/sourceloc.rs+++ b/cranelift/codegen/src/ir/sourceloc.rs@@ -9,11 +9,12 @@ use serde_derive::{Deserialize, Serialize};/// A source location.///-/// This is an opaque 32-bit number attached to each Cranelift IR instruction. Cranelift does not+/// This is an opaque 31-bit number attached to each Cranelift IR instruction. Cranelift does not/// interpret source locations in any way, they are simply preserved from the input to the output.///-/// The default source location uses the all-ones bit pattern `!0`. It is used for instructions-/// that can't be given a real source location.+/// The default source location uses the bit pattern `0x7fff_ffff`. It is used for instructions+/// that can't be given a real source location. The high bit is reserved for+/// distinguishing relative and absolute locations in [`MaybeRelSourceLoc`].#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]pub struct SourceLoc(u32);@@ -37,7 +38,7 @@ impl SourceLoc {impl Default for SourceLoc {fn default() -> Self {- Self(!0)+ Self(0x7fff_ffff)}}@@ -67,7 +68,9 @@ impl RelSourceLoc {if base.is_default() || offset.is_default() {Self::default()} else {- Self(offset.bits().wrapping_sub(base.bits()))+ // Wrap within the 31-bit source-location range, reserving the+ // high bit for MaybeRelSourceLoc's relative/absolute tag.+ Self(offset.bits().wrapping_sub(base.bits()) & MaybeRelSourceLoc::MASK)}}@@ -76,7 +79,7 @@ impl RelSourceLoc {if self.is_default() || base.is_default() {Default::default()} else {- SourceLoc::new(self.0.wrapping_add(base.bits()))+ SourceLoc::new(self.0.wrapping_add(base.bits()) & MaybeRelSourceLoc::MASK)}}@@ -88,7 +91,7 @@ impl RelSourceLoc {impl Default for RelSourceLoc {fn default() -> Self {- Self(!0)+ Self(0x7fff_ffff)}}@@ -102,6 +105,84 @@ impl fmt::Display for RelSourceLoc {}}+/// A source location that is either a `RelSourceLoc` or `SourceLoc`.+///+/// This is used to represent a source location in the `MachBuffer`+/// that is initially relative to some base and is later relocated. We+/// do this to permit better code caching during incremental+/// compilation: the MachBuffer records the first SourceLoc it is+/// given as a base, and if the same function IR is later compiled but+/// with a different starting SourceLoc, we can reuse the cached+/// compilation result and just relocate (offset) the SourceLocs.+///+/// This relocation is an in-place update pass, so we want a "union"+/// type, essentially, but we don't want to pay the overhead of a true+/// `enum` (8 bytes rather than 4 in a large array), so we bitpack+/// this representation.+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct MaybeRelSourceLoc(u32);++impl MaybeRelSourceLoc {+ const REL_BIT: u32 = 0x8000_0000;+ const MASK: u32 = !Self::REL_BIT;++ /// Create a relative SourceLoc.+ pub fn rel(loc: RelSourceLoc) -> Self {+ debug_assert!(loc.0 & Self::MASK == loc.0);+ MaybeRelSourceLoc(loc.0 | Self::REL_BIT)+ }++ /// Create an absolute SourceLoc.+ pub fn abs(loc: SourceLoc) -> Self {+ debug_assert!(loc.0 & Self::MASK == loc.0);+ MaybeRelSourceLoc(loc.0)+ }++ /// Is this a relative SourceLoc?+ pub fn is_rel(&self) -> bool {+ self.0 & Self::REL_BIT != 0+ }++ /// Is this an absolute SourceLoc?+ pub fn is_abs(&self) -> bool {+ self.0 & Self::REL_BIT == 0+ }++ /// Unwrap a relative SourceLoc.+ ///+ /// # Panics+ ///+ /// Panics if this is not a relative SourceLoc.+ pub fn as_rel(&self) -> RelSourceLoc {+ assert!(self.is_rel());+ RelSourceLoc(self.0 & Self::MASK)+ }++ /// Unwrap an absolute SourceLoc.+ ///+ /// # Panics+ ///+ /// Panics if this is not an absolute SourceLoc.+ pub fn as_abs(&self) -> SourceLoc {+ assert!(self.is_abs());+ SourceLoc(self.0)+ }++ /// Map a relative to an absolute SourceLoc, given another+ /// absolute SourceLoc as a base.+ ///+ /// # Panics+ ///+ /// Panics if this is not a relative SourceLoc.+ pub fn relocate(&self, base: SourceLoc) -> SourceLoc {+ self.as_rel().expand(base)+ }+}+#[cfg(test)]mod tests {use crate::ir::SourceLoc;diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rsindex 666d305838..f419633929 100644--- a/cranelift/codegen/src/isa/aarch64/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/mod.rs@@ -6,7 +6,6 @@ use crate::isa::aarch64::settings as aarch64_settings;#[cfg(feature = "unwind")]use crate::isa::unwind::systemv;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};-#[cfg(feature = "unwind")]use crate::machinst::CompiledCode;use crate::machinst::{CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,@@ -93,13 +92,13 @@ impl TargetIsa for AArch64Backend {log::debug!("disassembly:\n{disasm}");}- Ok(CompiledCodeStencil {+ Ok(CompiledCodeStencil(CompiledCode {buffer,vcode: emit_result.disasm,value_labels_ranges,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,- })+ }))}fn name(&self) -> &'static str {diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rsindex 349f10363c..2140bcb562 100644--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs@@ -12,7 +12,7 @@ use crate::{dominator_tree::DominatorTree,ir,isa::{self, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa},- machinst::{self, CompiledCodeStencil, MachInst, SigSet, VCode},+ machinst::{self, CompiledCode, CompiledCodeStencil, MachInst, SigSet, VCode},result::CodegenResult,settings::{self as shared_settings, Flags},};@@ -198,13 +198,13 @@ wherelog::debug!("disassembly:\n{disasm}");}- Ok(CompiledCodeStencil {+ Ok(CompiledCodeStencil(CompiledCode {buffer,vcode: emit_result.disasm,value_labels_ranges,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,- })+ }))}fn emit_unwind_info(diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rsindex 96ec6b81c7..562576a6fc 100644--- a/cranelift/codegen/src/isa/riscv64/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/mod.rs@@ -6,7 +6,6 @@ use crate::isa::riscv64::settings as riscv_settings;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa,};-#[cfg(feature = "unwind")]use crate::machinst::CompiledCode;use crate::machinst::{CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,@@ -95,13 +94,13 @@ impl TargetIsa for Riscv64Backend {log::debug!("disassembly:\n{disasm}");}- Ok(CompiledCodeStencil {+ Ok(CompiledCodeStencil(CompiledCode {buffer,vcode: emit_result.disasm,value_labels_ranges,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,- })+ }))}fn name(&self) -> &'static str {diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rsindex 8323bd50c7..2f8200644e 100644--- a/cranelift/codegen/src/isa/s390x/mod.rs+++ b/cranelift/codegen/src/isa/s390x/mod.rs@@ -92,13 +92,13 @@ impl TargetIsa for S390xBackend {log::debug!("disassembly:\n{disasm}");}- Ok(CompiledCodeStencil {+ Ok(CompiledCodeStencil(CompiledCode {buffer,vcode: emit_result.disasm,value_labels_ranges,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,- })+ }))}fn name(&self) -> &'static str {diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rsindex ae2f5daecd..3b1f9afd8d 100644--- a/cranelift/codegen/src/isa/x64/mod.rs+++ b/cranelift/codegen/src/isa/x64/mod.rs@@ -3,6 +3,7 @@pub use self::inst::{AtomicRmwSeqOp, EmitInfo, EmitState, Inst, args, external};use super::{OwnedTargetIsa, TargetIsa};+use crate::MachBufferFinalized;use crate::dominator_tree::DominatorTree;use crate::ir::{self, Function, Type, types};#[cfg(feature = "unwind")]@@ -10,12 +11,11 @@ use crate::isa::unwind::systemv;use crate::isa::x64::settings as x64_settings;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey};use crate::machinst::{- CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,- compile,+ CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,+ TextSectionBuilder, VCode, compile,};use crate::result::{CodegenError, CodegenResult};use crate::settings::{self as shared_settings, Flags};-use crate::{Final, MachBufferFinalized};use alloc::string::String;use alloc::{borrow::ToOwned, boxed::Box, vec::Vec};use core::fmt;@@ -101,13 +101,13 @@ impl TargetIsa for X64Backend {crate::trace!("disassembly:\n{}", disasm);}- Ok(CompiledCodeStencil {+ Ok(CompiledCodeStencil(CompiledCode {buffer,vcode: emit_result.disasm,value_labels_ranges,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,- })+ }))}fn flags(&self) -> &Flags {@@ -223,7 +223,7 @@ impl TargetIsa for X64Backend {/// Emit unwind info for an x86 target.pub fn emit_unwind_info(- buffer: &MachBufferFinalized<Final>,+ buffer: &MachBufferFinalized,kind: crate::isa::unwind::UnwindInfoKind,) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {#[cfg(feature = "unwind")]diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rsindex 787aff2b11..d2cfe3ce86 100644--- a/cranelift/codegen/src/lib.rs+++ b/cranelift/codegen/src/lib.rs@@ -70,12 +70,11 @@ pub mod write;pub use crate::entity::packed_option;pub use crate::machinst::buffer::{- ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, FinalizedMachReloc,- FinalizedRelocTarget, MachCallSite, MachExceptionHandler, MachSrcLoc, MachTextSectionBuilder,- MachTrap, OpenPatchRegion, PatchRegion,+ ExceptionContextLoc, MachCallSite, MachCallSiteItem, MachExceptionHandler, MachReloc,+ MachSrcLoc, MachTextSectionBuilder, MachTrap, OpenPatchRegion, PatchRegion, RelocTarget,};pub use crate::machinst::{- CallInfo, CompiledCode, Final, FrameLayout, MachBuffer, MachBufferDebugTagList,+ CallInfo, CompiledCode, FrameLayout, LabelOrOffset, MachBuffer, MachBufferDebugTagList,MachBufferFinalized, MachBufferFrameLayout, MachDebugTagPos, MachInst, MachInstEmit,MachInstEmitState, MachLabel, RealReg, Reg, RegClass, RelocDistance, TextSectionBuilder,VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, Writable,diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rsindex 9c5ab99ccd..996310db7d 100644--- a/cranelift/codegen/src/machinst/abi.rs+++ b/cranelift/codegen/src/machinst/abi.rs@@ -2620,8 +2620,8 @@ impl TryCallInfo {layout: &FrameLayout,) -> impl Iterator<Item = MachExceptionHandler> {self.exception_handlers.iter().map(|handler| match handler {- TryCallHandler::Tag(tag, label) => MachExceptionHandler::Tag(*tag, *label),- TryCallHandler::Default(label) => MachExceptionHandler::Default(*label),+ TryCallHandler::Tag(tag, label) => MachExceptionHandler::Tag(*tag, LabelOrOffset::from(*label)),+ TryCallHandler::Default(label) => MachExceptionHandler::Default(LabelOrOffset::from(*label)),TryCallHandler::Context(reg) => {let loc = if let Some(spillslot) = reg.to_spillslot() {// The spillslot offset is relative to the "fixeddiff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rsindex 6f6e8564e7..c86a19b644 100644--- a/cranelift/codegen/src/machinst/buffer.rs+++ b/cranelift/codegen/src/machinst/buffer.rs@@ -200,7 +200,9 @@use crate::binemit::{Addend, CodeOffset, Reloc};use crate::ir::function::FunctionParameters;-use crate::ir::{DebugTag, ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};+use crate::ir::{+ DebugTag, ExceptionTag, ExternalName, MaybeRelSourceLoc, RelSourceLoc, SourceLoc, TrapCode,+};use crate::isa::unwind::UnwindInst;use crate::machinst::{BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst,@@ -208,54 +210,99 @@ use crate::machinst::{use crate::trace;use crate::{MachInstEmitState, ir};use crate::{VCodeConstantData, timing};+use alloc::boxed::Box;use alloc::collections::BinaryHeap;use alloc::string::String;use alloc::vec::Vec;use core::cmp::Ordering;use core::mem;use core::ops::Range;+use core::ops::{Deref, DerefMut};use cranelift_control::ControlPlane;use cranelift_entity::{PrimaryMap, SecondaryMap, entity_impl};use smallvec::SmallVec;-#[cfg(feature = "enable-serde")]-use serde::{Deserialize, Serialize};--#[cfg(feature = "enable-serde")]-pub trait CompilePhase {- type MachSrcLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone;- type SourceLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone;+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum ForceVeneers {+ Yes,+ No,}-#[cfg(not(feature = "enable-serde"))]-pub trait CompilePhase {- type MachSrcLocType: core::fmt::Debug + PartialEq + Clone;- type SourceLocType: core::fmt::Debug + PartialEq + Clone;-}+/// A `MachLabel` or `CodeOffset`, bitpacked into a u32.+///+/// This type is used to represent a label reference in some+/// MachBuffer metadata (specifically, relocations and+/// exception-handler records). These start as labels before the+/// `MachBuffer` is finalized; once `finish()` is called, they become+/// code offsets.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct LabelOrOffset(u32);-/// Status of a compiled artifact that needs patching before being used.-#[derive(Clone, Debug, PartialEq)]-#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]-pub struct Stencil;+impl LabelOrOffset {+ const LABEL_BIT: u32 = 0x8000_0000;+ const MASK: u32 = !Self::LABEL_BIT;-/// Status of a compiled artifact ready to use.-#[derive(Clone, Debug, PartialEq)]-pub struct Final;+ /// Create a `LabelOrOffset` that refers to a label.+ pub fn label(label: MachLabel) -> Self {+ debug_assert!(label.0 & Self::MASK == label.0);+ LabelOrOffset(label.0 | Self::LABEL_BIT)+ }-impl CompilePhase for Stencil {- type MachSrcLocType = MachSrcLoc<Stencil>;- type SourceLocType = RelSourceLoc;+ /// Create a `LabelOrOffset` that refers to a code offset.+ pub fn offset(offset: CodeOffset) -> Self {+ debug_assert!(offset & Self::MASK == offset);+ LabelOrOffset(offset)+ }++ /// Is this a label?+ pub fn is_label(&self) -> bool {+ self.0 & Self::LABEL_BIT != 0+ }++ /// Is this a code offset?+ pub fn is_offset(&self) -> bool {+ self.0 & Self::LABEL_BIT == 0+ }++ /// Unwrap as a label.+ ///+ /// # Panics+ ///+ /// Panics if this is not a label.+ pub fn as_label(&self) -> MachLabel {+ assert!(self.is_label());+ MachLabel(self.0 & Self::MASK)+ }++ /// Unwrap as a code offset.+ ///+ /// # Panics+ ///+ /// Panics if this is not a code offset.+ pub fn as_offset(&self) -> CodeOffset {+ assert!(self.is_offset());+ self.0 & Self::MASK+ }}-impl CompilePhase for Final {- type MachSrcLocType = MachSrcLoc<Final>;- type SourceLocType = SourceLoc;+impl From<MachLabel> for LabelOrOffset {+ fn from(value: MachLabel) -> Self {+ LabelOrOffset::label(value)+ }}-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-enum ForceVeneers {- Yes,- No,+impl core::fmt::Display for LabelOrOffset {+ fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {+ if self.is_offset() {+ write!(fmt, "0x{:x}", self.as_offset())+ } else {+ write!(fmt, "label{}", self.as_label().0)+ }+ }}/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink@@ -266,35 +313,13 @@ enum ForceVeneers {/// likely fine as long as it is stack-allocated for function emission then/// thrown away; but beware if many buffer objects are retained persistently.pub struct MachBuffer<I: VCodeInst> {- /// The buffer contents, as raw bytes.- data: SmallVec<[u8; 1024]>,- /// The required alignment of this buffer.- min_alignment: u32,- /// Any relocations referring to this code. Note that only *external*- /// relocations are tracked here; references to labels within the buffer are- /// resolved before emission.- relocs: SmallVec<[MachReloc; 16]>,- /// Any trap records referring to this code.- traps: SmallVec<[MachTrap; 16]>,- /// Any call site records referring to this code.- call_sites: SmallVec<[MachCallSite; 16]>,- /// Any patchable call site locations.- patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,- /// Any exception-handler records referred to at call sites.- exception_handlers: SmallVec<[MachExceptionHandler; 16]>,- /// Any source location mappings referring to this code.- srclocs: SmallVec<[MachSrcLoc<Stencil>; 64]>,- /// Any debug tags referring to this code.- debug_tags: Vec<MachDebugTags>,- /// Pool of debug tags referenced by `MachDebugTags` entries.- debug_tag_pool: Vec<DebugTag>,- /// Any user stack maps for this code.- ///- /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted- /// by code offset, and each stack map covers `span` bytes on the stack.- user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,- /// Any unwind info at a given location.- unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,+ // --- machine-code data and metadata:+ //+ /// Data shared between the unfinalized and finalized MachBuffers.+ inner: Box<MachBufferInner>,++ // --- emission-pass state:+ ///// The current source location in progress (after `start_srcloc()` and/// before `end_srcloc()`). This is a (start_offset, src_loc) tuple.cur_srcloc: Option<(CodeOffset, RelSourceLoc)>,@@ -356,62 +381,44 @@ pub struct MachBuffer<I: VCodeInst> {/// Indicates when a patchable region is currently open, to guard that it's/// not possible to nest patchable regions.open_patchable: bool,- /// Stack frame layout metadata. If provided for a MachBuffer- /// containing a function body, this allows interpretation of- /// runtime state given a view of an active stack frame.- frame_layout: Option<MachBufferFrameLayout>,}-impl MachBufferFinalized<Stencil> {- /// Get a finalized machine buffer by applying the function's base source location.- pub fn apply_base_srcloc(self, base_srcloc: SourceLoc) -> MachBufferFinalized<Final> {- MachBufferFinalized {- data: self.data,- relocs: self.relocs,- traps: self.traps,- call_sites: self.call_sites,- patchable_call_sites: self.patchable_call_sites,- exception_handlers: self.exception_handlers,- srclocs: self- .srclocs- .into_iter()- .map(|srcloc| srcloc.apply_base_srcloc(base_srcloc))- .collect(),- debug_tags: self.debug_tags,- debug_tag_pool: self.debug_tag_pool,- user_stack_maps: self.user_stack_maps,- unwind_info: self.unwind_info,- alignment: self.alignment,- frame_layout: self.frame_layout,- nop_units: self.nop_units,- }- }-}--/// A `MachBuffer` once emission is completed: holds generated code and records,-/// without fixups. This allows the type to be independent of the backend.+/// Bulk data that is common between `MachBuffer` and+/// `MachBufferFinalized`.+///+/// The goal is to indirect the large allocations (`SmallVec`s) so+/// that we don't move a lot of memory during compilation.+///+/// The two named types are essentially a builder/final object pair.+/// The inner state is the same (except that the builder has further+/// transient state that is later dropped).+///+/// However, many of the fields are either moved over wholesale or+/// patched then moved over (data). We put these fields in+/// `MachBufferInner`, hold that shared data in a box so that+/// finalization can just move a pointer, and then impl `Deref` on the+/// two `MachBuffer` variants so accesses to these fields are+/// transparent.#[derive(PartialEq, Debug, Clone)]#[cfg_attr(feature = "enable-serde",derive(serde_derive::Serialize, serde_derive::Deserialize))]-pub struct MachBufferFinalized<T: CompilePhase> {+pub struct MachBufferInner {/// The buffer contents, as raw bytes.pub(crate) data: SmallVec<[u8; 1024]>,+ /// Any trap records referring to this code.+ pub(crate) traps: SmallVec<[MachTrap; 16]>,/// Any relocations referring to this code. Note that only *external*/// relocations are tracked here; references to labels within the buffer are/// resolved before emission.- pub(crate) relocs: SmallVec<[FinalizedMachReloc; 16]>,- /// Any trap records referring to this code.- pub(crate) traps: SmallVec<[MachTrap; 16]>,+ pub(crate) relocs: SmallVec<[MachReloc; 16]>,+ /// Any exception-handler records referred to at call sites.+ pub(crate) exception_handlers: SmallVec<[MachExceptionHandler; 16]>,/// Any call site records referring to this code.pub(crate) call_sites: SmallVec<[MachCallSite; 16]>,- /// Any patchable call site locations referring to this code.+ /// Any patchable call site locations.pub(crate) patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,- /// Any exception-handler records referred to at call sites.- pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,- /// Any source location mappings referring to this code.- pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>,/// Any debug tags referring to this code.pub(crate) debug_tags: Vec<MachDebugTags>,/// Pool of debug tags referenced by `MachDebugTags` entries.@@ -421,14 +428,60 @@ pub struct MachBufferFinalized<T: CompilePhase> {/// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted/// by code offset, and each stack map covers `span` bytes on the stack.pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,+ /// Any unwind info at a given location.+ pub(crate) unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,/// Stack frame layout metadata. If provided for a MachBuffer/// containing a function body, this allows interpretation of/// runtime state given a view of an active stack frame.pub(crate) frame_layout: Option<MachBufferFrameLayout>,- /// Any unwind info at a given location.- pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,+ /// Any source location mappings referring to this code.+ pub(crate) srclocs: SmallVec<[MachSrcLoc; 64]>,/// The required alignment of this buffer.- pub alignment: u32,+ pub min_alignment: u32,+}++impl<I: VCodeInst> Deref for MachBuffer<I> {+ type Target = MachBufferInner;+ fn deref(&self) -> &Self::Target {+ &*self.inner+ }+}+impl<I: VCodeInst> DerefMut for MachBuffer<I> {+ fn deref_mut(&mut self) -> &mut Self::Target {+ &mut *self.inner+ }+}+impl Deref for MachBufferFinalized {+ type Target = MachBufferInner;+ fn deref(&self) -> &Self::Target {+ &*self.inner+ }+}+impl DerefMut for MachBufferFinalized {+ fn deref_mut(&mut self) -> &mut Self::Target {+ &mut *self.inner+ }+}++impl MachBufferFinalized {+ /// Get a finalized machine buffer by applying the function's base source location.+ pub fn apply_base_srcloc(&mut self, base_srcloc: SourceLoc) {+ for loc in &mut self.inner.srclocs {+ loc.apply_base_srcloc(base_srcloc);+ }+ }+}++/// A `MachBuffer` once emission is completed: holds generated code and records,+/// without fixups. This allows the type to be independent of the backend.+#[derive(PartialEq, Debug, Clone)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct MachBufferFinalized {+ /// The raw data and finalization-invariant metadata attached to it.+ pub(crate) inner: Box<MachBufferInner>,/// The means by which to NOP out patchable call sites.////// This allows a consumer of a `MachBufferFinalized` to disable@@ -501,19 +554,23 @@ impl<I: VCodeInst> MachBuffer<I> {/// Create a new section, known to start at `start_offset` and with a size limited to/// `length_limit`.pub fn new() -> MachBuffer<I> {- MachBuffer {+ let inner = Box::new(MachBufferInner {data: SmallVec::new(),- min_alignment: I::function_alignment().minimum,- relocs: SmallVec::new(),traps: SmallVec::new(),+ relocs: SmallVec::new(),+ exception_handlers: SmallVec::new(),call_sites: SmallVec::new(),patchable_call_sites: SmallVec::new(),- exception_handlers: SmallVec::new(),- srclocs: SmallVec::new(),debug_tags: vec![],debug_tag_pool: vec![],user_stack_maps: SmallVec::new(),unwind_info: SmallVec::new(),+ frame_layout: None,+ srclocs: SmallVec::new(),+ min_alignment: I::function_alignment().minimum,+ });+ MachBuffer {+ inner,cur_srcloc: None,label_offsets: SmallVec::new(),label_aliases: SmallVec::new(),@@ -529,7 +586,6 @@ impl<I: VCodeInst> MachBuffer<I> {constants: Default::default(),used_constants: Default::default(),open_patchable: false,- frame_layout: None,}}@@ -682,15 +738,13 @@ impl<I: VCodeInst> MachBuffer<I> {/// Returns the alignment required for this entire buffer. Alignment starts/// at the ISA's minimum function alignment and can be increased due to/// constant requirements.- fn finish_constants(&mut self, constants: &VCodeConstants) -> u32 {- let mut alignment = self.min_alignment;+ fn finish_constants(&mut self, constants: &VCodeConstants) {for (constant, offset) in mem::take(&mut self.used_constants) {let constant = constants.get(constant);let data = constant.as_slice();self.data[offset as usize..][..data.len()].copy_from_slice(data);- alignment = constant.alignment().max(alignment);+ self.min_alignment = constant.alignment().max(self.min_alignment);}- alignment}/// Returns a label that can be used to refer to the `constant` provided.@@ -1236,14 +1290,14 @@ impl<I: VCodeInst> MachBuffer<I> {let off_before_edit = self.cur_offset();let prev_b = self.latest_branches.last_mut().unwrap();let not_inverted = SmallVec::from(- &self.data[(prev_b.start as usize)..(prev_b.end as usize)],+ &self.inner.data[(prev_b.start as usize)..(prev_b.end as usize)],);// Low-level edit: replaces bytes of branch with// inverted form. cur_off remains the same afterward, so// we do not need to modify label data structures.- self.data.truncate(prev_b.start as usize);- self.data.extend_from_slice(&data[..]);+ self.inner.data.truncate(prev_b.start as usize);+ self.inner.data.extend_from_slice(&data[..]);// Save the original code as the inversion of the// inverted branch, in case we later edit this branch@@ -1628,55 +1682,30 @@ impl<I: VCodeInst> MachBuffer<I> {mut self,constants: &VCodeConstants,ctrl_plane: &mut ControlPlane,- ) -> MachBufferFinalized<Stencil> {+ ) -> MachBufferFinalized {let _tt = timing::vcode_emit_finish();self.finish_emission_maybe_forcing_veneers(ForceVeneers::No, ctrl_plane);-- let alignment = self.finish_constants(constants);+ self.finish_constants(constants);// Resolve all labels to their offsets.- let finalized_relocs = self- .relocs- .iter()- .map(|reloc| FinalizedMachReloc {- offset: reloc.offset,- kind: reloc.kind,- addend: reloc.addend,- target: match &reloc.target {- RelocTarget::ExternalName(name) => {- FinalizedRelocTarget::ExternalName(name.clone())- }- RelocTarget::Label(label) => {- FinalizedRelocTarget::Func(self.resolve_label_offset(*label))- }- },- })- .collect();-- let finalized_exception_handlers = self- .exception_handlers- .iter()- .map(|handler| handler.finalize(|label| self.resolve_label_offset(label)))- .collect();-- let mut srclocs = self.srclocs;- srclocs.sort_by_key(|entry| entry.start);+ let mut relocs = core::mem::take(&mut self.relocs);+ let mut exception_handlers = core::mem::take(&mut self.exception_handlers);+ let resolve = |label: LabelOrOffset| {+ LabelOrOffset::offset(self.resolve_label_offset(label.as_label()))+ };+ for reloc in &mut relocs {+ reloc.target.map(resolve);+ }+ for handler in &mut exception_handlers {+ handler.map(resolve);+ }+ self.relocs = relocs;+ self.exception_handlers = exception_handlers;+ self.srclocs.sort_by_key(|entry| entry.start);MachBufferFinalized {- data: self.data,- relocs: finalized_relocs,- traps: self.traps,- call_sites: self.call_sites,- patchable_call_sites: self.patchable_call_sites,- exception_handlers: finalized_exception_handlers,- srclocs,- debug_tags: self.debug_tags,- debug_tag_pool: self.debug_tag_pool,- user_stack_maps: self.user_stack_maps,- unwind_info: self.unwind_info,- alignment,- frame_layout: self.frame_layout,+ inner: self.inner,nop_units: I::gen_nop_units(),}}@@ -1743,8 +1772,8 @@ impl<I: VCodeInst> MachBuffer<I> {/// Add a trap record at the current offset.pub fn add_trap(&mut self, code: TrapCode) {- self.traps.push(MachTrap {- offset: self.data.len() as CodeOffset,+ self.inner.traps.push(MachTrap {+ offset: self.inner.data.len() as CodeOffset,code,});}@@ -1766,8 +1795,8 @@ impl<I: VCodeInst> MachBuffer<I> {let end = u32::try_from(self.exception_handlers.len()).unwrap();let exception_handler_range = start..end;- self.call_sites.push(MachCallSite {- ret_addr: self.data.len() as CodeOffset,+ self.inner.call_sites.push(MachCallSite {+ ret_addr: self.inner.data.len() as CodeOffset,frame_offset,exception_handler_range,});@@ -1778,7 +1807,7 @@ impl<I: VCodeInst> MachBuffer<I> {/// specifies how to NOP it out, and we carry that information to/// the finalized Machbuffer.pub fn add_patchable_call_site(&mut self, len: u32) {- self.patchable_call_sites.push(MachPatchableCallSite {+ self.inner.patchable_call_sites.push(MachPatchableCallSite {ret_addr: self.cur_offset(),len,});@@ -1786,7 +1815,7 @@ impl<I: VCodeInst> MachBuffer<I> {/// Add an unwind record at the current offset.pub fn add_unwind(&mut self, unwind: UnwindInst) {- self.unwind_info.push((self.cur_offset(), unwind));+ self.inner.unwind_info.push((self.cur_offset(), unwind));}/// Set the `SourceLoc` for code from this offset until the offset at the@@ -1809,7 +1838,11 @@ impl<I: VCodeInst> MachBuffer<I> {// Skip zero-length extends.debug_assert!(end >= start);if end > start {- self.srclocs.push(MachSrcLoc { start, end, loc });+ self.srclocs.push(MachSrcLoc {+ start,+ end,+ loc: MaybeRelSourceLoc::rel(loc),+ });}}@@ -1867,7 +1900,7 @@ impl<I: VCodeInst> MachBuffer<I> {let start = u32::try_from(self.debug_tag_pool.len()).unwrap();self.debug_tag_pool.extend(tags.iter().cloned());let end = u32::try_from(self.debug_tag_pool.len()).unwrap();- self.debug_tags.push(MachDebugTags {+ self.inner.debug_tags.push(MachDebugTags {offset: self.cur_offset(),pos,range: start..end,@@ -1898,9 +1931,9 @@ impl<I: VCodeInst> Extend<u8> for MachBuffer<I> {}}-impl<T: CompilePhase> MachBufferFinalized<T> {+impl MachBufferFinalized {/// Get a list of source location mapping tuples in sorted-by-start-offset order.- pub fn get_srclocs_sorted(&self) -> &[T::MachSrcLocType] {+ pub fn get_srclocs_sorted(&self) -> &[MachSrcLoc] {&self.srclocs[..]}@@ -1956,7 +1989,7 @@ impl<T: CompilePhase> MachBufferFinalized<T> {}/// Get the list of external relocations for this code.- pub fn relocs(&self) -> &[FinalizedMachReloc] {+ pub fn relocs(&self) -> &[MachReloc] {&self.relocs[..]}@@ -1986,12 +2019,12 @@ impl<T: CompilePhase> MachBufferFinalized<T> {/// - The slice of pairs of exception tags and code offsets/// denoting exception-handler entry points associated with this/// call site.- pub fn call_sites(&self) -> impl Iterator<Item = FinalizedMachCallSite<'_>> + '_ {+ pub fn call_sites(&self) -> impl Iterator<Item = MachCallSiteItem<'_>> + '_ {self.call_sites.iter().map(|call_site| {let handler_range = call_site.exception_handler_range.clone();let handler_range = usize::try_from(handler_range.start).unwrap()..usize::try_from(handler_range.end).unwrap();- FinalizedMachCallSite {+ MachCallSiteItem {ret_addr: call_site.ret_addr,frame_offset: call_site.frame_offset,exception_handlers: &self.exception_handlers[handler_range],@@ -2022,13 +2055,17 @@ impl<T: CompilePhase> MachBufferFinalized<T> {/// references. Items are interpreted in left-to-right order and the/// first match wins.#[derive(Clone, Copy, Debug, PartialEq, Eq)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]pub enum MachExceptionHandler {/// A specific tag (in the current dynamic context) should be/// handled by the code at the given offset.- Tag(ExceptionTag, MachLabel),+ Tag(ExceptionTag, LabelOrOffset),/// All exceptions should be handled by the code at the given/// offset.- Default(MachLabel),+ Default(LabelOrOffset),/// The dynamic context for interpreting tags is updated to the/// value stored in the given machine location (in this frame's/// context).@@ -2036,36 +2073,19 @@ pub enum MachExceptionHandler {}impl MachExceptionHandler {- fn finalize<F: Fn(MachLabel) -> CodeOffset>(self, f: F) -> FinalizedMachExceptionHandler {+ fn map<F: Fn(LabelOrOffset) -> LabelOrOffset>(&mut self, f: F) {match self {- Self::Tag(tag, label) => FinalizedMachExceptionHandler::Tag(tag, f(label)),- Self::Default(label) => FinalizedMachExceptionHandler::Default(f(label)),- Self::Context(loc) => FinalizedMachExceptionHandler::Context(loc),+ Self::Tag(_, label) => {+ *label = f(*label);+ }+ Self::Default(label) => {+ *label = f(*label);+ }+ Self::Context(_loc) => {}}}}-/// An item in the exception-handler list for a callsite, with final-/// (lowered) code offsets. Items are interpreted in left-to-right-/// order and the first match wins.-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-#[cfg_attr(- feature = "enable-serde",- derive(serde_derive::Serialize, serde_derive::Deserialize)-)]-pub enum FinalizedMachExceptionHandler {- /// A specific tag (in the current dynamic context) should be- /// handled by the code at the given offset.- Tag(ExceptionTag, CodeOffset),- /// All exceptions should be handled by the code at the given- /// offset.- Default(CodeOffset),- /// The dynamic context for interpreting tags is updated to the- /// value stored in the given machine location (in this frame's- /// context).- Context(ExceptionContextLoc),-}-/// A location for a dynamic exception context value.#[derive(Clone, Copy, Debug, PartialEq, Eq)]#[cfg_attr(@@ -2154,34 +2174,35 @@ impl<I: VCodeInst> Ord for MachLabelFixup<I> {feature = "enable-serde",derive(serde_derive::Serialize, serde_derive::Deserialize))]-pub struct MachRelocBase<T> {+pub struct MachReloc {/// The offset at which the relocation applies, *relative to the/// containing section*.pub offset: CodeOffset,/// The kind of relocation.pub kind: Reloc,/// The external symbol / name to which this relocation refers.- pub target: T,+ pub target: RelocTarget,/// The addend to add to the symbol value.pub addend: i64,}-type MachReloc = MachRelocBase<RelocTarget>;--/// A relocation resulting from a compilation.-pub type FinalizedMachReloc = MachRelocBase<FinalizedRelocTarget>;-/// A Relocation target#[derive(Debug, Clone, PartialEq, Eq, Hash)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]pub enum RelocTarget {/// Points to an [ExternalName] outside the current function.ExternalName(ExternalName),- /// Points to a [MachLabel] inside this function.- /// This is different from [MachLabelFixup] in that both the relocation and the- /// label will be emitted and are only resolved at link time.+ /// Points to a [MachLabel] inside this function. This is different+ /// from an internal fixup/label reference in that both the+ /// relocation and the label will be emitted and are only resolved+ /// at link time.///- /// There is no reason to prefer this over [MachLabelFixup] unless the ABI requires it.- Label(MachLabel),+ /// There is no reason to prefer this over internal fixups unless+ /// the ABI requires it.+ Label(LabelOrOffset),}impl From<ExternalName> for RelocTarget {@@ -2192,30 +2213,26 @@ impl From<ExternalName> for RelocTarget {impl From<MachLabel> for RelocTarget {fn from(label: MachLabel) -> Self {- Self::Label(label)+ Self::Label(LabelOrOffset::label(label))}}-/// A Relocation target-#[derive(Debug, Clone, PartialEq, Eq, Hash)]-#[cfg_attr(- feature = "enable-serde",- derive(serde_derive::Serialize, serde_derive::Deserialize)-)]-pub enum FinalizedRelocTarget {- /// Points to an [ExternalName] outside the current function.- ExternalName(ExternalName),- /// Points to a [CodeOffset] from the start of the current function.- Func(CodeOffset),-}--impl FinalizedRelocTarget {- /// Returns a display for the current [FinalizedRelocTarget], with extra context to prettify the+impl RelocTarget {+ /// Returns a display for the current [RelocTarget], with extra context to prettify the/// output.pub fn display<'a>(&'a self, params: Option<&'a FunctionParameters>) -> String {match self {- FinalizedRelocTarget::ExternalName(name) => format!("{}", name.display(params)),- FinalizedRelocTarget::Func(offset) => format!("func+{offset}"),+ RelocTarget::ExternalName(name) => format!("{}", name.display(params)),+ RelocTarget::Label(offset) => format!("func+{offset}"),+ }+ }++ fn map<F: Fn(LabelOrOffset) -> LabelOrOffset>(&mut self, f: F) {+ match self {+ RelocTarget::ExternalName(_) => {}+ RelocTarget::Label(label) => {+ *label = f(*label);+ }}}}@@ -2263,29 +2280,22 @@ pub struct MachCallSite {exception_handler_range: Range<u32>,}-/// A call site record resulting from a compilation.+/// A view onto a call site record resulting from a compilation,+/// returned during iteration.#[derive(Clone, Debug, PartialEq)]-pub struct FinalizedMachCallSite<'a> {+pub struct MachCallSiteItem<'a> {/// The offset of the call's return address, *relative to the/// start of the buffer*.pub ret_addr: CodeOffset,/// The offset from the FP at this callsite down to the SP when- /// the call occurs, if known. In other words, the size of the- /// stack frame up to the saved FP slot. Useful to recover the- /// start of the stack frame and to look up dynamic contexts- /// stored in [`ExceptionContextLoc::SPOffset`].+ /// the call occurs, if known.///- /// If `None`, the compiler backend did not specify a frame- /// offset. The runtime in use with the compiled code may require- /// the frame offset if exception handlers are present or dynamic- /// context is used, but that is not Cranelift's concern: the- /// frame offset is optional at this level.+ /// See [`MachCallSite::frame_offset`] for more.pub frame_offset: Option<u32>,- /// Exception handlers at this callsite, with target offsets- /// *relative to the start of the buffer*.- pub exception_handlers: &'a [FinalizedMachExceptionHandler],+ /// Exception handlers at this site.+ pub exception_handlers: &'a [MachExceptionHandler],}/// A patchable call site record resulting from a compilation.@@ -2310,7 +2320,7 @@ pub struct MachPatchableCallSite {feature = "enable-serde",derive(serde_derive::Serialize, serde_derive::Deserialize))]-pub struct MachSrcLoc<T: CompilePhase> {+pub struct MachSrcLoc {/// The start of the region of code corresponding to a source location./// This is relative to the start of the function, not to the start of the/// section.@@ -2320,16 +2330,12 @@ pub struct MachSrcLoc<T: CompilePhase> {/// section.pub end: CodeOffset,/// The source location.- pub loc: T::SourceLocType,+ pub loc: MaybeRelSourceLoc,}-impl MachSrcLoc<Stencil> {- fn apply_base_srcloc(self, base_srcloc: SourceLoc) -> MachSrcLoc<Final> {- MachSrcLoc {- start: self.start,- end: self.end,- loc: self.loc.expand(base_srcloc),- }+impl MachSrcLoc {+ fn apply_base_srcloc(&mut self, base_srcloc: SourceLoc) {+ self.loc = MaybeRelSourceLoc::abs(self.loc.relocate(base_srcloc));}}@@ -2968,8 +2974,8 @@ mod test {buf.add_try_call_site(Some(0x10),[- MachExceptionHandler::Tag(ExceptionTag::new(42), label(2)),- MachExceptionHandler::Default(label(1)),+ MachExceptionHandler::Tag(ExceptionTag::new(42), label(2).into()),+ MachExceptionHandler::Default(label(1).into()),].into_iter(),);@@ -3010,8 +3016,8 @@ mod test {assert_eq!(call_sites[0].exception_handlers,&[- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(42), 5),- FinalizedMachExceptionHandler::Default(4)+ MachExceptionHandler::Tag(ExceptionTag::new(42), LabelOrOffset::offset(5)),+ MachExceptionHandler::Default(LabelOrOffset::offset(4))],);assert_eq!(diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rsindex 5b9b16a94a..42fab7cd8c 100644--- a/cranelift/codegen/src/machinst/mod.rs+++ b/cranelift/codegen/src/machinst/mod.rs@@ -580,9 +580,9 @@ pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {/// code (as bytes) and a disassembly, if requested.#[derive(PartialEq, Debug, Clone)]#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]-pub struct CompiledCodeBase<T: CompilePhase> {+pub struct CompiledCode {/// Machine code.- pub buffer: MachBufferFinalized<T>,+ pub buffer: MachBufferFinalized,/// Disassembly, if requested.pub vcode: Option<String>,/// Debug info: value labels to registers/stackslots at code offsets.@@ -601,20 +601,22 @@ pub struct CompiledCodeBase<T: CompilePhase> {pub bb_edges: Vec<(CodeOffset, CodeOffset)>,}+/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.+///+/// Only used internally, in a transient manner, for the incremental compilation cache.+#[derive(PartialEq, Debug, Clone)]+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]+pub struct CompiledCodeStencil(pub(crate) CompiledCode);+impl CompiledCodeStencil {/// Apply function parameters to finalize a stencil into its final form.- pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode {- CompiledCode {- buffer: self.buffer.apply_base_srcloc(params.base_srcloc()),- vcode: self.vcode,- value_labels_ranges: self.value_labels_ranges,- bb_starts: self.bb_starts,- bb_edges: self.bb_edges,- }+ pub fn apply_params(mut self, params: &FunctionParameters) -> CompiledCode {+ self.0.buffer.apply_base_srcloc(params.base_srcloc());+ self.0}}-impl<T: CompilePhase> CompiledCodeBase<T> {+impl CompiledCode {/// Get a `CodeInfo` describing section sizes from this compilation result.pub fn code_info(&self) -> CodeInfo {CodeInfo {@@ -749,18 +751,7 @@ impl<T: CompilePhase> CompiledCodeBase<T> {anyhow::format_err!("{err}")}}-}-/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.-///-/// Only used internally, in a transient manner, for the incremental compilation cache.-pub type CompiledCodeStencil = CompiledCodeBase<Stencil>;--/// `CompiledCode` in its final form (i.e. after `FunctionParameters` have been applied), ready for-/// consumption.-pub type CompiledCode = CompiledCodeBase<Final>;--impl CompiledCode {/// If available, return information about the code layout in the/// final machine code: the offsets (in bytes) of each basic-block/// start, and all basic-block edges.diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rsindex 87a4d05871..12af7f434b 100644--- a/cranelift/codegen/src/machinst/vcode.rs+++ b/cranelift/codegen/src/machinst/vcode.rs@@ -211,7 +211,7 @@ pub struct VCode<I: VCodeInst> {/// and optionally metadata about the code layout.pub struct EmitResult {/// The MachBuffer containing the machine code.- pub buffer: MachBufferFinalized<Stencil>,+ pub buffer: MachBufferFinalized,/// Offset of each basic block, recorded during emission. Computed/// only if `machine_code_cfg_info` is enabled.diff --git a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clifindex cee91ef108..9a7cbcb47f 100644--- a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif+++ b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif@@ -36,7 +36,7 @@ block0(v0: i64):; Disassembled:; block0: ; offset 0x0; auipc a1, 0 ; reloc_external RiscvPCRelHi20 %sym 0-; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0x0 0; fence rw, w; sd a0, 0(a1) ; trap: heap_oob; ret@@ -97,7 +97,7 @@ block0(v0: i32):; Disassembled:; block0: ; offset 0x0; auipc a1, 0 ; reloc_external RiscvPCRelHi20 %sym 0-; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0x0 0; fence rw, w; sw a0, 0(a1) ; trap: heap_oob; retdiff --git a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clifindex 5feaa4ce33..56885db4f9 100644--- a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif+++ b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif@@ -84,7 +84,7 @@ block0(v0: i64):; mv s0, sp; block1: ; offset 0x10; auipc t0, 0 ; reloc_external RiscvPCRelHi20 %callee_i64 0-; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+16 0+; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+0x10 0; ld ra, 8(sp); ld s0, 0(sp); addi sp, sp, 0x10diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clifindex 1e4d60a141..2c0a8d8f1e 100644--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif@@ -19,6 +19,6 @@ block0:; Disassembled:; block0: ; offset 0x0; auipc a0, 0 ; reloc_external RiscvGotHi20 %my_global 0-; ld a0, 0(a0) ; reloc_external RiscvPCRelLo12I func+0 0+; ld a0, 0(a0) ; reloc_external RiscvPCRelLo12I func+0x0 0; retdiff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clifindex 97d1594805..1093810f0f 100644--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif@@ -40,7 +40,7 @@ block0:; Disassembled:; block0: ; offset 0x0; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %func0 0-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0; retfunction %symbol_value() -> i64 {@@ -125,7 +125,7 @@ block0:; Disassembled:; block0: ; offset 0x0; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 0-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0; retfunction %colocated_symbol_value_plus_offset() -> i64 {@@ -144,7 +144,7 @@ block0:; Disassembled:; block0: ; offset 0x0; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 123-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0; retfunction %colocated_symbol_value_minus_offset() -> i64 {@@ -163,6 +163,6 @@ block0:; Disassembled:; block0: ; offset 0x0; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 -123-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0; retdiff --git a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clifindex b33b79c160..d59e0ce388 100644--- a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif+++ b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif@@ -40,7 +40,7 @@ block0(v0: i32):; block1: ; offset 0x18; mv s1, a0; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+28 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x1c 0; auipc ra, 0 ; reloc_external RiscvCallPlt %ElfTlsGetAddr 0; jalr ra; mv a1, a0@@ -85,7 +85,7 @@ block0(v0: i64):; mv s0, sp; block1: ; offset 0x10; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+16 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x10 0; auipc ra, 0 ; reloc_external RiscvCallPlt %ElfTlsGetAddr 0; jalr ra; sb zero, 0(a0)diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rsindex 7d50bebbe9..31a3b1e059 100644--- a/cranelift/jit/src/backend.rs+++ b/cranelift/jit/src/backend.rs@@ -484,7 +484,7 @@ impl Module for JITModule {// work around borrow-checker to allow reuse of ctx belowlet res = ctx.compile(self.isa(), ctrl_plane)?;- let alignment = res.buffer.alignment as u64;+ let alignment = res.buffer.min_alignment as u64;let compiled_code = ctx.compiled_code().unwrap();let align = alignmentdiff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rsindex 76972d2129..2afa4424ef 100644--- a/cranelift/module/src/module.rs+++ b/cranelift/module/src/module.rs@@ -13,9 +13,7 @@ use cranelift_codegen::entity::{PrimaryMap, entity_impl};use cranelift_codegen::ir::ExternalName;use cranelift_codegen::ir::function::{Function, VersionMarker};use cranelift_codegen::settings::SetError;-use cranelift_codegen::{- CodegenError, CompileError, Context, FinalizedMachReloc, FinalizedRelocTarget, ir, isa,-};+use cranelift_codegen::{CodegenError, CompileError, Context, MachReloc, RelocTarget, ir, isa};use cranelift_control::ControlPlane;use std::borrow::{Cow, ToOwned};use std::boxed::Box;@@ -36,25 +34,22 @@ pub struct ModuleReloc {}impl ModuleReloc {- /// Converts a `FinalizedMachReloc` produced from a `Function` into a `ModuleReloc`.- pub fn from_mach_reloc(- mach_reloc: &FinalizedMachReloc,- func: &Function,- func_id: FuncId,- ) -> Self {+ /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`.+ pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function, func_id: FuncId) -> Self {let name = match mach_reloc.target {- FinalizedRelocTarget::ExternalName(ExternalName::User(reff)) => {+ RelocTarget::ExternalName(ExternalName::User(reff)) => {let name = &func.params.user_named_funcs()[reff];ModuleRelocTarget::user(name.namespace, name.index)}- FinalizedRelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(),- FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {+ RelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(),+ RelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {ModuleRelocTarget::LibCall(libcall)}- FinalizedRelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => {+ RelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => {ModuleRelocTarget::KnownSymbol(ks)}- FinalizedRelocTarget::Func(offset) => {+ RelocTarget::Label(label) => {+ let offset = label.as_offset();ModuleRelocTarget::FunctionOffset(func_id, offset)}};diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rsindex 2aa58c1c63..71b0cf31e0 100644--- a/cranelift/object/src/backend.rs+++ b/cranelift/object/src/backend.rs@@ -448,7 +448,7 @@ impl Module for ObjectModule {info!("defining function {}: {}", func_id, ctx.func.display());let res = ctx.compile(self.isa(), ctrl_plane)?;- let alignment = res.buffer.alignment as u64;+ let alignment = res.buffer.min_alignment as u64;let compiled = ctx.compiled_code().unwrap();#[cfg(feature = "unwind")]diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rsindex cfd635c9ab..adee07b0bc 100644--- a/cranelift/src/disasm.rs+++ b/cranelift/src/disasm.rs@@ -2,12 +2,12 @@ use anyhow::Result;use cranelift_codegen::ir::Function;use cranelift_codegen::ir::function::FunctionParameters;use cranelift_codegen::isa::TargetIsa;-use cranelift_codegen::{FinalizedMachReloc, MachTrap};+use cranelift_codegen::{MachReloc, MachTrap};use std::fmt::Write;-fn print_relocs(func_params: &FunctionParameters, relocs: &[FinalizedMachReloc]) -> String {+fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String {let mut text = String::new();- for &FinalizedMachReloc {+ for &MachReloc {kind,offset,ref target,@@ -103,7 +103,7 @@ pub fn print_all(mem: &[u8],code_size: u32,print: bool,- relocs: &[FinalizedMachReloc],+ relocs: &[MachReloc],traps: &[MachTrap],) -> Result<()> {print_bytes(&mem);diff --git a/crates/cranelift/src/compiled_function.rs b/crates/cranelift/src/compiled_function.rsindex 6a6c351c27..5672bb9b58 100644--- a/crates/cranelift/src/compiled_function.rs+++ b/crates/cranelift/src/compiled_function.rs@@ -2,7 +2,7 @@ use std::ops::Range;use crate::{Relocation, mach_reloc_to_reloc, mach_trap_to_trap};use cranelift_codegen::{- Final, MachBufferFinalized, MachBufferFrameLayout, MachSrcLoc, ValueLabelsRanges, ir,+ MachBufferFinalized, MachBufferFrameLayout, MachSrcLoc, ValueLabelsRanges, ir,isa::unwind::CfaUnwindInfo, isa::unwind::UnwindInfo,};use wasmtime_environ::{@@ -58,7 +58,7 @@ pub struct CompiledFunctionMetadata {/// Compiled function: machine code body, jump table offsets, and unwind information.pub struct CompiledFunction {/// The machine code buffer for this function.- pub buffer: MachBufferFinalized<Final>,+ pub buffer: MachBufferFinalized,/// What names each name ref corresponds to.name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,/// The alignment for the compiled function.@@ -77,7 +77,7 @@ impl CompiledFunction {/// This function uses the information in the machine buffer to derive the traps and relocations/// fields. The compiled function metadata is loaded with the default values.pub fn new(- buffer: MachBufferFinalized<Final>,+ buffer: MachBufferFinalized,name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,alignment: u32,) -> Self {@@ -169,7 +169,7 @@ impl CompiledFunction {.buffer.get_srclocs_sorted().into_iter()- .map(|&MachSrcLoc { start, end, loc }| (loc, start, (end - start)));+ .map(|&MachSrcLoc { start, end, loc }| (loc.as_abs(), start, (end - start)));let instructions = if with_instruction_addresses {collect_address_maps(len.try_into().unwrap(), srclocs)} else {diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 980a498b7d..f540f1b860 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -19,7 +19,7 @@ use cranelift_codegen::isa::{};use cranelift_codegen::print_errors::pretty_error;use cranelift_codegen::{- CompiledCode, Context, FinalizedMachCallSite, MachBufferDebugTagList, MachBufferFrameLayout,+ CompiledCode, Context, MachBufferDebugTagList, MachBufferFrameLayout, MachCallSiteItem,MachDebugTagPos,};use cranelift_entity::PrimaryMap;@@ -1695,7 +1695,7 @@ impl FunctionCompiler<'_> {1};- let alignment = compiled_code.buffer.alignment.max(preferred_alignment);+ let alignment = compiled_code.buffer.min_alignment.max(preferred_alignment);let mut compiled_function = CompiledFunction::new(compiled_code.buffer.clone(),context.func.params.user_named_funcs().clone(),@@ -1787,7 +1787,7 @@ fn clif_to_env_stack_maps(fn clif_to_env_exception_tables<'a>(builder: &mut ExceptionTableBuilder,range: Range<u64>,- call_sites: impl Iterator<Item = FinalizedMachCallSite<'a>>,+ call_sites: impl Iterator<Item = MachCallSiteItem<'a>>,) -> Result<()> {builder.add_func(CodeOffset::try_from(range.start).unwrap(), call_sites)}diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rsindex fabf37bdec..bb5c2c5594 100644--- a/crates/cranelift/src/lib.rs+++ b/crates/cranelift/src/lib.rs@@ -15,7 +15,7 @@#![warn(clippy::cast_possible_truncation, clippy::cast_sign_loss)]use cranelift_codegen::{- FinalizedMachReloc, FinalizedRelocTarget, MachTrap, binemit,+ MachReloc, MachTrap, RelocTarget, binemit,cursor::FuncCursor,ir::{self, AbiParam, ArgumentPurpose, ExternalName, InstBuilder, Signature, TrapCode},isa::{CallConv, TargetIsa},@@ -307,21 +307,21 @@ fn clif_trap_to_env_trap(trap: ir::TrapCode, tunables: &Tunables) -> Option<Comp/// Converts machine relocations to relocation information/// to perform.fn mach_reloc_to_reloc(- reloc: &FinalizedMachReloc,+ reloc: &MachReloc,name_map: &PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,) -> Relocation {- let &FinalizedMachReloc {+ let &MachReloc {offset,kind,ref target,addend,} = reloc;let reloc_target = match *target {- FinalizedRelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {+ RelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {let name = &name_map[user_func_ref];FuncKey::from_raw_parts(name.namespace, name.index)}- FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {+ RelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {// We should have avoided any code that needs this style of libcalls// in the Wasm-to-Cranelift translator.panic!("unexpected libcall {libcall:?}");diff --git a/crates/unwinder/src/exception_table.rs b/crates/unwinder/src/exception_table.rsindex 2f10f7bb05..93475ea4f3 100644--- a/crates/unwinder/src/exception_table.rs+++ b/crates/unwinder/src/exception_table.rs@@ -17,7 +17,7 @@ use alloc::vec;use alloc::vec::Vec;#[cfg(feature = "cranelift")]use cranelift_codegen::{- ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, binemit::CodeOffset,+ ExceptionContextLoc, MachCallSiteItem, MachExceptionHandler, binemit::CodeOffset,};use wasmtime_environ::prelude::*;@@ -127,7 +127,7 @@ impl ExceptionTableBuilder {pub fn add_func<'a>(&mut self,start_offset: CodeOffset,- call_sites: impl Iterator<Item = FinalizedMachCallSite<'a>>,+ call_sites: impl Iterator<Item = MachCallSiteItem<'a>>,) -> Result<()> {// Ensure that we see functions in offset order.assert!(start_offset >= self.last_start_offset);@@ -144,7 +144,8 @@ impl ExceptionTableBuilder {let mut context = u32::MAX;for handler in call_site.exception_handlers {match handler {- FinalizedMachExceptionHandler::Tag(tag, offset) => {+ MachExceptionHandler::Tag(tag, label) => {+ let offset = label.as_offset();self.tags.push(U32::new(LittleEndian, tag.as_u32()));self.contexts.push(U32::new(LittleEndian, context));self.handlers.push(U32::new(@@ -152,7 +153,8 @@ impl ExceptionTableBuilder {offset.checked_add(start_offset).unwrap(),));}- FinalizedMachExceptionHandler::Default(offset) => {+ MachExceptionHandler::Default(label) => {+ let offset = label.as_offset();self.tags.push(U32::new(LittleEndian, u32::MAX));self.contexts.push(U32::new(LittleEndian, context));self.handlers.push(U32::new(@@ -160,12 +162,10 @@ impl ExceptionTableBuilder {offset.checked_add(start_offset).unwrap(),));}- FinalizedMachExceptionHandler::Context(ExceptionContextLoc::SPOffset(- offset,- )) => {+ MachExceptionHandler::Context(ExceptionContextLoc::SPOffset(offset)) => {context = *offset;}- FinalizedMachExceptionHandler::Context(ExceptionContextLoc::GPR(_)) => {+ MachExceptionHandler::Context(ExceptionContextLoc::GPR(_)) => {panic!("Wasmtime exception unwind info only supports dynamic contexts on the stack");@@ -235,9 +235,8 @@ pub struct ExceptionTable<'a> {/// Wasmtime exception table item, after parsing.////// Note that this is separately defined from the equivalent type in-/// Cranelift, `cranelift_codegen::FinalizedMachExceptionHandler`,-/// because we need this in runtime-only builds when Cranelift is not-/// included.+/// Cranelift, `cranelift_codegen::MachExceptionHandler`, because we+/// need this in runtime-only builds when Cranelift is not included.#[derive(Clone, Debug, PartialEq, Eq)]pub struct ExceptionHandler {/// A tag (arbitrary `u32` identifier from CLIF) or `None` for catch-all.@@ -388,30 +387,31 @@ fn option_from_u32(value: u32) -> Option<u32> {#[cfg(all(test, feature = "cranelift"))]mod test {use super::*;+ use cranelift_codegen::LabelOrOffset;use cranelift_codegen::entity::EntityRef;use cranelift_codegen::ir::ExceptionTag;#[test]fn serialize_exception_table() {let callsites = [- FinalizedMachCallSite {+ MachCallSiteItem {ret_addr: 0x10,frame_offset: None,exception_handlers: &[- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(1), 0x20),- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(2), 0x30),- FinalizedMachExceptionHandler::Default(0x40),+ MachExceptionHandler::Tag(ExceptionTag::new(1), LabelOrOffset::offset(0x20)),+ MachExceptionHandler::Tag(ExceptionTag::new(2), LabelOrOffset::offset(0x30)),+ MachExceptionHandler::Default(LabelOrOffset::offset(0x40)),],},- FinalizedMachCallSite {+ MachCallSiteItem {ret_addr: 0x48,frame_offset: None,exception_handlers: &[],},- FinalizedMachCallSite {+ MachCallSiteItem {ret_addr: 0x50,frame_offset: Some(0x20),- exception_handlers: &[FinalizedMachExceptionHandler::Default(0x60)],+ exception_handlers: &[MachExceptionHandler::Default(LabelOrOffset::offset(0x60))],},];diff --git a/winch/codegen/src/codegen/exceptions.rs b/winch/codegen/src/codegen/exceptions.rsindex 1722abf1f8..3832c9bfa0 100644--- a/winch/codegen/src/codegen/exceptions.rs+++ b/winch/codegen/src/codegen/exceptions.rs@@ -64,8 +64,8 @@ impl HandlerState {.copied().rev().map(|(tag, label)| match tag {- Some(tag) => MachExceptionHandler::Tag(tag, label),- None => MachExceptionHandler::Default(label),+ Some(tag) => MachExceptionHandler::Tag(tag, label.into()),+ None => MachExceptionHandler::Default(label.into()),})}diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rsindex 3c7d57fd50..6253bb0c19 100644--- a/winch/codegen/src/isa/aarch64/asm.rs+++ b/winch/codegen/src/isa/aarch64/asm.rs@@ -19,7 +19,7 @@ use cranelift_codegen::isa::aarch64::inst::{VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp,};use cranelift_codegen::{- Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,+ MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,Writable,ir::{ExternalName, MemFlagsData, SourceLoc, TrapCode, UserExternalNameRef},isa::aarch64::inst::{@@ -126,11 +126,12 @@ impl Assembler {impl Assembler {/// Return the emitted code.- pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> {- let stencil = self+ pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized {+ let mut buffer = self.buffer.finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut());- stencil.apply_base_srcloc(loc.unwrap_or_default())+ buffer.apply_base_srcloc(loc.unwrap_or_default());+ buffer}fn emit(&mut self, inst: Inst) {diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 08b89edd56..7d5ac568d2 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -29,7 +29,7 @@ use crate::{stack::{TypedReg, Val},};use cranelift_codegen::{- ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,+ ExceptionContextLoc, MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{MemFlagsData, RelSourceLoc, SourceLoc, types},isa::aarch64::{@@ -573,7 +573,7 @@ impl Masm for MacroAssembler {Ok(SPOffset::from_u32(self.sp_offset))}- fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> {+ fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized> {if let Some(patch) = self.stack_max_use_add {patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut());}diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rsindex d40a7a6325..750a8c59df 100644--- a/winch/codegen/src/isa/aarch64/mod.rs+++ b/winch/codegen/src/isa/aarch64/mod.rs@@ -10,7 +10,7 @@ use crate::{stack::Stack,};use cranelift_codegen::settings::{self, Flags};-use cranelift_codegen::{Final, MachBufferFinalized, isa::aarch64::settings as aarch64_settings};+use cranelift_codegen::{MachBufferFinalized, isa::aarch64::settings as aarch64_settings};use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};use masm::MacroAssembler as Aarch64Masm;use target_lexicon::Triple;@@ -147,7 +147,7 @@ impl TargetIsa for Aarch64 {fn emit_unwind_info(&self,- _result: &MachBufferFinalized<Final>,+ _result: &MachBufferFinalized,_kind: cranelift_codegen::isa::unwind::UnwindInfoKind,) -> Result<Option<cranelift_codegen::isa::unwind::UnwindInfo>> {// TODO: should fill this in with an actual implementationdiff --git a/winch/codegen/src/isa/mod.rs b/winch/codegen/src/isa/mod.rsindex 21301fb232..4ed9204c2d 100644--- a/winch/codegen/src/isa/mod.rs+++ b/winch/codegen/src/isa/mod.rs@@ -3,7 +3,7 @@ use core::fmt::Formatter;use cranelift_codegen::isa::unwind::{UnwindInfo, UnwindInfoKind};use cranelift_codegen::isa::{CallConv, IsaBuilder};use cranelift_codegen::settings;-use cranelift_codegen::{Final, MachBufferFinalized, TextSectionBuilder};+use cranelift_codegen::{MachBufferFinalized, TextSectionBuilder};use std::{error,fmt::{self, Debug, Display},@@ -199,7 +199,7 @@ pub trait TargetIsa: Send + Sync {fn emit_unwind_info(&self,- _result: &MachBufferFinalized<Final>,+ _result: &MachBufferFinalized,_kind: UnwindInfoKind,) -> Result<Option<UnwindInfo>>;diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rsindex 5f31eaba2f..094c0f1d41 100644--- a/winch/codegen/src/isa/x64/asm.rs+++ b/winch/codegen/src/isa/x64/asm.rs@@ -10,7 +10,7 @@ use crate::{reg::writable,};use cranelift_codegen::{- CallInfo, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState,+ CallInfo, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState,MachLabel, PatchRegion, Writable,ir::{ExternalName, MemFlagsData, SourceLoc, TrapCode, Type, UserExternalNameRef, types},isa::{@@ -327,11 +327,12 @@ impl Assembler {}/// Return the emitted code.- pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> {- let stencil = self+ pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized {+ let mut buffer = self.buffer.finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut());- stencil.apply_base_srcloc(loc.unwrap_or_default())+ buffer.apply_base_srcloc(loc.unwrap_or_default());+ buffer}fn emit(&mut self, inst: Inst) {diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex a1c9c456dd..f708ea7bad 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -34,7 +34,7 @@ use crate::{masm::CalleeKind,};use cranelift_codegen::{- ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,+ ExceptionContextLoc, MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{MemFlagsData, RelSourceLoc, SourceLoc},isa::{@@ -955,7 +955,7 @@ impl Masm for MacroAssembler {Ok(())}- fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> {+ fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized> {if let Some(patch) = self.stack_max_use_add {patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut());}diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rsindex 3463724635..20071d8911 100644--- a/winch/codegen/src/isa/x64/mod.rs+++ b/winch/codegen/src/isa/x64/mod.rs@@ -10,7 +10,7 @@ use crate::{codegen::{BuiltinFunctions, CodeGen, CodeGenContext, FuncEnv, TypeConverter},};use cranelift_codegen::settings::{self, Flags};-use cranelift_codegen::{Final, MachBufferFinalized, isa::x64::settings as x64_settings};+use cranelift_codegen::{MachBufferFinalized, isa::x64::settings as x64_settings};use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};use target_lexicon::Triple;use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};@@ -152,7 +152,7 @@ impl TargetIsa for X64 {fn emit_unwind_info(&self,- buffer: &MachBufferFinalized<Final>,+ buffer: &MachBufferFinalized,kind: cranelift_codegen::isa::unwind::UnwindInfoKind,) -> Result<Option<cranelift_codegen::isa::unwind::UnwindInfo>> {Ok(cranelift_codegen::isa::x64::emit_unwind_info(buffer, kind)?)diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rsindex 677117cda1..603bf86dc4 100644--- a/winch/codegen/src/masm.rs+++ b/winch/codegen/src/masm.rs@@ -6,7 +6,7 @@ use crate::isa::{reg::{Reg, RegClass, WritableReg, writable},};use cranelift_codegen::{- Final, MachBufferFinalized, MachExceptionHandler, MachLabel,+ MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{Endianness, MemFlagsData, RelSourceLoc, SourceLoc, UserExternalNameRef},};@@ -1870,7 +1870,7 @@ pub(crate) trait MacroAssembler {fn push(&mut self, src: Reg, size: OperandSize) -> Result<StackSlot>;/// Finalize the assembly and return the result.- fn finalize(self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>>;+ fn finalize(self, base: Option<SourceLoc>) -> Result<MachBufferFinalized>;/// Zero a particular register.fn zero(&mut self, reg: WritableReg) -> Result<()>;