Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: box a few fields in `MachBuffer`/`MachBufferFinalized` to avoid memmoves. (#14300)

Chris Fallin817c58787f43
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs
index d8acfed3ab..5783fd7798 100644
--- a/cranelift/codegen/src/ir/mod.rs
+++ b/cranelift/codegen/src/ir/mod.rs
@@ -61,6 +61,7 @@ pub use crate::ir::memflags::{
MemFlagsSet,
};
pub use crate::ir::progpoint::ProgramPoint;
+pub use crate::ir::sourceloc::MaybeRelSourceLoc;
pub use crate::ir::sourceloc::RelSourceLoc;
pub use crate::ir::sourceloc::SourceLoc;
pub use crate::ir::stackslot::{
diff --git a/cranelift/codegen/src/ir/sourceloc.rs b/cranelift/codegen/src/ir/sourceloc.rs
index 6704574d0d..95817d2e79 100644
--- a/cranelift/codegen/src/ir/sourceloc.rs
+++ b/cranelift/codegen/src/ir/sourceloc.rs
@@ -9,11 +9,12 @@ use serde_derive::{Deserialize, Serialize};
/// A source location.
///
-/// This is an opaque 32-bit number attached to each Cranelift IR instruction. Cranelift does not
+/// This is an opaque 31-bit number attached to each Cranelift IR instruction. Cranelift does not
/// interpret source locations in any way, they are simply preserved from the input to the output.
///
-/// The default source location uses the all-ones bit pattern `!0`. It is used for instructions
-/// that can't be given a real source location.
+/// The default source location uses the bit pattern `0x7fff_ffff`. It is used for instructions
+/// that can't be given a real source location. The high bit is reserved for
+/// distinguishing relative and absolute locations in [`MaybeRelSourceLoc`].
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct SourceLoc(u32);
@@ -37,7 +38,7 @@ impl SourceLoc {
impl Default for SourceLoc {
fn default() -> Self {
- Self(!0)
+ Self(0x7fff_ffff)
}
}
@@ -67,7 +68,9 @@ impl RelSourceLoc {
if base.is_default() || offset.is_default() {
Self::default()
} else {
- Self(offset.bits().wrapping_sub(base.bits()))
+ // Wrap within the 31-bit source-location range, reserving the
+ // high bit for MaybeRelSourceLoc's relative/absolute tag.
+ Self(offset.bits().wrapping_sub(base.bits()) & MaybeRelSourceLoc::MASK)
}
}
@@ -76,7 +79,7 @@ impl RelSourceLoc {
if self.is_default() || base.is_default() {
Default::default()
} else {
- SourceLoc::new(self.0.wrapping_add(base.bits()))
+ SourceLoc::new(self.0.wrapping_add(base.bits()) & MaybeRelSourceLoc::MASK)
}
}
@@ -88,7 +91,7 @@ impl RelSourceLoc {
impl Default for RelSourceLoc {
fn default() -> Self {
- Self(!0)
+ Self(0x7fff_ffff)
}
}
@@ -102,6 +105,84 @@ impl fmt::Display for RelSourceLoc {
}
}
+/// A source location that is either a `RelSourceLoc` or `SourceLoc`.
+///
+/// This is used to represent a source location in the `MachBuffer`
+/// that is initially relative to some base and is later relocated. We
+/// do this to permit better code caching during incremental
+/// compilation: the MachBuffer records the first SourceLoc it is
+/// given as a base, and if the same function IR is later compiled but
+/// with a different starting SourceLoc, we can reuse the cached
+/// compilation result and just relocate (offset) the SourceLocs.
+///
+/// This relocation is an in-place update pass, so we want a "union"
+/// type, essentially, but we don't want to pay the overhead of a true
+/// `enum` (8 bytes rather than 4 in a large array), so we bitpack
+/// this representation.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct MaybeRelSourceLoc(u32);
+
+impl MaybeRelSourceLoc {
+ const REL_BIT: u32 = 0x8000_0000;
+ const MASK: u32 = !Self::REL_BIT;
+
+ /// Create a relative SourceLoc.
+ pub fn rel(loc: RelSourceLoc) -> Self {
+ debug_assert!(loc.0 & Self::MASK == loc.0);
+ MaybeRelSourceLoc(loc.0 | Self::REL_BIT)
+ }
+
+ /// Create an absolute SourceLoc.
+ pub fn abs(loc: SourceLoc) -> Self {
+ debug_assert!(loc.0 & Self::MASK == loc.0);
+ MaybeRelSourceLoc(loc.0)
+ }
+
+ /// Is this a relative SourceLoc?
+ pub fn is_rel(&self) -> bool {
+ self.0 & Self::REL_BIT != 0
+ }
+
+ /// Is this an absolute SourceLoc?
+ pub fn is_abs(&self) -> bool {
+ self.0 & Self::REL_BIT == 0
+ }
+
+ /// Unwrap a relative SourceLoc.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not a relative SourceLoc.
+ pub fn as_rel(&self) -> RelSourceLoc {
+ assert!(self.is_rel());
+ RelSourceLoc(self.0 & Self::MASK)
+ }
+
+ /// Unwrap an absolute SourceLoc.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not an absolute SourceLoc.
+ pub fn as_abs(&self) -> SourceLoc {
+ assert!(self.is_abs());
+ SourceLoc(self.0)
+ }
+
+ /// Map a relative to an absolute SourceLoc, given another
+ /// absolute SourceLoc as a base.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not a relative SourceLoc.
+ pub fn relocate(&self, base: SourceLoc) -> SourceLoc {
+ self.as_rel().expand(base)
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::ir::SourceLoc;
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index 666d305838..f419633929 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -6,7 +6,6 @@ use crate::isa::aarch64::settings as aarch64_settings;
#[cfg(feature = "unwind")]
use crate::isa::unwind::systemv;
use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};
-#[cfg(feature = "unwind")]
use crate::machinst::CompiledCode;
use crate::machinst::{
CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,
@@ -93,13 +92,13 @@ impl TargetIsa for AArch64Backend {
log::debug!("disassembly:\n{disasm}");
}
- Ok(CompiledCodeStencil {
+ Ok(CompiledCodeStencil(CompiledCode {
buffer,
vcode: emit_result.disasm,
value_labels_ranges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
- })
+ }))
}
fn name(&self) -> &'static str {
diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rs
index 349f10363c..2140bcb562 100644
--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs
@@ -12,7 +12,7 @@ use crate::{
dominator_tree::DominatorTree,
ir,
isa::{self, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa},
- machinst::{self, CompiledCodeStencil, MachInst, SigSet, VCode},
+ machinst::{self, CompiledCode, CompiledCodeStencil, MachInst, SigSet, VCode},
result::CodegenResult,
settings::{self as shared_settings, Flags},
};
@@ -198,13 +198,13 @@ where
log::debug!("disassembly:\n{disasm}");
}
- Ok(CompiledCodeStencil {
+ Ok(CompiledCodeStencil(CompiledCode {
buffer,
vcode: emit_result.disasm,
value_labels_ranges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
- })
+ }))
}
fn emit_unwind_info(
diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs
index 96ec6b81c7..562576a6fc 100644
--- a/cranelift/codegen/src/isa/riscv64/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/mod.rs
@@ -6,7 +6,6 @@ use crate::isa::riscv64::settings as riscv_settings;
use crate::isa::{
Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa,
};
-#[cfg(feature = "unwind")]
use crate::machinst::CompiledCode;
use crate::machinst::{
CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,
@@ -95,13 +94,13 @@ impl TargetIsa for Riscv64Backend {
log::debug!("disassembly:\n{disasm}");
}
- Ok(CompiledCodeStencil {
+ Ok(CompiledCodeStencil(CompiledCode {
buffer,
vcode: emit_result.disasm,
value_labels_ranges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
- })
+ }))
}
fn name(&self) -> &'static str {
diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs
index 8323bd50c7..2f8200644e 100644
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -92,13 +92,13 @@ impl TargetIsa for S390xBackend {
log::debug!("disassembly:\n{disasm}");
}
- Ok(CompiledCodeStencil {
+ Ok(CompiledCodeStencil(CompiledCode {
buffer,
vcode: emit_result.disasm,
value_labels_ranges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
- })
+ }))
}
fn name(&self) -> &'static str {
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index ae2f5daecd..3b1f9afd8d 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -3,6 +3,7 @@
pub use self::inst::{AtomicRmwSeqOp, EmitInfo, EmitState, Inst, args, external};
use super::{OwnedTargetIsa, TargetIsa};
+use crate::MachBufferFinalized;
use crate::dominator_tree::DominatorTree;
use crate::ir::{self, Function, Type, types};
#[cfg(feature = "unwind")]
@@ -10,12 +11,11 @@ use crate::isa::unwind::systemv;
use crate::isa::x64::settings as x64_settings;
use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey};
use crate::machinst::{
- CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,
- compile,
+ CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,
+ TextSectionBuilder, VCode, compile,
};
use crate::result::{CodegenError, CodegenResult};
use crate::settings::{self as shared_settings, Flags};
-use crate::{Final, MachBufferFinalized};
use alloc::string::String;
use alloc::{borrow::ToOwned, boxed::Box, vec::Vec};
use core::fmt;
@@ -101,13 +101,13 @@ impl TargetIsa for X64Backend {
crate::trace!("disassembly:\n{}", disasm);
}
- Ok(CompiledCodeStencil {
+ Ok(CompiledCodeStencil(CompiledCode {
buffer,
vcode: emit_result.disasm,
value_labels_ranges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
- })
+ }))
}
fn flags(&self) -> &Flags {
@@ -223,7 +223,7 @@ impl TargetIsa for X64Backend {
/// Emit unwind info for an x86 target.
pub fn emit_unwind_info(
- buffer: &MachBufferFinalized<Final>,
+ buffer: &MachBufferFinalized,
kind: crate::isa::unwind::UnwindInfoKind,
) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
#[cfg(feature = "unwind")]
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs
index 787aff2b11..d2cfe3ce86 100644
--- a/cranelift/codegen/src/lib.rs
+++ b/cranelift/codegen/src/lib.rs
@@ -70,12 +70,11 @@ pub mod write;
pub use crate::entity::packed_option;
pub use crate::machinst::buffer::{
- ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, FinalizedMachReloc,
- FinalizedRelocTarget, MachCallSite, MachExceptionHandler, MachSrcLoc, MachTextSectionBuilder,
- MachTrap, OpenPatchRegion, PatchRegion,
+ ExceptionContextLoc, MachCallSite, MachCallSiteItem, MachExceptionHandler, MachReloc,
+ MachSrcLoc, MachTextSectionBuilder, MachTrap, OpenPatchRegion, PatchRegion, RelocTarget,
};
pub use crate::machinst::{
- CallInfo, CompiledCode, Final, FrameLayout, MachBuffer, MachBufferDebugTagList,
+ CallInfo, CompiledCode, FrameLayout, LabelOrOffset, MachBuffer, MachBufferDebugTagList,
MachBufferFinalized, MachBufferFrameLayout, MachDebugTagPos, MachInst, MachInstEmit,
MachInstEmitState, MachLabel, RealReg, Reg, RegClass, RelocDistance, TextSectionBuilder,
VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, Writable,
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 9c5ab99ccd..996310db7d 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -2620,8 +2620,8 @@ impl TryCallInfo {
layout: &FrameLayout,
) -> impl Iterator<Item = MachExceptionHandler> {
self.exception_handlers.iter().map(|handler| match handler {
- TryCallHandler::Tag(tag, label) => MachExceptionHandler::Tag(*tag, *label),
- TryCallHandler::Default(label) => MachExceptionHandler::Default(*label),
+ TryCallHandler::Tag(tag, label) => MachExceptionHandler::Tag(*tag, LabelOrOffset::from(*label)),
+ TryCallHandler::Default(label) => MachExceptionHandler::Default(LabelOrOffset::from(*label)),
TryCallHandler::Context(reg) => {
let loc = if let Some(spillslot) = reg.to_spillslot() {
// The spillslot offset is relative to the "fixed
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index 6f6e8564e7..c86a19b644 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -200,7 +200,9 @@
use crate::binemit::{Addend, CodeOffset, Reloc};
use crate::ir::function::FunctionParameters;
-use crate::ir::{DebugTag, ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};
+use crate::ir::{
+ DebugTag, ExceptionTag, ExternalName, MaybeRelSourceLoc, RelSourceLoc, SourceLoc, TrapCode,
+};
use crate::isa::unwind::UnwindInst;
use crate::machinst::{
BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst,
@@ -208,54 +210,99 @@ use crate::machinst::{
use crate::trace;
use crate::{MachInstEmitState, ir};
use crate::{VCodeConstantData, timing};
+use alloc::boxed::Box;
use alloc::collections::BinaryHeap;
use alloc::string::String;
use alloc::vec::Vec;
use core::cmp::Ordering;
use core::mem;
use core::ops::Range;
+use core::ops::{Deref, DerefMut};
use cranelift_control::ControlPlane;
use cranelift_entity::{PrimaryMap, SecondaryMap, entity_impl};
use smallvec::SmallVec;
-#[cfg(feature = "enable-serde")]
-use serde::{Deserialize, Serialize};
-
-#[cfg(feature = "enable-serde")]
-pub trait CompilePhase {
- type MachSrcLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone;
- type SourceLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum ForceVeneers {
+ Yes,
+ No,
}
-#[cfg(not(feature = "enable-serde"))]
-pub trait CompilePhase {
- type MachSrcLocType: core::fmt::Debug + PartialEq + Clone;
- type SourceLocType: core::fmt::Debug + PartialEq + Clone;
-}
+/// A `MachLabel` or `CodeOffset`, bitpacked into a u32.
+///
+/// This type is used to represent a label reference in some
+/// MachBuffer metadata (specifically, relocations and
+/// exception-handler records). These start as labels before the
+/// `MachBuffer` is finalized; once `finish()` is called, they become
+/// code offsets.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct LabelOrOffset(u32);
-/// Status of a compiled artifact that needs patching before being used.
-#[derive(Clone, Debug, PartialEq)]
-#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
-pub struct Stencil;
+impl LabelOrOffset {
+ const LABEL_BIT: u32 = 0x8000_0000;
+ const MASK: u32 = !Self::LABEL_BIT;
-/// Status of a compiled artifact ready to use.
-#[derive(Clone, Debug, PartialEq)]
-pub struct Final;
+ /// Create a `LabelOrOffset` that refers to a label.
+ pub fn label(label: MachLabel) -> Self {
+ debug_assert!(label.0 & Self::MASK == label.0);
+ LabelOrOffset(label.0 | Self::LABEL_BIT)
+ }
-impl CompilePhase for Stencil {
- type MachSrcLocType = MachSrcLoc<Stencil>;
- type SourceLocType = RelSourceLoc;
+ /// Create a `LabelOrOffset` that refers to a code offset.
+ pub fn offset(offset: CodeOffset) -> Self {
+ debug_assert!(offset & Self::MASK == offset);
+ LabelOrOffset(offset)
+ }
+
+ /// Is this a label?
+ pub fn is_label(&self) -> bool {
+ self.0 & Self::LABEL_BIT != 0
+ }
+
+ /// Is this a code offset?
+ pub fn is_offset(&self) -> bool {
+ self.0 & Self::LABEL_BIT == 0
+ }
+
+ /// Unwrap as a label.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not a label.
+ pub fn as_label(&self) -> MachLabel {
+ assert!(self.is_label());
+ MachLabel(self.0 & Self::MASK)
+ }
+
+ /// Unwrap as a code offset.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not a code offset.
+ pub fn as_offset(&self) -> CodeOffset {
+ assert!(self.is_offset());
+ self.0 & Self::MASK
+ }
}
-impl CompilePhase for Final {
- type MachSrcLocType = MachSrcLoc<Final>;
- type SourceLocType = SourceLoc;
+impl From<MachLabel> for LabelOrOffset {
+ fn from(value: MachLabel) -> Self {
+ LabelOrOffset::label(value)
+ }
}
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum ForceVeneers {
- Yes,
- No,
+impl core::fmt::Display for LabelOrOffset {
+ fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+ if self.is_offset() {
+ write!(fmt, "0x{:x}", self.as_offset())
+ } else {
+ write!(fmt, "label{}", self.as_label().0)
+ }
+ }
}
/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
@@ -266,35 +313,13 @@ enum ForceVeneers {
/// likely fine as long as it is stack-allocated for function emission then
/// thrown away; but beware if many buffer objects are retained persistently.
pub struct MachBuffer<I: VCodeInst> {
- /// The buffer contents, as raw bytes.
- data: SmallVec<[u8; 1024]>,
- /// The required alignment of this buffer.
- min_alignment: u32,
- /// Any relocations referring to this code. Note that only *external*
- /// relocations are tracked here; references to labels within the buffer are
- /// resolved before emission.
- relocs: SmallVec<[MachReloc; 16]>,
- /// Any trap records referring to this code.
- traps: SmallVec<[MachTrap; 16]>,
- /// Any call site records referring to this code.
- call_sites: SmallVec<[MachCallSite; 16]>,
- /// Any patchable call site locations.
- patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,
- /// Any exception-handler records referred to at call sites.
- exception_handlers: SmallVec<[MachExceptionHandler; 16]>,
- /// Any source location mappings referring to this code.
- srclocs: SmallVec<[MachSrcLoc<Stencil>; 64]>,
- /// Any debug tags referring to this code.
- debug_tags: Vec<MachDebugTags>,
- /// Pool of debug tags referenced by `MachDebugTags` entries.
- debug_tag_pool: Vec<DebugTag>,
- /// Any user stack maps for this code.
- ///
- /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
- /// by code offset, and each stack map covers `span` bytes on the stack.
- user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,
- /// Any unwind info at a given location.
- unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
+ // --- machine-code data and metadata:
+ //
+ /// Data shared between the unfinalized and finalized MachBuffers.
+ inner: Box<MachBufferInner>,
+
+ // --- emission-pass state:
+ //
/// The current source location in progress (after `start_srcloc()` and
/// before `end_srcloc()`). This is a (start_offset, src_loc) tuple.
cur_srcloc: Option<(CodeOffset, RelSourceLoc)>,
@@ -356,62 +381,44 @@ pub struct MachBuffer<I: VCodeInst> {
/// Indicates when a patchable region is currently open, to guard that it's
/// not possible to nest patchable regions.
open_patchable: bool,
- /// Stack frame layout metadata. If provided for a MachBuffer
- /// containing a function body, this allows interpretation of
- /// runtime state given a view of an active stack frame.
- frame_layout: Option<MachBufferFrameLayout>,
}
-impl MachBufferFinalized<Stencil> {
- /// Get a finalized machine buffer by applying the function's base source location.
- pub fn apply_base_srcloc(self, base_srcloc: SourceLoc) -> MachBufferFinalized<Final> {
- MachBufferFinalized {
- data: self.data,
- relocs: self.relocs,
- traps: self.traps,
- call_sites: self.call_sites,
- patchable_call_sites: self.patchable_call_sites,
- exception_handlers: self.exception_handlers,
- srclocs: self
- .srclocs
- .into_iter()
- .map(|srcloc| srcloc.apply_base_srcloc(base_srcloc))
- .collect(),
- debug_tags: self.debug_tags,
- debug_tag_pool: self.debug_tag_pool,
- user_stack_maps: self.user_stack_maps,
- unwind_info: self.unwind_info,
- alignment: self.alignment,
- frame_layout: self.frame_layout,
- nop_units: self.nop_units,
- }
- }
-}
-
-/// A `MachBuffer` once emission is completed: holds generated code and records,
-/// without fixups. This allows the type to be independent of the backend.
+/// Bulk data that is common between `MachBuffer` and
+/// `MachBufferFinalized`.
+///
+/// The goal is to indirect the large allocations (`SmallVec`s) so
+/// that we don't move a lot of memory during compilation.
+///
+/// The two named types are essentially a builder/final object pair.
+/// The inner state is the same (except that the builder has further
+/// transient state that is later dropped).
+///
+/// However, many of the fields are either moved over wholesale or
+/// patched then moved over (data). We put these fields in
+/// `MachBufferInner`, hold that shared data in a box so that
+/// finalization can just move a pointer, and then impl `Deref` on the
+/// two `MachBuffer` variants so accesses to these fields are
+/// transparent.
#[derive(PartialEq, Debug, Clone)]
#[cfg_attr(
feature = "enable-serde",
derive(serde_derive::Serialize, serde_derive::Deserialize)
)]
-pub struct MachBufferFinalized<T: CompilePhase> {
+pub struct MachBufferInner {
/// The buffer contents, as raw bytes.
pub(crate) data: SmallVec<[u8; 1024]>,
+ /// Any trap records referring to this code.
+ pub(crate) traps: SmallVec<[MachTrap; 16]>,
/// Any relocations referring to this code. Note that only *external*
/// relocations are tracked here; references to labels within the buffer are
/// resolved before emission.
- pub(crate) relocs: SmallVec<[FinalizedMachReloc; 16]>,
- /// Any trap records referring to this code.
- pub(crate) traps: SmallVec<[MachTrap; 16]>,
+ pub(crate) relocs: SmallVec<[MachReloc; 16]>,
+ /// Any exception-handler records referred to at call sites.
+ pub(crate) exception_handlers: SmallVec<[MachExceptionHandler; 16]>,
/// Any call site records referring to this code.
pub(crate) call_sites: SmallVec<[MachCallSite; 16]>,
- /// Any patchable call site locations referring to this code.
+ /// Any patchable call site locations.
pub(crate) patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,
- /// Any exception-handler records referred to at call sites.
- pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,
- /// Any source location mappings referring to this code.
- pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>,
/// Any debug tags referring to this code.
pub(crate) debug_tags: Vec<MachDebugTags>,
/// Pool of debug tags referenced by `MachDebugTags` entries.
@@ -421,14 +428,60 @@ pub struct MachBufferFinalized<T: CompilePhase> {
/// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
/// by code offset, and each stack map covers `span` bytes on the stack.
pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,
+ /// Any unwind info at a given location.
+ pub(crate) unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
/// Stack frame layout metadata. If provided for a MachBuffer
/// containing a function body, this allows interpretation of
/// runtime state given a view of an active stack frame.
pub(crate) frame_layout: Option<MachBufferFrameLayout>,
- /// Any unwind info at a given location.
- pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
+ /// Any source location mappings referring to this code.
+ pub(crate) srclocs: SmallVec<[MachSrcLoc; 64]>,
/// The required alignment of this buffer.
- pub alignment: u32,
+ pub min_alignment: u32,
+}
+
+impl<I: VCodeInst> Deref for MachBuffer<I> {
+ type Target = MachBufferInner;
+ fn deref(&self) -> &Self::Target {
+ &*self.inner
+ }
+}
+impl<I: VCodeInst> DerefMut for MachBuffer<I> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut *self.inner
+ }
+}
+impl Deref for MachBufferFinalized {
+ type Target = MachBufferInner;
+ fn deref(&self) -> &Self::Target {
+ &*self.inner
+ }
+}
+impl DerefMut for MachBufferFinalized {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut *self.inner
+ }
+}
+
+impl MachBufferFinalized {
+ /// Get a finalized machine buffer by applying the function's base source location.
+ pub fn apply_base_srcloc(&mut self, base_srcloc: SourceLoc) {
+ for loc in &mut self.inner.srclocs {
+ loc.apply_base_srcloc(base_srcloc);
+ }
+ }
+}
+
+/// A `MachBuffer` once emission is completed: holds generated code and records,
+/// without fixups. This allows the type to be independent of the backend.
+#[derive(PartialEq, Debug, Clone)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct MachBufferFinalized {
+ /// The raw data and finalization-invariant metadata attached to it.
+ pub(crate) inner: Box<MachBufferInner>,
/// The means by which to NOP out patchable call sites.
///
/// This allows a consumer of a `MachBufferFinalized` to disable
@@ -501,19 +554,23 @@ impl<I: VCodeInst> MachBuffer<I> {
/// Create a new section, known to start at `start_offset` and with a size limited to
/// `length_limit`.
pub fn new() -> MachBuffer<I> {
- MachBuffer {
+ let inner = Box::new(MachBufferInner {
data: SmallVec::new(),
- min_alignment: I::function_alignment().minimum,
- relocs: SmallVec::new(),
traps: SmallVec::new(),
+ relocs: SmallVec::new(),
+ exception_handlers: SmallVec::new(),
call_sites: SmallVec::new(),
patchable_call_sites: SmallVec::new(),
- exception_handlers: SmallVec::new(),
- srclocs: SmallVec::new(),
debug_tags: vec![],
debug_tag_pool: vec![],
user_stack_maps: SmallVec::new(),
unwind_info: SmallVec::new(),
+ frame_layout: None,
+ srclocs: SmallVec::new(),
+ min_alignment: I::function_alignment().minimum,
+ });
+ MachBuffer {
+ inner,
cur_srcloc: None,
label_offsets: SmallVec::new(),
label_aliases: SmallVec::new(),
@@ -529,7 +586,6 @@ impl<I: VCodeInst> MachBuffer<I> {
constants: Default::default(),
used_constants: Default::default(),
open_patchable: false,
- frame_layout: None,
}
}
@@ -682,15 +738,13 @@ impl<I: VCodeInst> MachBuffer<I> {
/// Returns the alignment required for this entire buffer. Alignment starts
/// at the ISA's minimum function alignment and can be increased due to
/// constant requirements.
- fn finish_constants(&mut self, constants: &VCodeConstants) -> u32 {
- let mut alignment = self.min_alignment;
+ fn finish_constants(&mut self, constants: &VCodeConstants) {
for (constant, offset) in mem::take(&mut self.used_constants) {
let constant = constants.get(constant);
let data = constant.as_slice();
self.data[offset as usize..][..data.len()].copy_from_slice(data);
- alignment = constant.alignment().max(alignment);
+ self.min_alignment = constant.alignment().max(self.min_alignment);
}
- alignment
}
/// Returns a label that can be used to refer to the `constant` provided.
@@ -1236,14 +1290,14 @@ impl<I: VCodeInst> MachBuffer<I> {
let off_before_edit = self.cur_offset();
let prev_b = self.latest_branches.last_mut().unwrap();
let not_inverted = SmallVec::from(
- &self.data[(prev_b.start as usize)..(prev_b.end as usize)],
+ &self.inner.data[(prev_b.start as usize)..(prev_b.end as usize)],
);
// Low-level edit: replaces bytes of branch with
// inverted form. cur_off remains the same afterward, so
// we do not need to modify label data structures.
- self.data.truncate(prev_b.start as usize);
- self.data.extend_from_slice(&data[..]);
+ self.inner.data.truncate(prev_b.start as usize);
+ self.inner.data.extend_from_slice(&data[..]);
// Save the original code as the inversion of the
// inverted branch, in case we later edit this branch
@@ -1628,55 +1682,30 @@ impl<I: VCodeInst> MachBuffer<I> {
mut self,
constants: &VCodeConstants,
ctrl_plane: &mut ControlPlane,
- ) -> MachBufferFinalized<Stencil> {
+ ) -> MachBufferFinalized {
let _tt = timing::vcode_emit_finish();
self.finish_emission_maybe_forcing_veneers(ForceVeneers::No, ctrl_plane);
-
- let alignment = self.finish_constants(constants);
+ self.finish_constants(constants);
// Resolve all labels to their offsets.
- let finalized_relocs = self
- .relocs
- .iter()
- .map(|reloc| FinalizedMachReloc {
- offset: reloc.offset,
- kind: reloc.kind,
- addend: reloc.addend,
- target: match &reloc.target {
- RelocTarget::ExternalName(name) => {
- FinalizedRelocTarget::ExternalName(name.clone())
- }
- RelocTarget::Label(label) => {
- FinalizedRelocTarget::Func(self.resolve_label_offset(*label))
- }
- },
- })
- .collect();
-
- let finalized_exception_handlers = self
- .exception_handlers
- .iter()
- .map(|handler| handler.finalize(|label| self.resolve_label_offset(label)))
- .collect();
-
- let mut srclocs = self.srclocs;
- srclocs.sort_by_key(|entry| entry.start);
+ let mut relocs = core::mem::take(&mut self.relocs);
+ let mut exception_handlers = core::mem::take(&mut self.exception_handlers);
+ let resolve = |label: LabelOrOffset| {
+ LabelOrOffset::offset(self.resolve_label_offset(label.as_label()))
+ };
+ for reloc in &mut relocs {
+ reloc.target.map(resolve);
+ }
+ for handler in &mut exception_handlers {
+ handler.map(resolve);
+ }
+ self.relocs = relocs;
+ self.exception_handlers = exception_handlers;
+ self.srclocs.sort_by_key(|entry| entry.start);
MachBufferFinalized {
- data: self.data,
- relocs: finalized_relocs,
- traps: self.traps,
- call_sites: self.call_sites,
- patchable_call_sites: self.patchable_call_sites,
- exception_handlers: finalized_exception_handlers,
- srclocs,
- debug_tags: self.debug_tags,
- debug_tag_pool: self.debug_tag_pool,
- user_stack_maps: self.user_stack_maps,
- unwind_info: self.unwind_info,
- alignment,
- frame_layout: self.frame_layout,
+ inner: self.inner,
nop_units: I::gen_nop_units(),
}
}
@@ -1743,8 +1772,8 @@ impl<I: VCodeInst> MachBuffer<I> {
/// Add a trap record at the current offset.
pub fn add_trap(&mut self, code: TrapCode) {
- self.traps.push(MachTrap {
- offset: self.data.len() as CodeOffset,
+ self.inner.traps.push(MachTrap {
+ offset: self.inner.data.len() as CodeOffset,
code,
});
}
@@ -1766,8 +1795,8 @@ impl<I: VCodeInst> MachBuffer<I> {
let end = u32::try_from(self.exception_handlers.len()).unwrap();
let exception_handler_range = start..end;
- self.call_sites.push(MachCallSite {
- ret_addr: self.data.len() as CodeOffset,
+ self.inner.call_sites.push(MachCallSite {
+ ret_addr: self.inner.data.len() as CodeOffset,
frame_offset,
exception_handler_range,
});
@@ -1778,7 +1807,7 @@ impl<I: VCodeInst> MachBuffer<I> {
/// specifies how to NOP it out, and we carry that information to
/// the finalized Machbuffer.
pub fn add_patchable_call_site(&mut self, len: u32) {
- self.patchable_call_sites.push(MachPatchableCallSite {
+ self.inner.patchable_call_sites.push(MachPatchableCallSite {
ret_addr: self.cur_offset(),
len,
});
@@ -1786,7 +1815,7 @@ impl<I: VCodeInst> MachBuffer<I> {
/// Add an unwind record at the current offset.
pub fn add_unwind(&mut self, unwind: UnwindInst) {
- self.unwind_info.push((self.cur_offset(), unwind));
+ self.inner.unwind_info.push((self.cur_offset(), unwind));
}
/// Set the `SourceLoc` for code from this offset until the offset at the
@@ -1809,7 +1838,11 @@ impl<I: VCodeInst> MachBuffer<I> {
// Skip zero-length extends.
debug_assert!(end >= start);
if end > start {
- self.srclocs.push(MachSrcLoc { start, end, loc });
+ self.srclocs.push(MachSrcLoc {
+ start,
+ end,
+ loc: MaybeRelSourceLoc::rel(loc),
+ });
}
}
@@ -1867,7 +1900,7 @@ impl<I: VCodeInst> MachBuffer<I> {
let start = u32::try_from(self.debug_tag_pool.len()).unwrap();
self.debug_tag_pool.extend(tags.iter().cloned());
let end = u32::try_from(self.debug_tag_pool.len()).unwrap();
- self.debug_tags.push(MachDebugTags {
+ self.inner.debug_tags.push(MachDebugTags {
offset: self.cur_offset(),
pos,
range: start..end,
@@ -1898,9 +1931,9 @@ impl<I: VCodeInst> Extend<u8> for MachBuffer<I> {
}
}
-impl<T: CompilePhase> MachBufferFinalized<T> {
+impl MachBufferFinalized {
/// Get a list of source location mapping tuples in sorted-by-start-offset order.
- pub fn get_srclocs_sorted(&self) -> &[T::MachSrcLocType] {
+ pub fn get_srclocs_sorted(&self) -> &[MachSrcLoc] {
&self.srclocs[..]
}
@@ -1956,7 +1989,7 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
}
/// Get the list of external relocations for this code.
- pub fn relocs(&self) -> &[FinalizedMachReloc] {
+ pub fn relocs(&self) -> &[MachReloc] {
&self.relocs[..]
}
@@ -1986,12 +2019,12 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
/// - The slice of pairs of exception tags and code offsets
/// denoting exception-handler entry points associated with this
/// call site.
- pub fn call_sites(&self) -> impl Iterator<Item = FinalizedMachCallSite<'_>> + '_ {
+ pub fn call_sites(&self) -> impl Iterator<Item = MachCallSiteItem<'_>> + '_ {
self.call_sites.iter().map(|call_site| {
let handler_range = call_site.exception_handler_range.clone();
let handler_range = usize::try_from(handler_range.start).unwrap()
..usize::try_from(handler_range.end).unwrap();
- FinalizedMachCallSite {
+ MachCallSiteItem {
ret_addr: call_site.ret_addr,
frame_offset: call_site.frame_offset,
exception_handlers: &self.exception_handlers[handler_range],
@@ -2022,13 +2055,17 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
/// references. Items are interpreted in left-to-right order and the
/// first match wins.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
pub enum MachExceptionHandler {
/// A specific tag (in the current dynamic context) should be
/// handled by the code at the given offset.
- Tag(ExceptionTag, MachLabel),
+ Tag(ExceptionTag, LabelOrOffset),
/// All exceptions should be handled by the code at the given
/// offset.
- Default(MachLabel),
+ Default(LabelOrOffset),
/// The dynamic context for interpreting tags is updated to the
/// value stored in the given machine location (in this frame's
/// context).
@@ -2036,36 +2073,19 @@ pub enum MachExceptionHandler {
}
impl MachExceptionHandler {
- fn finalize<F: Fn(MachLabel) -> CodeOffset>(self, f: F) -> FinalizedMachExceptionHandler {
+ fn map<F: Fn(LabelOrOffset) -> LabelOrOffset>(&mut self, f: F) {
match self {
- Self::Tag(tag, label) => FinalizedMachExceptionHandler::Tag(tag, f(label)),
- Self::Default(label) => FinalizedMachExceptionHandler::Default(f(label)),
- Self::Context(loc) => FinalizedMachExceptionHandler::Context(loc),
+ Self::Tag(_, label) => {
+ *label = f(*label);
+ }
+ Self::Default(label) => {
+ *label = f(*label);
+ }
+ Self::Context(_loc) => {}
}
}
}
-/// An item in the exception-handler list for a callsite, with final
-/// (lowered) code offsets. Items are interpreted in left-to-right
-/// order and the first match wins.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-#[cfg_attr(
- feature = "enable-serde",
- derive(serde_derive::Serialize, serde_derive::Deserialize)
-)]
-pub enum FinalizedMachExceptionHandler {
- /// A specific tag (in the current dynamic context) should be
- /// handled by the code at the given offset.
- Tag(ExceptionTag, CodeOffset),
- /// All exceptions should be handled by the code at the given
- /// offset.
- Default(CodeOffset),
- /// The dynamic context for interpreting tags is updated to the
- /// value stored in the given machine location (in this frame's
- /// context).
- Context(ExceptionContextLoc),
-}
-
/// A location for a dynamic exception context value.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[cfg_attr(
@@ -2154,34 +2174,35 @@ impl<I: VCodeInst> Ord for MachLabelFixup<I> {
feature = "enable-serde",
derive(serde_derive::Serialize, serde_derive::Deserialize)
)]
-pub struct MachRelocBase<T> {
+pub struct MachReloc {
/// The offset at which the relocation applies, *relative to the
/// containing section*.
pub offset: CodeOffset,
/// The kind of relocation.
pub kind: Reloc,
/// The external symbol / name to which this relocation refers.
- pub target: T,
+ pub target: RelocTarget,
/// The addend to add to the symbol value.
pub addend: i64,
}
-type MachReloc = MachRelocBase<RelocTarget>;
-
-/// A relocation resulting from a compilation.
-pub type FinalizedMachReloc = MachRelocBase<FinalizedRelocTarget>;
-
/// A Relocation target
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
pub enum RelocTarget {
/// Points to an [ExternalName] outside the current function.
ExternalName(ExternalName),
- /// Points to a [MachLabel] inside this function.
- /// This is different from [MachLabelFixup] in that both the relocation and the
- /// label will be emitted and are only resolved at link time.
+ /// Points to a [MachLabel] inside this function. This is different
+ /// from an internal fixup/label reference in that both the
+ /// relocation and the label will be emitted and are only resolved
+ /// at link time.
///
- /// There is no reason to prefer this over [MachLabelFixup] unless the ABI requires it.
- Label(MachLabel),
+ /// There is no reason to prefer this over internal fixups unless
+ /// the ABI requires it.
+ Label(LabelOrOffset),
}
impl From<ExternalName> for RelocTarget {
@@ -2192,30 +2213,26 @@ impl From<ExternalName> for RelocTarget {
impl From<MachLabel> for RelocTarget {
fn from(label: MachLabel) -> Self {
- Self::Label(label)
+ Self::Label(LabelOrOffset::label(label))
}
}
-/// A Relocation target
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-#[cfg_attr(
- feature = "enable-serde",
- derive(serde_derive::Serialize, serde_derive::Deserialize)
-)]
-pub enum FinalizedRelocTarget {
- /// Points to an [ExternalName] outside the current function.
- ExternalName(ExternalName),
- /// Points to a [CodeOffset] from the start of the current function.
- Func(CodeOffset),
-}
-
-impl FinalizedRelocTarget {
- /// Returns a display for the current [FinalizedRelocTarget], with extra context to prettify the
+impl RelocTarget {
+ /// Returns a display for the current [RelocTarget], with extra context to prettify the
/// output.
pub fn display<'a>(&'a self, params: Option<&'a FunctionParameters>) -> String {
match self {
- FinalizedRelocTarget::ExternalName(name) => format!("{}", name.display(params)),
- FinalizedRelocTarget::Func(offset) => format!("func+{offset}"),
+ RelocTarget::ExternalName(name) => format!("{}", name.display(params)),
+ RelocTarget::Label(offset) => format!("func+{offset}"),
+ }
+ }
+
+ fn map<F: Fn(LabelOrOffset) -> LabelOrOffset>(&mut self, f: F) {
+ match self {
+ RelocTarget::ExternalName(_) => {}
+ RelocTarget::Label(label) => {
+ *label = f(*label);
+ }
}
}
}
@@ -2263,29 +2280,22 @@ pub struct MachCallSite {
exception_handler_range: Range<u32>,
}
-/// A call site record resulting from a compilation.
+/// A view onto a call site record resulting from a compilation,
+/// returned during iteration.
#[derive(Clone, Debug, PartialEq)]
-pub struct FinalizedMachCallSite<'a> {
+pub struct MachCallSiteItem<'a> {
/// The offset of the call's return address, *relative to the
/// start of the buffer*.
pub ret_addr: CodeOffset,
/// The offset from the FP at this callsite down to the SP when
- /// the call occurs, if known. In other words, the size of the
- /// stack frame up to the saved FP slot. Useful to recover the
- /// start of the stack frame and to look up dynamic contexts
- /// stored in [`ExceptionContextLoc::SPOffset`].
+ /// the call occurs, if known.
///
- /// If `None`, the compiler backend did not specify a frame
- /// offset. The runtime in use with the compiled code may require
- /// the frame offset if exception handlers are present or dynamic
- /// context is used, but that is not Cranelift's concern: the
- /// frame offset is optional at this level.
+ /// See [`MachCallSite::frame_offset`] for more.
pub frame_offset: Option<u32>,
- /// Exception handlers at this callsite, with target offsets
- /// *relative to the start of the buffer*.
- pub exception_handlers: &'a [FinalizedMachExceptionHandler],
+ /// Exception handlers at this site.
+ pub exception_handlers: &'a [MachExceptionHandler],
}
/// A patchable call site record resulting from a compilation.
@@ -2310,7 +2320,7 @@ pub struct MachPatchableCallSite {
feature = "enable-serde",
derive(serde_derive::Serialize, serde_derive::Deserialize)
)]
-pub struct MachSrcLoc<T: CompilePhase> {
+pub struct MachSrcLoc {
/// The start of the region of code corresponding to a source location.
/// This is relative to the start of the function, not to the start of the
/// section.
@@ -2320,16 +2330,12 @@ pub struct MachSrcLoc<T: CompilePhase> {
/// section.
pub end: CodeOffset,
/// The source location.
- pub loc: T::SourceLocType,
+ pub loc: MaybeRelSourceLoc,
}
-impl MachSrcLoc<Stencil> {
- fn apply_base_srcloc(self, base_srcloc: SourceLoc) -> MachSrcLoc<Final> {
- MachSrcLoc {
- start: self.start,
- end: self.end,
- loc: self.loc.expand(base_srcloc),
- }
+impl MachSrcLoc {
+ fn apply_base_srcloc(&mut self, base_srcloc: SourceLoc) {
+ self.loc = MaybeRelSourceLoc::abs(self.loc.relocate(base_srcloc));
}
}
@@ -2968,8 +2974,8 @@ mod test {
buf.add_try_call_site(
Some(0x10),
[
- MachExceptionHandler::Tag(ExceptionTag::new(42), label(2)),
- MachExceptionHandler::Default(label(1)),
+ MachExceptionHandler::Tag(ExceptionTag::new(42), label(2).into()),
+ MachExceptionHandler::Default(label(1).into()),
]
.into_iter(),
);
@@ -3010,8 +3016,8 @@ mod test {
assert_eq!(
call_sites[0].exception_handlers,
&[
- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(42), 5),
- FinalizedMachExceptionHandler::Default(4)
+ MachExceptionHandler::Tag(ExceptionTag::new(42), LabelOrOffset::offset(5)),
+ MachExceptionHandler::Default(LabelOrOffset::offset(4))
],
);
assert_eq!(
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index 5b9b16a94a..42fab7cd8c 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -580,9 +580,9 @@ pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
/// code (as bytes) and a disassembly, if requested.
#[derive(PartialEq, Debug, Clone)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
-pub struct CompiledCodeBase<T: CompilePhase> {
+pub struct CompiledCode {
/// Machine code.
- pub buffer: MachBufferFinalized<T>,
+ pub buffer: MachBufferFinalized,
/// Disassembly, if requested.
pub vcode: Option<String>,
/// Debug info: value labels to registers/stackslots at code offsets.
@@ -601,20 +601,22 @@ pub struct CompiledCodeBase<T: CompilePhase> {
pub bb_edges: Vec<(CodeOffset, CodeOffset)>,
}
+/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.
+///
+/// Only used internally, in a transient manner, for the incremental compilation cache.
+#[derive(PartialEq, Debug, Clone)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct CompiledCodeStencil(pub(crate) CompiledCode);
+
impl CompiledCodeStencil {
/// Apply function parameters to finalize a stencil into its final form.
- pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode {
- CompiledCode {
- buffer: self.buffer.apply_base_srcloc(params.base_srcloc()),
- vcode: self.vcode,
- value_labels_ranges: self.value_labels_ranges,
- bb_starts: self.bb_starts,
- bb_edges: self.bb_edges,
- }
+ pub fn apply_params(mut self, params: &FunctionParameters) -> CompiledCode {
+ self.0.buffer.apply_base_srcloc(params.base_srcloc());
+ self.0
}
}
-impl<T: CompilePhase> CompiledCodeBase<T> {
+impl CompiledCode {
/// Get a `CodeInfo` describing section sizes from this compilation result.
pub fn code_info(&self) -> CodeInfo {
CodeInfo {
@@ -749,18 +751,7 @@ impl<T: CompilePhase> CompiledCodeBase<T> {
anyhow::format_err!("{err}")
}
}
-}
-/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.
-///
-/// Only used internally, in a transient manner, for the incremental compilation cache.
-pub type CompiledCodeStencil = CompiledCodeBase<Stencil>;
-
-/// `CompiledCode` in its final form (i.e. after `FunctionParameters` have been applied), ready for
-/// consumption.
-pub type CompiledCode = CompiledCodeBase<Final>;
-
-impl CompiledCode {
/// If available, return information about the code layout in the
/// final machine code: the offsets (in bytes) of each basic-block
/// start, and all basic-block edges.
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index 87a4d05871..12af7f434b 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -211,7 +211,7 @@ pub struct VCode<I: VCodeInst> {
/// and optionally metadata about the code layout.
pub struct EmitResult {
/// The MachBuffer containing the machine code.
- pub buffer: MachBufferFinalized<Stencil>,
+ pub buffer: MachBufferFinalized,
/// Offset of each basic block, recorded during emission. Computed
/// only if `machine_code_cfg_info` is enabled.
diff --git a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
index cee91ef108..9a7cbcb47f 100644
--- a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
@@ -36,7 +36,7 @@ block0(v0: i64):
; Disassembled:
; block0: ; offset 0x0
; auipc a1, 0 ; reloc_external RiscvPCRelHi20 %sym 0
-; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0x0 0
; fence rw, w
; sd a0, 0(a1) ; trap: heap_oob
; ret
@@ -97,7 +97,7 @@ block0(v0: i32):
; Disassembled:
; block0: ; offset 0x0
; auipc a1, 0 ; reloc_external RiscvPCRelHi20 %sym 0
-; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a1, a1 ; reloc_external RiscvPCRelLo12I func+0x0 0
; fence rw, w
; sw a0, 0(a1) ; trap: heap_oob
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
index 5feaa4ce33..56885db4f9 100644
--- a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
@@ -84,7 +84,7 @@ block0(v0: i64):
; mv s0, sp
; block1: ; offset 0x10
; auipc t0, 0 ; reloc_external RiscvPCRelHi20 %callee_i64 0
-; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+16 0
+; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+0x10 0
; ld ra, 8(sp)
; ld s0, 0(sp)
; addi sp, sp, 0x10
diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
index 1e4d60a141..2c0a8d8f1e 100644
--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
@@ -19,6 +19,6 @@ block0:
; Disassembled:
; block0: ; offset 0x0
; auipc a0, 0 ; reloc_external RiscvGotHi20 %my_global 0
-; ld a0, 0(a0) ; reloc_external RiscvPCRelLo12I func+0 0
+; ld a0, 0(a0) ; reloc_external RiscvPCRelLo12I func+0x0 0
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
index 97d1594805..1093810f0f 100644
--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
@@ -40,7 +40,7 @@ block0:
; Disassembled:
; block0: ; offset 0x0
; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %func0 0
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0
; ret
function %symbol_value() -> i64 {
@@ -125,7 +125,7 @@ block0:
; Disassembled:
; block0: ; offset 0x0
; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 0
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0
; ret
function %colocated_symbol_value_plus_offset() -> i64 {
@@ -144,7 +144,7 @@ block0:
; Disassembled:
; block0: ; offset 0x0
; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 123
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0
; ret
function %colocated_symbol_value_minus_offset() -> i64 {
@@ -163,6 +163,6 @@ block0:
; Disassembled:
; block0: ; offset 0x0
; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 -123
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x0 0
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif
index b33b79c160..d59e0ce388 100644
--- a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif
@@ -40,7 +40,7 @@ block0(v0: i32):
; block1: ; offset 0x18
; mv s1, a0
; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+28 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x1c 0
; auipc ra, 0 ; reloc_external RiscvCallPlt %ElfTlsGetAddr 0
; jalr ra
; mv a1, a0
@@ -85,7 +85,7 @@ block0(v0: i64):
; mv s0, sp
; block1: ; offset 0x10
; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0
-; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+16 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0x10 0
; auipc ra, 0 ; reloc_external RiscvCallPlt %ElfTlsGetAddr 0
; jalr ra
; sb zero, 0(a0)
diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs
index 7d50bebbe9..31a3b1e059 100644
--- a/cranelift/jit/src/backend.rs
+++ b/cranelift/jit/src/backend.rs
@@ -484,7 +484,7 @@ impl Module for JITModule {
// work around borrow-checker to allow reuse of ctx below
let res = ctx.compile(self.isa(), ctrl_plane)?;
- let alignment = res.buffer.alignment as u64;
+ let alignment = res.buffer.min_alignment as u64;
let compiled_code = ctx.compiled_code().unwrap();
let align = alignment
diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs
index 76972d2129..2afa4424ef 100644
--- a/cranelift/module/src/module.rs
+++ b/cranelift/module/src/module.rs
@@ -13,9 +13,7 @@ use cranelift_codegen::entity::{PrimaryMap, entity_impl};
use cranelift_codegen::ir::ExternalName;
use cranelift_codegen::ir::function::{Function, VersionMarker};
use cranelift_codegen::settings::SetError;
-use cranelift_codegen::{
- CodegenError, CompileError, Context, FinalizedMachReloc, FinalizedRelocTarget, ir, isa,
-};
+use cranelift_codegen::{CodegenError, CompileError, Context, MachReloc, RelocTarget, ir, isa};
use cranelift_control::ControlPlane;
use std::borrow::{Cow, ToOwned};
use std::boxed::Box;
@@ -36,25 +34,22 @@ pub struct ModuleReloc {
}
impl ModuleReloc {
- /// Converts a `FinalizedMachReloc` produced from a `Function` into a `ModuleReloc`.
- pub fn from_mach_reloc(
- mach_reloc: &FinalizedMachReloc,
- func: &Function,
- func_id: FuncId,
- ) -> Self {
+ /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`.
+ pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function, func_id: FuncId) -> Self {
let name = match mach_reloc.target {
- FinalizedRelocTarget::ExternalName(ExternalName::User(reff)) => {
+ RelocTarget::ExternalName(ExternalName::User(reff)) => {
let name = &func.params.user_named_funcs()[reff];
ModuleRelocTarget::user(name.namespace, name.index)
}
- FinalizedRelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(),
- FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
+ RelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(),
+ RelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
ModuleRelocTarget::LibCall(libcall)
}
- FinalizedRelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => {
+ RelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => {
ModuleRelocTarget::KnownSymbol(ks)
}
- FinalizedRelocTarget::Func(offset) => {
+ RelocTarget::Label(label) => {
+ let offset = label.as_offset();
ModuleRelocTarget::FunctionOffset(func_id, offset)
}
};
diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs
index 2aa58c1c63..71b0cf31e0 100644
--- a/cranelift/object/src/backend.rs
+++ b/cranelift/object/src/backend.rs
@@ -448,7 +448,7 @@ impl Module for ObjectModule {
info!("defining function {}: {}", func_id, ctx.func.display());
let res = ctx.compile(self.isa(), ctrl_plane)?;
- let alignment = res.buffer.alignment as u64;
+ let alignment = res.buffer.min_alignment as u64;
let compiled = ctx.compiled_code().unwrap();
#[cfg(feature = "unwind")]
diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs
index cfd635c9ab..adee07b0bc 100644
--- a/cranelift/src/disasm.rs
+++ b/cranelift/src/disasm.rs
@@ -2,12 +2,12 @@ use anyhow::Result;
use cranelift_codegen::ir::Function;
use cranelift_codegen::ir::function::FunctionParameters;
use cranelift_codegen::isa::TargetIsa;
-use cranelift_codegen::{FinalizedMachReloc, MachTrap};
+use cranelift_codegen::{MachReloc, MachTrap};
use std::fmt::Write;
-fn print_relocs(func_params: &FunctionParameters, relocs: &[FinalizedMachReloc]) -> String {
+fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String {
let mut text = String::new();
- for &FinalizedMachReloc {
+ for &MachReloc {
kind,
offset,
ref target,
@@ -103,7 +103,7 @@ pub fn print_all(
mem: &[u8],
code_size: u32,
print: bool,
- relocs: &[FinalizedMachReloc],
+ relocs: &[MachReloc],
traps: &[MachTrap],
) -> Result<()> {
print_bytes(&mem);
diff --git a/crates/cranelift/src/compiled_function.rs b/crates/cranelift/src/compiled_function.rs
index 6a6c351c27..5672bb9b58 100644
--- a/crates/cranelift/src/compiled_function.rs
+++ b/crates/cranelift/src/compiled_function.rs
@@ -2,7 +2,7 @@ use std::ops::Range;
use crate::{Relocation, mach_reloc_to_reloc, mach_trap_to_trap};
use cranelift_codegen::{
- Final, MachBufferFinalized, MachBufferFrameLayout, MachSrcLoc, ValueLabelsRanges, ir,
+ MachBufferFinalized, MachBufferFrameLayout, MachSrcLoc, ValueLabelsRanges, ir,
isa::unwind::CfaUnwindInfo, isa::unwind::UnwindInfo,
};
use wasmtime_environ::{
@@ -58,7 +58,7 @@ pub struct CompiledFunctionMetadata {
/// Compiled function: machine code body, jump table offsets, and unwind information.
pub struct CompiledFunction {
/// The machine code buffer for this function.
- pub buffer: MachBufferFinalized<Final>,
+ pub buffer: MachBufferFinalized,
/// What names each name ref corresponds to.
name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
/// The alignment for the compiled function.
@@ -77,7 +77,7 @@ impl CompiledFunction {
/// This function uses the information in the machine buffer to derive the traps and relocations
/// fields. The compiled function metadata is loaded with the default values.
pub fn new(
- buffer: MachBufferFinalized<Final>,
+ buffer: MachBufferFinalized,
name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
alignment: u32,
) -> Self {
@@ -169,7 +169,7 @@ impl CompiledFunction {
.buffer
.get_srclocs_sorted()
.into_iter()
- .map(|&MachSrcLoc { start, end, loc }| (loc, start, (end - start)));
+ .map(|&MachSrcLoc { start, end, loc }| (loc.as_abs(), start, (end - start)));
let instructions = if with_instruction_addresses {
collect_address_maps(len.try_into().unwrap(), srclocs)
} else {
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 980a498b7d..f540f1b860 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -19,7 +19,7 @@ use cranelift_codegen::isa::{
};
use cranelift_codegen::print_errors::pretty_error;
use cranelift_codegen::{
- CompiledCode, Context, FinalizedMachCallSite, MachBufferDebugTagList, MachBufferFrameLayout,
+ CompiledCode, Context, MachBufferDebugTagList, MachBufferFrameLayout, MachCallSiteItem,
MachDebugTagPos,
};
use cranelift_entity::PrimaryMap;
@@ -1695,7 +1695,7 @@ impl FunctionCompiler<'_> {
1
};
- let alignment = compiled_code.buffer.alignment.max(preferred_alignment);
+ let alignment = compiled_code.buffer.min_alignment.max(preferred_alignment);
let mut compiled_function = CompiledFunction::new(
compiled_code.buffer.clone(),
context.func.params.user_named_funcs().clone(),
@@ -1787,7 +1787,7 @@ fn clif_to_env_stack_maps(
fn clif_to_env_exception_tables<'a>(
builder: &mut ExceptionTableBuilder,
range: Range<u64>,
- call_sites: impl Iterator<Item = FinalizedMachCallSite<'a>>,
+ call_sites: impl Iterator<Item = MachCallSiteItem<'a>>,
) -> Result<()> {
builder.add_func(CodeOffset::try_from(range.start).unwrap(), call_sites)
}
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index fabf37bdec..bb5c2c5594 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -15,7 +15,7 @@
#![warn(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
use cranelift_codegen::{
- FinalizedMachReloc, FinalizedRelocTarget, MachTrap, binemit,
+ MachReloc, MachTrap, RelocTarget, binemit,
cursor::FuncCursor,
ir::{self, AbiParam, ArgumentPurpose, ExternalName, InstBuilder, Signature, TrapCode},
isa::{CallConv, TargetIsa},
@@ -307,21 +307,21 @@ fn clif_trap_to_env_trap(trap: ir::TrapCode, tunables: &Tunables) -> Option<Comp
/// Converts machine relocations to relocation information
/// to perform.
fn mach_reloc_to_reloc(
- reloc: &FinalizedMachReloc,
+ reloc: &MachReloc,
name_map: &PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
) -> Relocation {
- let &FinalizedMachReloc {
+ let &MachReloc {
offset,
kind,
ref target,
addend,
} = reloc;
let reloc_target = match *target {
- FinalizedRelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {
+ RelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {
let name = &name_map[user_func_ref];
FuncKey::from_raw_parts(name.namespace, name.index)
}
- FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
+ RelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
// We should have avoided any code that needs this style of libcalls
// in the Wasm-to-Cranelift translator.
panic!("unexpected libcall {libcall:?}");
diff --git a/crates/unwinder/src/exception_table.rs b/crates/unwinder/src/exception_table.rs
index 2f10f7bb05..93475ea4f3 100644
--- a/crates/unwinder/src/exception_table.rs
+++ b/crates/unwinder/src/exception_table.rs
@@ -17,7 +17,7 @@ use alloc::vec;
use alloc::vec::Vec;
#[cfg(feature = "cranelift")]
use cranelift_codegen::{
- ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, binemit::CodeOffset,
+ ExceptionContextLoc, MachCallSiteItem, MachExceptionHandler, binemit::CodeOffset,
};
use wasmtime_environ::prelude::*;
@@ -127,7 +127,7 @@ impl ExceptionTableBuilder {
pub fn add_func<'a>(
&mut self,
start_offset: CodeOffset,
- call_sites: impl Iterator<Item = FinalizedMachCallSite<'a>>,
+ call_sites: impl Iterator<Item = MachCallSiteItem<'a>>,
) -> Result<()> {
// Ensure that we see functions in offset order.
assert!(start_offset >= self.last_start_offset);
@@ -144,7 +144,8 @@ impl ExceptionTableBuilder {
let mut context = u32::MAX;
for handler in call_site.exception_handlers {
match handler {
- FinalizedMachExceptionHandler::Tag(tag, offset) => {
+ MachExceptionHandler::Tag(tag, label) => {
+ let offset = label.as_offset();
self.tags.push(U32::new(LittleEndian, tag.as_u32()));
self.contexts.push(U32::new(LittleEndian, context));
self.handlers.push(U32::new(
@@ -152,7 +153,8 @@ impl ExceptionTableBuilder {
offset.checked_add(start_offset).unwrap(),
));
}
- FinalizedMachExceptionHandler::Default(offset) => {
+ MachExceptionHandler::Default(label) => {
+ let offset = label.as_offset();
self.tags.push(U32::new(LittleEndian, u32::MAX));
self.contexts.push(U32::new(LittleEndian, context));
self.handlers.push(U32::new(
@@ -160,12 +162,10 @@ impl ExceptionTableBuilder {
offset.checked_add(start_offset).unwrap(),
));
}
- FinalizedMachExceptionHandler::Context(ExceptionContextLoc::SPOffset(
- offset,
- )) => {
+ MachExceptionHandler::Context(ExceptionContextLoc::SPOffset(offset)) => {
context = *offset;
}
- FinalizedMachExceptionHandler::Context(ExceptionContextLoc::GPR(_)) => {
+ MachExceptionHandler::Context(ExceptionContextLoc::GPR(_)) => {
panic!(
"Wasmtime exception unwind info only supports dynamic contexts on the stack"
);
@@ -235,9 +235,8 @@ pub struct ExceptionTable<'a> {
/// Wasmtime exception table item, after parsing.
///
/// Note that this is separately defined from the equivalent type in
-/// Cranelift, `cranelift_codegen::FinalizedMachExceptionHandler`,
-/// because we need this in runtime-only builds when Cranelift is not
-/// included.
+/// Cranelift, `cranelift_codegen::MachExceptionHandler`, because we
+/// need this in runtime-only builds when Cranelift is not included.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExceptionHandler {
/// A tag (arbitrary `u32` identifier from CLIF) or `None` for catch-all.
@@ -388,30 +387,31 @@ fn option_from_u32(value: u32) -> Option<u32> {
#[cfg(all(test, feature = "cranelift"))]
mod test {
use super::*;
+ use cranelift_codegen::LabelOrOffset;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::ExceptionTag;
#[test]
fn serialize_exception_table() {
let callsites = [
- FinalizedMachCallSite {
+ MachCallSiteItem {
ret_addr: 0x10,
frame_offset: None,
exception_handlers: &[
- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(1), 0x20),
- FinalizedMachExceptionHandler::Tag(ExceptionTag::new(2), 0x30),
- FinalizedMachExceptionHandler::Default(0x40),
+ MachExceptionHandler::Tag(ExceptionTag::new(1), LabelOrOffset::offset(0x20)),
+ MachExceptionHandler::Tag(ExceptionTag::new(2), LabelOrOffset::offset(0x30)),
+ MachExceptionHandler::Default(LabelOrOffset::offset(0x40)),
],
},
- FinalizedMachCallSite {
+ MachCallSiteItem {
ret_addr: 0x48,
frame_offset: None,
exception_handlers: &[],
},
- FinalizedMachCallSite {
+ MachCallSiteItem {
ret_addr: 0x50,
frame_offset: Some(0x20),
- exception_handlers: &[FinalizedMachExceptionHandler::Default(0x60)],
+ exception_handlers: &[MachExceptionHandler::Default(LabelOrOffset::offset(0x60))],
},
];
diff --git a/winch/codegen/src/codegen/exceptions.rs b/winch/codegen/src/codegen/exceptions.rs
index 1722abf1f8..3832c9bfa0 100644
--- a/winch/codegen/src/codegen/exceptions.rs
+++ b/winch/codegen/src/codegen/exceptions.rs
@@ -64,8 +64,8 @@ impl HandlerState {
.copied()
.rev()
.map(|(tag, label)| match tag {
- Some(tag) => MachExceptionHandler::Tag(tag, label),
- None => MachExceptionHandler::Default(label),
+ Some(tag) => MachExceptionHandler::Tag(tag, label.into()),
+ None => MachExceptionHandler::Default(label.into()),
})
}
diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs
index 3c7d57fd50..6253bb0c19 100644
--- a/winch/codegen/src/isa/aarch64/asm.rs
+++ b/winch/codegen/src/isa/aarch64/asm.rs
@@ -19,7 +19,7 @@ use cranelift_codegen::isa::aarch64::inst::{
VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp,
};
use cranelift_codegen::{
- Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,
+ MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,
Writable,
ir::{ExternalName, MemFlagsData, SourceLoc, TrapCode, UserExternalNameRef},
isa::aarch64::inst::{
@@ -126,11 +126,12 @@ impl Assembler {
impl Assembler {
/// Return the emitted code.
- pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> {
- let stencil = self
+ pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized {
+ let mut buffer = self
.buffer
.finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut());
- stencil.apply_base_srcloc(loc.unwrap_or_default())
+ buffer.apply_base_srcloc(loc.unwrap_or_default());
+ buffer
}
fn emit(&mut self, inst: Inst) {
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 08b89edd56..7d5ac568d2 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -29,7 +29,7 @@ use crate::{
stack::{TypedReg, Val},
};
use cranelift_codegen::{
- ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,
+ ExceptionContextLoc, MachBufferFinalized, MachExceptionHandler, MachLabel,
binemit::CodeOffset,
ir::{MemFlagsData, RelSourceLoc, SourceLoc, types},
isa::aarch64::{
@@ -573,7 +573,7 @@ impl Masm for MacroAssembler {
Ok(SPOffset::from_u32(self.sp_offset))
}
- fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> {
+ fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized> {
if let Some(patch) = self.stack_max_use_add {
patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut());
}
diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs
index d40a7a6325..750a8c59df 100644
--- a/winch/codegen/src/isa/aarch64/mod.rs
+++ b/winch/codegen/src/isa/aarch64/mod.rs
@@ -10,7 +10,7 @@ use crate::{
stack::Stack,
};
use cranelift_codegen::settings::{self, Flags};
-use cranelift_codegen::{Final, MachBufferFinalized, isa::aarch64::settings as aarch64_settings};
+use cranelift_codegen::{MachBufferFinalized, isa::aarch64::settings as aarch64_settings};
use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};
use masm::MacroAssembler as Aarch64Masm;
use target_lexicon::Triple;
@@ -147,7 +147,7 @@ impl TargetIsa for Aarch64 {
fn emit_unwind_info(
&self,
- _result: &MachBufferFinalized<Final>,
+ _result: &MachBufferFinalized,
_kind: cranelift_codegen::isa::unwind::UnwindInfoKind,
) -> Result<Option<cranelift_codegen::isa::unwind::UnwindInfo>> {
// TODO: should fill this in with an actual implementation
diff --git a/winch/codegen/src/isa/mod.rs b/winch/codegen/src/isa/mod.rs
index 21301fb232..4ed9204c2d 100644
--- a/winch/codegen/src/isa/mod.rs
+++ b/winch/codegen/src/isa/mod.rs
@@ -3,7 +3,7 @@ use core::fmt::Formatter;
use cranelift_codegen::isa::unwind::{UnwindInfo, UnwindInfoKind};
use cranelift_codegen::isa::{CallConv, IsaBuilder};
use cranelift_codegen::settings;
-use cranelift_codegen::{Final, MachBufferFinalized, TextSectionBuilder};
+use cranelift_codegen::{MachBufferFinalized, TextSectionBuilder};
use std::{
error,
fmt::{self, Debug, Display},
@@ -199,7 +199,7 @@ pub trait TargetIsa: Send + Sync {
fn emit_unwind_info(
&self,
- _result: &MachBufferFinalized<Final>,
+ _result: &MachBufferFinalized,
_kind: UnwindInfoKind,
) -> Result<Option<UnwindInfo>>;
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 5f31eaba2f..094c0f1d41 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -10,7 +10,7 @@ use crate::{
reg::writable,
};
use cranelift_codegen::{
- CallInfo, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState,
+ CallInfo, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState,
MachLabel, PatchRegion, Writable,
ir::{ExternalName, MemFlagsData, SourceLoc, TrapCode, Type, UserExternalNameRef, types},
isa::{
@@ -327,11 +327,12 @@ impl Assembler {
}
/// Return the emitted code.
- pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> {
- let stencil = self
+ pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized {
+ let mut buffer = self
.buffer
.finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut());
- stencil.apply_base_srcloc(loc.unwrap_or_default())
+ buffer.apply_base_srcloc(loc.unwrap_or_default());
+ buffer
}
fn emit(&mut self, inst: Inst) {
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index a1c9c456dd..f708ea7bad 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -34,7 +34,7 @@ use crate::{
masm::CalleeKind,
};
use cranelift_codegen::{
- ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,
+ ExceptionContextLoc, MachBufferFinalized, MachExceptionHandler, MachLabel,
binemit::CodeOffset,
ir::{MemFlagsData, RelSourceLoc, SourceLoc},
isa::{
@@ -955,7 +955,7 @@ impl Masm for MacroAssembler {
Ok(())
}
- fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> {
+ fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized> {
if let Some(patch) = self.stack_max_use_add {
patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut());
}
diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs
index 3463724635..20071d8911 100644
--- a/winch/codegen/src/isa/x64/mod.rs
+++ b/winch/codegen/src/isa/x64/mod.rs
@@ -10,7 +10,7 @@ use crate::{
codegen::{BuiltinFunctions, CodeGen, CodeGenContext, FuncEnv, TypeConverter},
};
use cranelift_codegen::settings::{self, Flags};
-use cranelift_codegen::{Final, MachBufferFinalized, isa::x64::settings as x64_settings};
+use cranelift_codegen::{MachBufferFinalized, isa::x64::settings as x64_settings};
use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};
use target_lexicon::Triple;
use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};
@@ -152,7 +152,7 @@ impl TargetIsa for X64 {
fn emit_unwind_info(
&self,
- buffer: &MachBufferFinalized<Final>,
+ buffer: &MachBufferFinalized,
kind: cranelift_codegen::isa::unwind::UnwindInfoKind,
) -> Result<Option<cranelift_codegen::isa::unwind::UnwindInfo>> {
Ok(cranelift_codegen::isa::x64::emit_unwind_info(buffer, kind)?)
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 677117cda1..603bf86dc4 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -6,7 +6,7 @@ use crate::isa::{
reg::{Reg, RegClass, WritableReg, writable},
};
use cranelift_codegen::{
- Final, MachBufferFinalized, MachExceptionHandler, MachLabel,
+ MachBufferFinalized, MachExceptionHandler, MachLabel,
binemit::CodeOffset,
ir::{Endianness, MemFlagsData, RelSourceLoc, SourceLoc, UserExternalNameRef},
};
@@ -1870,7 +1870,7 @@ pub(crate) trait MacroAssembler {
fn push(&mut self, src: Reg, size: OperandSize) -> Result<StackSlot>;
/// Finalize the assembly and return the result.
- fn finalize(self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>>;
+ fn finalize(self, base: Option<SourceLoc>) -> Result<MachBufferFinalized>;
/// Zero a particular register.
fn zero(&mut self, reg: WritableReg) -> Result<()>;