A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
x64: add APX NDD form of 64-bit add (#13916)
Johnnie Birch31ddba9ef0cb
diff --git a/cranelift/assembler-x64/meta/src/dsl.rs b/cranelift/assembler-x64/meta/src/dsl.rsindex 69e7778d38..293d914620 100644--- a/cranelift/assembler-x64/meta/src/dsl.rs+++ b/cranelift/assembler-x64/meta/src/dsl.rs@@ -10,8 +10,8 @@ mod features;pub mod format;pub use custom::{Custom, Customization};+pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};pub use encoding::{Encoding, ModRmKind, OpcodeMod};-pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};pub use encoding::{Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex,};diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rsindex c843e609d7..2d254be9c4 100644--- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs+++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs@@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex {modrm: None,imm: Imm::None,tuple_type,+ apx: None,+ nd: None,+ nf: None,}}@@ -86,6 +89,13 @@ impl Encoding {Encoding::Evex(evex) => evex.opcode,}}++ /// Return whether this encoding sets the APX `ND` ("new data destination")+ /// bit, meaning the architectural destination is the `vvvv`-encoded+ /// register rather than an operand named by ModRM.+ pub fn is_nd(&self) -> bool {+ matches!(self, Encoding::Evex(evex) if evex.nd == Some(true))+ }}impl fmt::Display for Encoding {@@ -841,6 +851,10 @@ pub enum VexEscape {_0F,_0F3A,_0F38,+ /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX)+ /// encodings, never for VEX. This is the map that enables promoting legacy+ /// general-purpose-register instructions into the EVEX space.+ _MAP4,}impl VexEscape {@@ -851,6 +865,7 @@ impl VexEscape {Self::_0F => 0b01,Self::_0F38 => 0b10,Self::_0F3A => 0b11,+ Self::_MAP4 => 0b100,}}}@@ -861,6 +876,7 @@ impl fmt::Display for VexEscape {Self::_0F => write!(f, "0F"),Self::_0F3A => write!(f, "0F3A"),Self::_0F38 => write!(f, "0F38"),+ Self::_MAP4 => write!(f, "MAP4"),}}}@@ -1247,6 +1263,30 @@ pub struct Evex {/// The "Tuple Type" corresponding to scaling of the 8-bit displacement/// parameter for memory operands. See [`TupleType`] for more information.pub tuple_type: TupleType,+ /// The APX "Extended EVEX" class, when this instruction is an APX+ /// promotion.+ ///+ /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX+ /// identifier but does *not* define a single static payload layout.+ /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is+ /// re-mapped depending on which class of instruction is being promoted (see+ /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding;+ /// `Some(_)` selects one of the APX layouts and enables addressing of the+ /// extended general-purpose registers `R16`–`R31` ("EGPR").+ pub apx: Option<ApxClass>,+ /// The APX `ND` (New Data destination) bit.+ ///+ /// When set, a legacy destructive two-operand instruction is promoted into+ /// a non-destructive three-operand form (the extra destination is encoded+ /// in the `EVEX.vvvv` field). `None` when the bit is not part of this+ /// encoding; only valid for [`ApxClass::LegacyGpr`].+ pub nd: Option<bool>,+ /// The APX `NF` (No Flags) bit.+ ///+ /// When set, the status-flag writes that a legacy integer instruction would+ /// otherwise perform are suppressed. `None` when the bit is not part of+ /// this encoding; only valid for [`ApxClass::LegacyGpr`].+ pub nf: Option<bool>,}impl Evex {@@ -1310,6 +1350,27 @@ impl Evex {}}+ /// Select the APX "opcode map 4", promoting a legacy general-purpose-register+ /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the+ /// manual.+ ///+ /// This both sets the `mmm` map bits and marks the encoding as+ /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended+ /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits.+ ///+ /// # Panics+ ///+ /// Panics if the map (`mmm`) or APX class has already been set.+ pub fn map4(self) -> Self {+ assert!(self.mmm.is_none());+ assert!(self.apx.is_none());+ Self {+ mmm: Some(VexEscape::_MAP4),+ apx: Some(ApxClass::LegacyGpr),+ ..self+ }+ }+/// Set the `W` bit to `0`; equivalent to `.W0` in the manual.pub fn w0(self) -> Self {assert!(self.w.is_ignored());@@ -1352,9 +1413,72 @@ impl Evex {}}+ /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`].+ ///+ /// This selects the APX payload layout to emit and enables addressing of+ /// the extended general-purpose registers `R16`–`R31` ("EGPR").+ ///+ /// # Panics+ ///+ /// Panics if an APX class has already been set.+ pub fn apx(self, class: ApxClass) -> Self {+ assert!(self.apx.is_none());+ Self {+ apx: Some(class),+ ..self+ }+ }++ /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the+ /// manual.+ ///+ /// # Panics+ ///+ /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit+ /// has already been set.+ pub fn nd(self) -> Self {+ assert_eq!(+ self.apx,+ Some(ApxClass::LegacyGpr),+ "the ND bit is only valid for APX legacy-GPR promotions"+ );+ assert!(self.nd.is_none());+ Self {+ nd: Some(true),+ ..self+ }+ }++ /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual.+ ///+ /// # Panics+ ///+ /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit+ /// has already been set.+ pub fn nf(self) -> Self {+ assert_eq!(+ self.apx,+ Some(ApxClass::LegacyGpr),+ "the NF bit is only valid for APX legacy-GPR promotions"+ );+ assert!(self.nf.is_none());+ Self {+ nf: Some(true),+ ..self+ }+ }+fn validate(&self, _operands: &[Operand]) {assert!(self.opcode != u8::MAX);assert!(self.mmm.is_some());+ // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions.+ if self.nd.is_some() || self.nf.is_some() {+ assert_eq!(+ self.apx,+ Some(ApxClass::LegacyGpr),+ "the ND/NF bits require an APX legacy-GPR encoding"+ );+ }}/// Retrieve the digit extending the opcode, if available.@@ -1412,7 +1536,14 @@ impl fmt::Display for Evex {if let Some(mmmmm) = self.mmm {write!(f, ".{mmmmm}")?;}- write!(f, ".{} {:#04X}", self.w, self.opcode)?;+ write!(f, ".{}", self.w)?;+ if self.nd == Some(true) {+ write!(f, ".ND")?;+ }+ if self.nf == Some(true) {+ write!(f, ".NF")?;+ }+ write!(f, " {:#04X}", self.opcode)?;if let Some(modrm) = self.modrm {write!(f, " {modrm}")?;}@@ -1423,6 +1554,35 @@ impl fmt::Display for Evex {}}+/// The class of an APX "Extended EVEX" encoding.+///+/// Intel APX does not define a single static Extended-EVEX layout. Although+/// every APX instruction still begins with the `0x62` EVEX identifier, the+/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on+/// the class of instruction being promoted. This enum selects which layout the+/// assembler should emit; all classes gain access to the extended+/// general-purpose registers `R16`–`R31` ("EGPR").+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub enum ApxClass {+ /// Promotion of a legacy general-purpose-register (GPR) instruction using+ /// extended opcode map 4. Only this class may set the `ND` and `NF` bits.+ LegacyGpr,+ /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space.+ Vector,+ /// An existing AVX-512 (EVEX) instruction extended with APX register bits.+ Avx512,+}++impl fmt::Display for ApxClass {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match self {+ Self::LegacyGpr => write!(f, "LegacyGpr"),+ Self::Vector => write!(f, "Vector"),+ Self::Avx512 => write!(f, "Avx512"),+ }+ }+}+/// Tuple Type definitions used in EVEX encodings.////// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual.diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rsindex d7cfdbe5eb..e9266d7db5 100644--- a/cranelift/assembler-x64/meta/src/dsl/features.rs+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs@@ -98,6 +98,7 @@ pub enum Feature {fma,avx_vnni,avx512vnni,+ apx,}/// List all CPU features.@@ -131,6 +132,7 @@ pub const ALL_FEATURES: &[Feature] = &[Feature::fma,Feature::avx_vnni,Feature::avx512vnni,+ Feature::apx,];impl fmt::Display for Feature {diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rsindex e1e4a8bb3b..6cb6c1cd5b 100644--- a/cranelift/assembler-x64/meta/src/generate/format.rs+++ b/cranelift/assembler-x64/meta/src/generate/format.rs@@ -54,29 +54,46 @@ impl dsl::Format {/// once Cranelift has switched to using this assembler predominantly/// (TODO).#[must_use]- pub(crate) fn generate_att_style_operands(&self) -> String {- let ordered_ops: Vec<_> = self- .operands- .iter()- .filter(|o| !o.implicit)- .rev()- .map(|o| format!("{{{}}}", o.location))- .collect();- ordered_ops.join(", ")+ pub(crate) fn generate_att_style_operands(&self, nd: bool) -> String {+ self.ordered_operands(nd, false)}/// Like [`Self::generate_att_style_operands`], but omits the fixed `%xmm0`/// mask operand, which XED leaves implicit.#[must_use]- pub(crate) fn generate_xed_style_operands(&self) -> String {- let ordered_ops: Vec<_> = self+ pub(crate) fn generate_xed_style_operands(&self, nd: bool) -> String {+ self.ordered_operands(nd, true)+ }++ /// Shared operand ordering for AT&T-style printing.+ ///+ /// Normally this is just the reverse of the DSL's Intel-style order. APX+ /// "new data destination" (NDD) forms are the exception: their+ /// architectural destination is the `vvvv`-encoded register, which the DSL+ /// must list in the middle so that the positional slot assignment in+ /// `generate_vex_or_evex_prefix` maps operands to `reg`/`vvvv`/`rm`+ /// correctly. Blindly reversing would therefore print the destination in+ /// the middle. Instead, for `ND = 1` the sources keep their DSL order and+ /// the destination is printed last, which matches both AT&T convention and+ /// the XED disassembly used as a fuzzing oracle.+ #[must_use]+ fn ordered_operands(&self, nd: bool, xed_style: bool) -> String {+ let ops = self.operands.iter()- .filter(|o| !o.implicit && o.location != dsl::Location::xmm0)- .rev()+ .filter(|o| !o.implicit && !(xed_style && o.location == dsl::Location::xmm0));+ let ordered: Vec<&dsl::Operand> = if nd {+ let (dst, srcs): (Vec<_>, Vec<_>) =+ ops.partition(|o| matches!(o.mutability, dsl::Mutability::Write));+ srcs.into_iter().chain(dst).collect()+ } else {+ ops.rev().collect()+ };+ ordered+ .into_iter().map(|o| format!("{{{}}}", o.location))- .collect();- ordered_ops.join(", ")+ .collect::<Vec<_>>()+ .join(", ")}#[must_use]@@ -243,24 +260,51 @@ impl dsl::Format {fmtln!(f, "let w = {};", vex.w.as_bool());let bits = "len, pp, mmmmm, w";- self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || {- vex.unwrap_digit()- })+ self.generate_vex_or_evex_prefix(+ f,+ "VexPrefix",+ &bits,+ vex.is4,+ None,+ "two_op",+ "three_op",+ || vex.unwrap_digit(),+ )}fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle {f.empty_line();f.comment("Emit EVEX prefix.");- let ll = evex.length.evex_bits();- fmtln!(f, "let ll = {ll:#04b};");++ // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a+ // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime+ // assembler and section 3.1.2.3.1 of the APX spec). In that case we+ // emit the ND/NF bits and select the `legacy_*` constructors instead of+ // the AVX-512 ones.+ let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr));+fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits()));fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits());fmtln!(f, "let w = {};", evex.w.as_bool());- // NB: when bcast is supported in the future the `evex_scaling`- // calculation for `Full` and `Half` below need to be updated.++ let (bits, two_op, three_op);+ if apx_legacy {+ fmtln!(f, "let nd = {};", evex.nd == Some(true));+ fmtln!(f, "let nf = {};", evex.nf == Some(true));+ bits = String::from("pp, mmm, w, nd, nf");+ two_op = "legacy_two_op";+ three_op = "legacy_three_op";+ } else {+ let ll = evex.length.evex_bits();+ fmtln!(f, "let ll = {ll:#04b};");+ // NB: when bcast is supported in the future the `evex_scaling`+ // calculation for `Full` and `Half` below need to be updated.+ fmtln!(f, "let bcast = false;");+ bits = String::from("ll, pp, mmm, w, bcast");+ two_op = "two_op";+ three_op = "three_op";+ }let bcast = false;- fmtln!(f, "let bcast = {bcast};");- let bits = format!("ll, pp, mmm, w, bcast");let is4 = false;let length_bytes = match evex.length {@@ -273,39 +317,59 @@ impl dsl::Format {// Figure out, according to table 2-34 and 2-35 in the Intel manual,// what the scaling factor is for 8-bit displacements to pass through to// encoding.- let evex_scaling = Some(match evex.tuple_type {- dsl::TupleType::Full => {- assert!(!bcast);- length_bytes- }- dsl::TupleType::Half => {- assert!(!bcast);- length_bytes / 2- }- dsl::TupleType::FullMem => length_bytes,- // FIXME: according to table 2-35 this needs to take into account- // "InputSize" which isn't accounted for in our `Evex` structure at- // this time.- dsl::TupleType::Tuple1Scalar => unimplemented!(),- dsl::TupleType::Tuple1Fixed => unimplemented!(),- dsl::TupleType::Tuple2 => unimplemented!(),- dsl::TupleType::Tuple4 => unimplemented!(),- dsl::TupleType::Tuple8 => 32,- dsl::TupleType::HalfMem => length_bytes / 2,- dsl::TupleType::QuarterMem => length_bytes / 4,- dsl::TupleType::EigthMem => length_bytes / 8,- dsl::TupleType::Mem128 => 16,- dsl::TupleType::Movddup => match evex.length {- dsl::Length::LZ | dsl::Length::LIG => unimplemented!(),- dsl::Length::L128 => 8,- dsl::Length::L256 => 32,- dsl::Length::L512 => 64,- },- });+ //+ // The compressed-displacement scheme of section 2.7.5 only applies to+ // the vector EVEX encodings. APX promotes legacy general-purpose-+ // register instructions into extended-EVEX "map 4", and those keep+ // legacy displacement semantics: `disp8` is a plain byte offset rather+ // than a multiple of a tuple-derived scaling factor `N`. Scaling them+ // would emit an offset wrong by a factor of `N`. Note this is specific+ // to the legacy-GPR class; promoted vector instructions and APX-extended+ // AVX-512 instructions still use compressed displacements.+ let evex_scaling = if matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)) {+ None+ } else {+ Some(match evex.tuple_type {+ dsl::TupleType::Full => {+ assert!(!bcast);+ length_bytes+ }+ dsl::TupleType::Half => {+ assert!(!bcast);+ length_bytes / 2+ }+ dsl::TupleType::FullMem => length_bytes,+ // FIXME: according to table 2-35 this needs to take into account+ // "InputSize" which isn't accounted for in our `Evex` structure at+ // this time.+ dsl::TupleType::Tuple1Scalar => unimplemented!(),+ dsl::TupleType::Tuple1Fixed => unimplemented!(),+ dsl::TupleType::Tuple2 => unimplemented!(),+ dsl::TupleType::Tuple4 => unimplemented!(),+ dsl::TupleType::Tuple8 => 32,+ dsl::TupleType::HalfMem => length_bytes / 2,+ dsl::TupleType::QuarterMem => length_bytes / 4,+ dsl::TupleType::EigthMem => length_bytes / 8,+ dsl::TupleType::Mem128 => 16,+ dsl::TupleType::Movddup => match evex.length {+ dsl::Length::LZ | dsl::Length::LIG => unimplemented!(),+ dsl::Length::L128 => 8,+ dsl::Length::L256 => 32,+ dsl::Length::L512 => 64,+ },+ })+ };- self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || {- evex.unwrap_digit()- })+ self.generate_vex_or_evex_prefix(+ f,+ "EvexPrefix",+ &bits,+ is4,+ evex_scaling,+ two_op,+ three_op,+ || evex.unwrap_digit(),+ )}/// Helper function to generate either a vex or evex prefix, mostly handling@@ -318,6 +382,8 @@ impl dsl::Format {bits: &str,is4: bool,evex_scaling: Option<i8>,+ two_op: &str,+ three_op: &str,unwrap_digit: impl Fn() -> Option<u8>,) -> ModRmStyle {use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem};@@ -330,7 +396,7 @@ impl dsl::Format {fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");fmtln!(f,- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});");ModRmStyle::Reg {reg: ModRmReg::Reg(*reg),@@ -347,7 +413,7 @@ impl dsl::Format {fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");fmtln!(f,- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});");ModRmStyle::RegMem {reg: ModRmReg::Reg(*reg),@@ -362,7 +428,7 @@ impl dsl::Format {fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");fmtln!(f,- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});");ModRmStyle::RegMemIs4 {reg: ModRmReg::Reg(*reg),@@ -382,7 +448,7 @@ impl dsl::Format {fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");fmtln!(f,- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});");ModRmStyle::RegMem {reg: ModRmReg::Digit(digit),@@ -395,7 +461,7 @@ impl dsl::Format {let reg = reg_or_vvvv;fmtln!(f, "let reg = self.{reg}.enc();");fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");ModRmStyle::RegMem {reg: ModRmReg::Reg(*reg),rm: *rm,@@ -413,7 +479,7 @@ impl dsl::Format {fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");fmtln!(f,- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});");ModRmStyle::Reg {reg: ModRmReg::Digit(digit),@@ -425,7 +491,7 @@ impl dsl::Format {let reg = reg_or_vvvv;fmtln!(f, "let reg = self.{reg}.enc();");fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");ModRmStyle::Reg {reg: ModRmReg::Reg(*reg),rm: *rm,@@ -437,7 +503,7 @@ impl dsl::Format {assert!(!is4);fmtln!(f, "let reg = self.{reg}.enc();");fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");ModRmStyle::RegMem {reg: ModRmReg::Reg(*reg),rm: *rm,diff --git a/cranelift/assembler-x64/meta/src/generate/inst.rs b/cranelift/assembler-x64/meta/src/generate/inst.rsindex a2eb137356..71ec243abd 100644--- a/cranelift/assembler-x64/meta/src/generate/inst.rs+++ b/cranelift/assembler-x64/meta/src/generate/inst.rs@@ -324,8 +324,9 @@ impl dsl::Inst {None => fmtln!(f, "let {location} = {to_string};"),}}- let ordered_ops = self.format.generate_att_style_operands();- let xed_ops = self.format.generate_xed_style_operands();+ let nd = self.encoding.is_nd();+ let ordered_ops = self.format.generate_att_style_operands(nd);+ let xed_ops = self.format.generate_xed_style_operands(nd);let mut implicit_ops = self.format.generate_implicit_operands();if self.has_trap {fmtln!(f, "let trap = self.trap;");diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rsindex f5969061b0..f4b6b4fb29 100644--- a/cranelift/assembler-x64/meta/src/instructions/add.rs+++ b/cranelift/assembler-x64/meta/src/instructions/add.rs@@ -97,5 +97,12 @@ pub fn list() -> Vec<Inst> {inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx),inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx),inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl),+ // APX: NDD (new data destination) form of `ADD`, promoted into EVEX+ // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural+ // destination is the `vvvv`-encoded register, so the written operand is+ // placed in the `V` slot (the middle operand of the `RVM` format) while+ // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so+ // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form.+ inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx),]}diff --git a/cranelift/assembler-x64/src/evex.rs b/cranelift/assembler-x64/src/evex.rsindex bf89dd13bd..2734fae7ac 100644--- a/cranelift/assembler-x64/src/evex.rs+++ b/cranelift/assembler-x64/src/evex.rs@@ -92,6 +92,107 @@ impl EvexPrefix {EvexPrefix::new(reg, vvvv, (b, x), ll, pp, mmm, w, broadcast)}+ // ---------------------------------------------------------------------+ // Intel APX "Extended EVEX" prefix for promoted *legacy* GPR instructions+ // (EVEX map 4). See the Intel APX Architecture Specification (rev 8),+ // section 3.1.2.3.1 "EVEX Extension of Legacy Instructions", Figure 3.3.+ //+ // The four bytes still begin with `0x62`, but several payload bits are+ // re-purposed relative to the AVX-512 layout above:+ //+ // ┌────┬────┬────┬────┬────┬────┬────┬────┐+ // Byte 1: │ R3 │ X3 │ B3 │ R4 │ B4 │ 1 │ 0 │ 0 │ (map id = 4)+ // ├────┼────┼────┼────┼────┼────┼────┼────┤+ // Byte 2: │ W │ V3 │ V2 │ V1 │ V0 │ U │ p │ p │ (U = ~X4)+ // ├────┼────┼────┼────┼────┼────┼────┼────┤+ // Byte 3: │ 0 │ 0 │ 0 │ ND │ V4 │ NF │ 0 │ 0 │+ // └────┴────┴────┴────┴────┴────┴────┴────┘+ //+ // The "underlined" fields (`R3`, `X3`, `B3`, `R4`, the `vvvv` bits and `V4`)+ // are stored inverted, exactly as in the AVX-512 layout. `B4` and `X4` are+ // newly repurposed reserved bits: `B4` uses *true* polarity (fixed value 0)+ // and `X4` is carried inverted in the `U` bit (`EVEX.X4 = ~EVEX.U`), so a+ // register-form instruction (ModRM.Mod = 3, no index) has `U = 1`.++ /// Construct the extended-EVEX (APX map 4) prefix for a legacy GPR+ /// instruction.+ ///+ /// `reg` is the ModRM.reg register, `vvvv` is the `V` register identifier+ /// (the NDD register when `nd` is set), and `(b, x)` are the ModRM.r/m base+ /// and (optional) SIB index registers. `nd`/`nf` select the New Data+ /// destination and No Flags bits respectively.+ pub fn legacy(+ reg: u8,+ vvvv: u8,+ (b, x): (Option<u8>, Option<u8>),+ pp: u8,+ mmm: u8,+ w: bool,+ nd: bool,+ nf: bool,+ ) -> Self {+ let base = b.unwrap_or(0);+ let index = x.unwrap_or(0);++ // byte1 (P0)+ let r3 = invert_top_bit(reg);+ let x3 = invert_top_bit(index);+ let b3 = invert_top_bit(base);+ let r4 = invert_top_bit(reg >> 1);+ let b4 = (base >> 4) & 1; // true polarity+ debug_assert!(mmm <= 0b111);+ let byte1 = r3 << 7 | x3 << 6 | b3 << 5 | r4 << 4 | b4 << 3 | mmm;++ // byte2 (P1)+ debug_assert!(vvvv <= 0b11111);+ debug_assert!(pp <= 0b11);+ let vvvv_value = !vvvv & 0b1111;+ // `EVEX.X4 = ~EVEX.U`; with no index register X4 = 0 so U = 1.+ let x4 = (index >> 4) & 1;+ let u = (!x4) & 1;+ let byte2 = (w as u8) << 7 | vvvv_value << 3 | u << 2 | (pp & 0b11);++ // byte3 (P2)+ let v_prime = invert_top_bit(vvvv >> 1); // V4, inverted+ let byte3 = (nd as u8) << 4 | v_prime << 3 | (nf as u8) << 2;++ Self {+ byte1,+ byte2,+ byte3,+ }+ }++ /// Construct the extended-EVEX (APX map 4) prefix for a two-operand legacy+ /// GPR instruction (no NDD); the `V` register identifier is unused.+ #[allow(dead_code, reason = "not all APX legacy forms are emitted yet")]+ pub fn legacy_two_op(+ reg: u8,+ (b, x): (Option<u8>, Option<u8>),+ pp: u8,+ mmm: u8,+ w: bool,+ nd: bool,+ nf: bool,+ ) -> Self {+ EvexPrefix::legacy(reg, 0, (b, x), pp, mmm, w, nd, nf)+ }++ /// Construct the extended-EVEX (APX map 4) prefix for a three-operand+ /// legacy GPR instruction; `vvvv` carries the NDD register.+ pub fn legacy_three_op(+ reg: u8,+ vvvv: u8,+ (b, x): (Option<u8>, Option<u8>),+ pp: u8,+ mmm: u8,+ w: bool,+ nd: bool,+ nf: bool,+ ) -> Self {+ EvexPrefix::legacy(reg, vvvv, (b, x), pp, mmm, w, nd, nf)+ }+pub(crate) fn encode(&self, sink: &mut impl CodeSink) {sink.put1(0x62);sink.put1(self.byte1);diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rsindex 3efe0acfc4..99166feca7 100644--- a/cranelift/assembler-x64/src/fuzz.rs+++ b/cranelift/assembler-x64/src/fuzz.rs@@ -34,6 +34,13 @@ pub fn roundtrip(inst: &Inst<FuzzRegs>) {return;}+ // Likewise, capstone cannot decode the APX extended-EVEX ("map 4")+ // encodings at all, returning zero instructions. These are instead checked+ // against XED by `roundtrip_xed`, which does understand map 4.+ if features_mention(inst.features(), Feature::apx) {+ return;+ }+roundtrip_with(inst,"capstone",@@ -589,4 +596,62 @@ mod test {}).budget_ms(1_000);}++ /// Byte-level encoding check for the APX NDD (new-data-destination) form of+ /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix.+ ///+ /// We assert the exact bytes rather than round-tripping through Capstone+ /// because the bundled disassembler does not yet understand APX. With+ /// `ND = 1` the architectural destination is the `vvvv`-encoded register, so+ /// the `RVM` operands are `[ModRM.reg source, vvvv destination, ModRM.rm+ /// source]`. For `addq %rax, %rcx, %rdx` (destination `%rax` = 0 in `vvvv`,+ /// source `%rcx` = 1 in ModRM.reg, source `%rdx` = 2 in ModRM.rm), `W = 1`,+ /// `ND = 1`, `NF = 0`, the expected sequence is:+ ///+ /// ```text+ /// 62 F4 FC 18 01 CA+ /// ^^ ^^ ^^ ^^ ^^ ^^+ /// | | | | | └ ModRM: mod=11 reg=rcx r/m=rdx+ /// | | | | └ opcode 0x01 (ADD r/m, reg)+ /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0+ /// | | └ P1: W=1, vvvv=~rax=1111, U=1, pp=00+ /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4)+ /// └ EVEX identifier+ /// ```+ #[test]+ fn apx_addq_rvm_ndd_encoding() {+ use crate::inst::addq_rvm;+ // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm+ // source], so the constructor arguments are (reg source = %rcx,+ // destination = %rax, r/m source = %rdx).+ let inst = addq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA");+ }++ /// Companion to [`apx_addq_rvm_ndd_encoding`] covering a memory operand.+ ///+ /// APX map-4 instructions are legacy instructions promoted into EVEX, so+ /// their `disp8` keeps legacy semantics: a plain byte offset. The+ /// compressed-displacement scheme that divides `disp8` by a tuple-derived+ /// factor `N` applies only to the vector EVEX encodings. Encoding+ /// `0x50(%rdi)` must therefore emit `0x50` and not `0x50 / 16 = 0x05`,+ /// which would silently address the wrong memory.+ #[test]+ fn apx_addq_rvm_ndd_disp8_is_unscaled() {+ use crate::inst::addq_rvm;+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};++ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {+ base: FuzzReg::new(7),+ simm32: AmodeOffsetPlusKnownOffset {+ simm32: AmodeOffset::new(0x50),+ offset: None,+ },+ trap: None,+ });+ let inst = addq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50");+ }}diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rsindex d72be07de2..836338027f 100644--- a/cranelift/codegen/meta/src/isa/x86.rs+++ b/cranelift/codegen/meta/src/isa/x86.rs@@ -95,6 +95,15 @@ pub(crate) fn define() -> TargetIsa {"AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]",false,);+ // Registered so `target x86_64 has_apx` parses; not yet referenced by any+ // preset because no shipping microarchitecture ships APX today. Drop the+ // leading underscore once a preset needs it.+ let _has_apx = settings.add_bool(+ "has_apx",+ "Has support for APX.",+ "APX_F: CPUID.(EAX=07H,ECX=1H):EDX.APX_F[bit 21]",+ false,+ );let has_popcnt = settings.add_bool("has_popcnt","Has support for POPCNT.",diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex 24c1d7ded8..62d14633ac 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -1336,6 +1336,9 @@(decl pure use_avx2 () bool)(extern constructor use_avx2 use_avx2)+(decl pure use_apx () bool)+(extern constructor use_apx use_apx)+(decl pure has_cmpxchg16b () bool)(extern constructor has_cmpxchg16b has_cmpxchg16b)diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex 3f2b6aa4ac..1ee94966b3 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1595,6 +1595,10 @@ impl asm::AvailableFeatures for &EmitInfo {fn avx512vnni(&self) -> bool {self.isa_flags.has_avx512vnni()}++ fn apx(&self) -> bool {+ self.isa_flags.has_apx()+ }}impl MachInstEmit for Inst {diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex ed9b8fac38..617616445f 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -94,6 +94,15 @@(rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y))(x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset))))+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-+;; destination) form of `add` for 64-bit adds. Unlike the legacy two-operand+;; `add`, the NDD form writes its result to a fresh register encoded in the+;; EVEX `vvvv` field, so the register allocator does not need to insert a move+;; to preserve either input.+(rule iadd_apx_ndd 1 (lower (iadd $I64 x y))+ (if-let true (use_apx))+ (x64_addq_rvm x y))+;; Higher-priority cases than the previous two where a load can be sunk into;; the add instruction itself. Note that both operands are tested for;; sink-ability since addition is commutativediff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rsindex 8c04e7f893..c5b33e1623 100644--- a/cranelift/codegen/src/isa/x64/lower/isle.rs+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs@@ -492,6 +492,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {self.backend.x64_flags.has_avx512vbmi()}+ #[inline]+ fn use_apx(&mut self) -> bool {+ self.backend.x64_flags.has_apx()+ }+#[inline]fn has_lzcnt(&mut self) -> bool {self.backend.x64_flags.has_lzcnt()diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clifnew file mode 100644index 0000000000..94fcbfd180--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif@@ -0,0 +1,26 @@+test compile precise-output+target x86_64 has_apx++function %add_i64(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ v2 = iadd v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; addq %rdi, %rsi, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x01, 0xfe, 0x48, 0x89+; .byte 0xec, 0x5d, 0xc3+