Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: Add bitwidth-specific newtypes for registers. (#13184)

Chris Fallindfbe99771566
diff --git a/cranelift/codegen/meta/src/gen_asm.rs b/cranelift/codegen/meta/src/gen_asm.rs
index 30d42d808d..8a70b26895 100644
--- a/cranelift/codegen/meta/src/gen_asm.rs
+++ b/cranelift/codegen/meta/src/gen_asm.rs
@@ -19,6 +19,9 @@ fn include_inst(inst: &Inst) -> bool {
}
/// Returns the Rust type used for the `IsleConstructorRaw` variants.
+///
+/// RegMem operand types are width-tagged based on the DSL operand width
+/// (e.g. `rm8` → `GprMem8`, `xmm_m128` with `align` → `XmmMemAligned128`).
fn rust_param_raw(op: &Operand) -> String {
match op.location.kind() {
OperandKind::Imm(loc) => {
@@ -32,7 +35,8 @@ fn rust_param_raw(op: &Operand) -> String {
OperandKind::RegMem(rm) => {
let reg = rm.reg_class().unwrap();
let aligned = if op.align { "Aligned" } else { "" };
- format!("&{reg}Mem{aligned}")
+ let bits = rm.bits();
+ format!("&{reg}Mem{aligned}{bits}")
}
OperandKind::Mem(_) => {
format!("&SyntheticAmode")
@@ -243,7 +247,8 @@ pub fn generate_rust_macro(f: &mut Formatter, insts: &[Inst]) {
}
/// Returns the type of this operand in ISLE as a part of the ISLE "raw"
-/// constructors.
+/// constructors. RegMem operand types are width-tagged based on the DSL
+/// operand width; see [`rust_param_raw`].
fn isle_param_raw(op: &Operand) -> String {
match op.location.kind() {
OperandKind::Imm(loc) => {
@@ -265,7 +270,8 @@ fn isle_param_raw(op: &Operand) -> String {
OperandKind::RegMem(rm) => {
let reg = rm.reg_class().unwrap();
let aligned = if op.align { "Aligned" } else { "" };
- format!("{reg}Mem{aligned}")
+ let bits = rm.bits();
+ format!("{reg}Mem{aligned}{bits}")
}
}
}
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 1140773e81..ef8954dda8 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -694,15 +694,43 @@
(type WritableGpr (primitive WritableGpr))
(type OptionWritableGpr (primitive OptionWritableGpr))
(type GprMem extern (enum))
+(type GprMem8 extern (enum))
+(type GprMem16 extern (enum))
+(type GprMem32 extern (enum))
+(type GprMem64 extern (enum))
(type GprMemImm extern (enum))
+(type GprMemImm8 extern (enum))
+(type GprMemImm16 extern (enum))
+(type GprMemImm32 extern (enum))
+(type GprMemImm64 extern (enum))
(type Xmm (primitive Xmm))
(type WritableXmm (primitive WritableXmm))
(type OptionWritableXmm (primitive OptionWritableXmm))
(type XmmMem extern (enum))
+(type XmmMem8 extern (enum))
+(type XmmMem16 extern (enum))
+(type XmmMem32 extern (enum))
+(type XmmMem64 extern (enum))
+(type XmmMem128 extern (enum))
(type XmmMemAligned extern (enum))
+(type XmmMemAligned8 extern (enum))
+(type XmmMemAligned16 extern (enum))
+(type XmmMemAligned32 extern (enum))
+(type XmmMemAligned64 extern (enum))
+(type XmmMemAligned128 extern (enum))
(type XmmMemImm extern (enum))
+(type XmmMemImm8 extern (enum))
+(type XmmMemImm16 extern (enum))
+(type XmmMemImm32 extern (enum))
+(type XmmMemImm64 extern (enum))
+(type XmmMemImm128 extern (enum))
(type XmmMemAlignedImm extern (enum))
+(type XmmMemAlignedImm8 extern (enum))
+(type XmmMemAlignedImm16 extern (enum))
+(type XmmMemAlignedImm32 extern (enum))
+(type XmmMemAlignedImm64 extern (enum))
+(type XmmMemAlignedImm128 extern (enum))
;; Convert a `WritableGpr` to a `WritableReg`.
(decl writable_gpr_to_reg (WritableGpr) WritableReg)
@@ -874,6 +902,211 @@
(rule (put_in_gpr_mem_imm val)
(gpr_mem_imm_new (put_in_reg_mem_imm val)))
+;; Sized operand helpers.
+;;
+;; It is possible to convert:
+;; * Value -> sized operand (checks size dynamically)
+;; * Reg -> sized operand (the size is a check for the memory operand
+;; size; it's fine to upsize a register)
+;; * immediate -> sized operand (likewise)
+;; * sized operand -> unsized operand (it's always safe to drop
+;; the type-level information)
+
+(decl put_in_gpr_mem_8 (Value) GprMem8)
+(extern constructor put_in_gpr_mem_8 put_in_gpr_mem_8)
+(decl put_in_gpr_mem_imm_8 (Value) GprMemImm8)
+(extern constructor put_in_gpr_mem_imm_8 put_in_gpr_mem_imm_8)
+(decl gpr_to_gpr_mem_8 (Gpr) GprMem8)
+(extern constructor gpr_to_gpr_mem_8 gpr_to_gpr_mem_8)
+(decl gpr_to_gpr_mem_imm_8 (Gpr) GprMemImm8)
+(extern constructor gpr_to_gpr_mem_imm_8 gpr_to_gpr_mem_imm_8)
+(decl gpr_mem_imm_8_imm (u32) GprMemImm8)
+(extern constructor gpr_mem_imm_8_imm gpr_mem_imm_8_imm)
+(decl gpr_mem_8_to_gpr_mem (GprMem8) GprMem)
+(extern constructor gpr_mem_8_to_gpr_mem gpr_mem_8_to_gpr_mem)
+(decl gpr_mem_imm_8_to_gpr_mem_imm (GprMemImm8) GprMemImm)
+(extern constructor gpr_mem_imm_8_to_gpr_mem_imm gpr_mem_imm_8_to_gpr_mem_imm)
+
+(decl put_in_gpr_mem_16 (Value) GprMem16)
+(extern constructor put_in_gpr_mem_16 put_in_gpr_mem_16)
+(decl put_in_gpr_mem_imm_16 (Value) GprMemImm16)
+(extern constructor put_in_gpr_mem_imm_16 put_in_gpr_mem_imm_16)
+(decl gpr_to_gpr_mem_16 (Gpr) GprMem16)
+(extern constructor gpr_to_gpr_mem_16 gpr_to_gpr_mem_16)
+(decl gpr_to_gpr_mem_imm_16 (Gpr) GprMemImm16)
+(extern constructor gpr_to_gpr_mem_imm_16 gpr_to_gpr_mem_imm_16)
+(decl gpr_mem_imm_16_imm (u32) GprMemImm16)
+(extern constructor gpr_mem_imm_16_imm gpr_mem_imm_16_imm)
+(decl gpr_mem_16_to_gpr_mem (GprMem16) GprMem)
+(extern constructor gpr_mem_16_to_gpr_mem gpr_mem_16_to_gpr_mem)
+(decl gpr_mem_imm_16_to_gpr_mem_imm (GprMemImm16) GprMemImm)
+(extern constructor gpr_mem_imm_16_to_gpr_mem_imm gpr_mem_imm_16_to_gpr_mem_imm)
+
+(decl put_in_gpr_mem_32 (Value) GprMem32)
+(extern constructor put_in_gpr_mem_32 put_in_gpr_mem_32)
+(decl put_in_gpr_mem_imm_32 (Value) GprMemImm32)
+(extern constructor put_in_gpr_mem_imm_32 put_in_gpr_mem_imm_32)
+(decl gpr_to_gpr_mem_32 (Gpr) GprMem32)
+(extern constructor gpr_to_gpr_mem_32 gpr_to_gpr_mem_32)
+(decl gpr_to_gpr_mem_imm_32 (Gpr) GprMemImm32)
+(extern constructor gpr_to_gpr_mem_imm_32 gpr_to_gpr_mem_imm_32)
+(decl gpr_mem_imm_32_imm (u32) GprMemImm32)
+(extern constructor gpr_mem_imm_32_imm gpr_mem_imm_32_imm)
+(decl gpr_mem_32_to_gpr_mem (GprMem32) GprMem)
+(extern constructor gpr_mem_32_to_gpr_mem gpr_mem_32_to_gpr_mem)
+(decl gpr_mem_imm_32_to_gpr_mem_imm (GprMemImm32) GprMemImm)
+(extern constructor gpr_mem_imm_32_to_gpr_mem_imm gpr_mem_imm_32_to_gpr_mem_imm)
+
+(decl put_in_gpr_mem_64 (Value) GprMem64)
+(extern constructor put_in_gpr_mem_64 put_in_gpr_mem_64)
+(decl put_in_gpr_mem_imm_64 (Value) GprMemImm64)
+(extern constructor put_in_gpr_mem_imm_64 put_in_gpr_mem_imm_64)
+(decl gpr_to_gpr_mem_64 (Gpr) GprMem64)
+(extern constructor gpr_to_gpr_mem_64 gpr_to_gpr_mem_64)
+(decl gpr_to_gpr_mem_imm_64 (Gpr) GprMemImm64)
+(extern constructor gpr_to_gpr_mem_imm_64 gpr_to_gpr_mem_imm_64)
+(decl gpr_mem_imm_64_imm (u32) GprMemImm64)
+(extern constructor gpr_mem_imm_64_imm gpr_mem_imm_64_imm)
+(decl gpr_mem_64_to_gpr_mem (GprMem64) GprMem)
+(extern constructor gpr_mem_64_to_gpr_mem gpr_mem_64_to_gpr_mem)
+(decl gpr_mem_imm_64_to_gpr_mem_imm (GprMemImm64) GprMemImm)
+(extern constructor gpr_mem_imm_64_to_gpr_mem_imm gpr_mem_imm_64_to_gpr_mem_imm)
+
+(decl put_in_xmm_mem_8 (Value) XmmMem8)
+(extern constructor put_in_xmm_mem_8 put_in_xmm_mem_8)
+(decl put_in_xmm_mem_aligned_8 (Value) XmmMemAligned8)
+(extern constructor put_in_xmm_mem_aligned_8 put_in_xmm_mem_aligned_8)
+(decl put_in_xmm_mem_imm_8 (Value) XmmMemImm8)
+(extern constructor put_in_xmm_mem_imm_8 put_in_xmm_mem_imm_8)
+(decl put_in_xmm_mem_aligned_imm_8 (Value) XmmMemAlignedImm8)
+(extern constructor put_in_xmm_mem_aligned_imm_8 put_in_xmm_mem_aligned_imm_8)
+(decl xmm_to_xmm_mem_8 (Xmm) XmmMem8)
+(extern constructor xmm_to_xmm_mem_8 xmm_to_xmm_mem_8)
+(decl xmm_to_xmm_mem_aligned_8 (Xmm) XmmMemAligned8)
+(extern constructor xmm_to_xmm_mem_aligned_8 xmm_to_xmm_mem_aligned_8)
+(decl xmm_to_xmm_mem_imm_8 (Xmm) XmmMemImm8)
+(extern constructor xmm_to_xmm_mem_imm_8 xmm_to_xmm_mem_imm_8)
+(decl xmm_to_xmm_mem_aligned_imm_8 (Xmm) XmmMemAlignedImm8)
+(extern constructor xmm_to_xmm_mem_aligned_imm_8 xmm_to_xmm_mem_aligned_imm_8)
+(decl xmm_mem_imm_8_imm (u32) XmmMemImm8)
+(extern constructor xmm_mem_imm_8_imm xmm_mem_imm_8_imm)
+(decl xmm_mem_8_to_xmm_mem (XmmMem8) XmmMem)
+(extern constructor xmm_mem_8_to_xmm_mem xmm_mem_8_to_xmm_mem)
+(decl xmm_mem_aligned_8_to_xmm_mem_aligned (XmmMemAligned8) XmmMemAligned)
+(extern constructor xmm_mem_aligned_8_to_xmm_mem_aligned xmm_mem_aligned_8_to_xmm_mem_aligned)
+(decl xmm_mem_imm_8_to_xmm_mem_imm (XmmMemImm8) XmmMemImm)
+(extern constructor xmm_mem_imm_8_to_xmm_mem_imm xmm_mem_imm_8_to_xmm_mem_imm)
+(decl xmm_mem_aligned_imm_8_to_xmm_mem_aligned_imm (XmmMemAlignedImm8) XmmMemAlignedImm)
+(extern constructor xmm_mem_aligned_imm_8_to_xmm_mem_aligned_imm xmm_mem_aligned_imm_8_to_xmm_mem_aligned_imm)
+
+(decl put_in_xmm_mem_16 (Value) XmmMem16)
+(extern constructor put_in_xmm_mem_16 put_in_xmm_mem_16)
+(decl put_in_xmm_mem_aligned_16 (Value) XmmMemAligned16)
+(extern constructor put_in_xmm_mem_aligned_16 put_in_xmm_mem_aligned_16)
+(decl put_in_xmm_mem_imm_16 (Value) XmmMemImm16)
+(extern constructor put_in_xmm_mem_imm_16 put_in_xmm_mem_imm_16)
+(decl put_in_xmm_mem_aligned_imm_16 (Value) XmmMemAlignedImm16)
+(extern constructor put_in_xmm_mem_aligned_imm_16 put_in_xmm_mem_aligned_imm_16)
+(decl xmm_to_xmm_mem_16 (Xmm) XmmMem16)
+(extern constructor xmm_to_xmm_mem_16 xmm_to_xmm_mem_16)
+(decl xmm_to_xmm_mem_aligned_16 (Xmm) XmmMemAligned16)
+(extern constructor xmm_to_xmm_mem_aligned_16 xmm_to_xmm_mem_aligned_16)
+(decl xmm_to_xmm_mem_imm_16 (Xmm) XmmMemImm16)
+(extern constructor xmm_to_xmm_mem_imm_16 xmm_to_xmm_mem_imm_16)
+(decl xmm_to_xmm_mem_aligned_imm_16 (Xmm) XmmMemAlignedImm16)
+(extern constructor xmm_to_xmm_mem_aligned_imm_16 xmm_to_xmm_mem_aligned_imm_16)
+(decl xmm_mem_imm_16_imm (u32) XmmMemImm16)
+(extern constructor xmm_mem_imm_16_imm xmm_mem_imm_16_imm)
+(decl xmm_mem_16_to_xmm_mem (XmmMem16) XmmMem)
+(extern constructor xmm_mem_16_to_xmm_mem xmm_mem_16_to_xmm_mem)
+(decl xmm_mem_aligned_16_to_xmm_mem_aligned (XmmMemAligned16) XmmMemAligned)
+(extern constructor xmm_mem_aligned_16_to_xmm_mem_aligned xmm_mem_aligned_16_to_xmm_mem_aligned)
+(decl xmm_mem_imm_16_to_xmm_mem_imm (XmmMemImm16) XmmMemImm)
+(extern constructor xmm_mem_imm_16_to_xmm_mem_imm xmm_mem_imm_16_to_xmm_mem_imm)
+(decl xmm_mem_aligned_imm_16_to_xmm_mem_aligned_imm (XmmMemAlignedImm16) XmmMemAlignedImm)
+(extern constructor xmm_mem_aligned_imm_16_to_xmm_mem_aligned_imm xmm_mem_aligned_imm_16_to_xmm_mem_aligned_imm)
+
+(decl put_in_xmm_mem_32 (Value) XmmMem32)
+(extern constructor put_in_xmm_mem_32 put_in_xmm_mem_32)
+(decl put_in_xmm_mem_aligned_32 (Value) XmmMemAligned32)
+(extern constructor put_in_xmm_mem_aligned_32 put_in_xmm_mem_aligned_32)
+(decl put_in_xmm_mem_imm_32 (Value) XmmMemImm32)
+(extern constructor put_in_xmm_mem_imm_32 put_in_xmm_mem_imm_32)
+(decl put_in_xmm_mem_aligned_imm_32 (Value) XmmMemAlignedImm32)
+(extern constructor put_in_xmm_mem_aligned_imm_32 put_in_xmm_mem_aligned_imm_32)
+(decl xmm_to_xmm_mem_32 (Xmm) XmmMem32)
+(extern constructor xmm_to_xmm_mem_32 xmm_to_xmm_mem_32)
+(decl xmm_to_xmm_mem_aligned_32 (Xmm) XmmMemAligned32)
+(extern constructor xmm_to_xmm_mem_aligned_32 xmm_to_xmm_mem_aligned_32)
+(decl xmm_to_xmm_mem_imm_32 (Xmm) XmmMemImm32)
+(extern constructor xmm_to_xmm_mem_imm_32 xmm_to_xmm_mem_imm_32)
+(decl xmm_to_xmm_mem_aligned_imm_32 (Xmm) XmmMemAlignedImm32)
+(extern constructor xmm_to_xmm_mem_aligned_imm_32 xmm_to_xmm_mem_aligned_imm_32)
+(decl xmm_mem_imm_32_imm (u32) XmmMemImm32)
+(extern constructor xmm_mem_imm_32_imm xmm_mem_imm_32_imm)
+(decl xmm_mem_32_to_xmm_mem (XmmMem32) XmmMem)
+(extern constructor xmm_mem_32_to_xmm_mem xmm_mem_32_to_xmm_mem)
+(decl xmm_mem_aligned_32_to_xmm_mem_aligned (XmmMemAligned32) XmmMemAligned)
+(extern constructor xmm_mem_aligned_32_to_xmm_mem_aligned xmm_mem_aligned_32_to_xmm_mem_aligned)
+(decl xmm_mem_imm_32_to_xmm_mem_imm (XmmMemImm32) XmmMemImm)
+(extern constructor xmm_mem_imm_32_to_xmm_mem_imm xmm_mem_imm_32_to_xmm_mem_imm)
+(decl xmm_mem_aligned_imm_32_to_xmm_mem_aligned_imm (XmmMemAlignedImm32) XmmMemAlignedImm)
+(extern constructor xmm_mem_aligned_imm_32_to_xmm_mem_aligned_imm xmm_mem_aligned_imm_32_to_xmm_mem_aligned_imm)
+
+(decl put_in_xmm_mem_64 (Value) XmmMem64)
+(extern constructor put_in_xmm_mem_64 put_in_xmm_mem_64)
+(decl put_in_xmm_mem_aligned_64 (Value) XmmMemAligned64)
+(extern constructor put_in_xmm_mem_aligned_64 put_in_xmm_mem_aligned_64)
+(decl put_in_xmm_mem_imm_64 (Value) XmmMemImm64)
+(extern constructor put_in_xmm_mem_imm_64 put_in_xmm_mem_imm_64)
+(decl put_in_xmm_mem_aligned_imm_64 (Value) XmmMemAlignedImm64)
+(extern constructor put_in_xmm_mem_aligned_imm_64 put_in_xmm_mem_aligned_imm_64)
+(decl xmm_to_xmm_mem_64 (Xmm) XmmMem64)
+(extern constructor xmm_to_xmm_mem_64 xmm_to_xmm_mem_64)
+(decl xmm_to_xmm_mem_aligned_64 (Xmm) XmmMemAligned64)
+(extern constructor xmm_to_xmm_mem_aligned_64 xmm_to_xmm_mem_aligned_64)
+(decl xmm_to_xmm_mem_imm_64 (Xmm) XmmMemImm64)
+(extern constructor xmm_to_xmm_mem_imm_64 xmm_to_xmm_mem_imm_64)
+(decl xmm_to_xmm_mem_aligned_imm_64 (Xmm) XmmMemAlignedImm64)
+(extern constructor xmm_to_xmm_mem_aligned_imm_64 xmm_to_xmm_mem_aligned_imm_64)
+(decl xmm_mem_imm_64_imm (u32) XmmMemImm64)
+(extern constructor xmm_mem_imm_64_imm xmm_mem_imm_64_imm)
+(decl xmm_mem_64_to_xmm_mem (XmmMem64) XmmMem)
+(extern constructor xmm_mem_64_to_xmm_mem xmm_mem_64_to_xmm_mem)
+(decl xmm_mem_aligned_64_to_xmm_mem_aligned (XmmMemAligned64) XmmMemAligned)
+(extern constructor xmm_mem_aligned_64_to_xmm_mem_aligned xmm_mem_aligned_64_to_xmm_mem_aligned)
+(decl xmm_mem_imm_64_to_xmm_mem_imm (XmmMemImm64) XmmMemImm)
+(extern constructor xmm_mem_imm_64_to_xmm_mem_imm xmm_mem_imm_64_to_xmm_mem_imm)
+(decl xmm_mem_aligned_imm_64_to_xmm_mem_aligned_imm (XmmMemAlignedImm64) XmmMemAlignedImm)
+(extern constructor xmm_mem_aligned_imm_64_to_xmm_mem_aligned_imm xmm_mem_aligned_imm_64_to_xmm_mem_aligned_imm)
+
+(decl put_in_xmm_mem_128 (Value) XmmMem128)
+(extern constructor put_in_xmm_mem_128 put_in_xmm_mem_128)
+(decl put_in_xmm_mem_aligned_128 (Value) XmmMemAligned128)
+(extern constructor put_in_xmm_mem_aligned_128 put_in_xmm_mem_aligned_128)
+(decl put_in_xmm_mem_imm_128 (Value) XmmMemImm128)
+(extern constructor put_in_xmm_mem_imm_128 put_in_xmm_mem_imm_128)
+(decl put_in_xmm_mem_aligned_imm_128 (Value) XmmMemAlignedImm128)
+(extern constructor put_in_xmm_mem_aligned_imm_128 put_in_xmm_mem_aligned_imm_128)
+(decl xmm_to_xmm_mem_128 (Xmm) XmmMem128)
+(extern constructor xmm_to_xmm_mem_128 xmm_to_xmm_mem_128)
+(decl xmm_to_xmm_mem_aligned_128 (Xmm) XmmMemAligned128)
+(extern constructor xmm_to_xmm_mem_aligned_128 xmm_to_xmm_mem_aligned_128)
+(decl xmm_to_xmm_mem_imm_128 (Xmm) XmmMemImm128)
+(extern constructor xmm_to_xmm_mem_imm_128 xmm_to_xmm_mem_imm_128)
+(decl xmm_to_xmm_mem_aligned_imm_128 (Xmm) XmmMemAlignedImm128)
+(extern constructor xmm_to_xmm_mem_aligned_imm_128 xmm_to_xmm_mem_aligned_imm_128)
+(decl xmm_mem_imm_128_imm (u32) XmmMemImm128)
+(extern constructor xmm_mem_imm_128_imm xmm_mem_imm_128_imm)
+(decl xmm_mem_128_to_xmm_mem (XmmMem128) XmmMem)
+(extern constructor xmm_mem_128_to_xmm_mem xmm_mem_128_to_xmm_mem)
+(decl xmm_mem_aligned_128_to_xmm_mem_aligned (XmmMemAligned128) XmmMemAligned)
+(extern constructor xmm_mem_aligned_128_to_xmm_mem_aligned xmm_mem_aligned_128_to_xmm_mem_aligned)
+(decl xmm_mem_imm_128_to_xmm_mem_imm (XmmMemImm128) XmmMemImm)
+(extern constructor xmm_mem_imm_128_to_xmm_mem_imm xmm_mem_imm_128_to_xmm_mem_imm)
+(decl xmm_mem_aligned_imm_128_to_xmm_mem_aligned_imm (XmmMemAlignedImm128) XmmMemAlignedImm)
+(extern constructor xmm_mem_aligned_imm_128_to_xmm_mem_aligned_imm xmm_mem_aligned_imm_128_to_xmm_mem_aligned_imm)
+
;; Put a value into a XMM.
;;
;; Asserts that the value goes into a XMM.
@@ -985,6 +1218,20 @@
(decl is_mem (SyntheticAmode) XmmMem)
(extern extractor is_mem is_mem)
+;; Extract a register or amode from a size-newtyped operand.
+(decl is_xmm_32 (Xmm) XmmMem32)
+(extern extractor is_xmm_32 is_xmm_32)
+(decl is_xmm_64 (Xmm) XmmMem64)
+(extern extractor is_xmm_64 is_xmm_64)
+(decl is_xmm_128 (Xmm) XmmMem128)
+(extern extractor is_xmm_128 is_xmm_128)
+(decl is_mem_32 (SyntheticAmode) XmmMem32)
+(extern extractor is_mem_32 is_mem_32)
+(decl is_mem_64 (SyntheticAmode) XmmMem64)
+(extern extractor is_mem_64 is_mem_64)
+(decl is_mem_128 (SyntheticAmode) XmmMem128)
+(extern extractor is_mem_128 is_mem_128)
+
;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl pure has_avx512vl () bool)
@@ -1090,6 +1337,65 @@
(rule (sink_load_to_xmm_mem load)
(reg_mem_to_xmm_mem load))
+;; Width-tagged sinkable loads. Unlike the un-sized `SinkableLoad`, these
+;; newtypes carry the CLIF type width in their static type; their
+;; extractors (`sinkable_load_N`) only fire when the loaded `Value` is
+;; exactly N bits wide. Converting a `SinkableLoadN` to an N-bit memory
+;; operand is then always safe.
+
+(type SinkableLoad8 extern (enum))
+(type SinkableLoad16 extern (enum))
+(type SinkableLoad32 extern (enum))
+(type SinkableLoad64 extern (enum))
+(type SinkableLoad128 extern (enum))
+
+(decl sinkable_load_8 (SinkableLoad8) Value)
+(extern extractor sinkable_load_8 sinkable_load_8)
+(decl sinkable_load_16 (SinkableLoad16) Value)
+(extern extractor sinkable_load_16 sinkable_load_16)
+(decl sinkable_load_32 (SinkableLoad32) Value)
+(extern extractor sinkable_load_32 sinkable_load_32)
+(decl sinkable_load_64 (SinkableLoad64) Value)
+(extern extractor sinkable_load_64 sinkable_load_64)
+(decl sinkable_load_128 (SinkableLoad128) Value)
+(extern extractor sinkable_load_128 sinkable_load_128)
+
+;; Actually sink a width-tagged load, returning its address. The returned
+;; `SyntheticAmode` is then wrapped at the known width to form the final
+;; memory operand.
+(decl sink_load_8 (SinkableLoad8) SyntheticAmode)
+(extern constructor sink_load_8 sink_load_8)
+(decl sink_load_16 (SinkableLoad16) SyntheticAmode)
+(extern constructor sink_load_16 sink_load_16)
+(decl sink_load_32 (SinkableLoad32) SyntheticAmode)
+(extern constructor sink_load_32 sink_load_32)
+(decl sink_load_64 (SinkableLoad64) SyntheticAmode)
+(extern constructor sink_load_64 sink_load_64)
+(decl sink_load_128 (SinkableLoad128) SyntheticAmode)
+(extern constructor sink_load_128 sink_load_128)
+
+;; Sized conversions from sinkable loads to operands. The width is
+;; already fixed, so no dynamic check is needed.
+(decl sink_load_8_to_gpr_mem_8 (SinkableLoad8) GprMem8)
+(extern constructor sink_load_8_to_gpr_mem_8 sink_load_8_to_gpr_mem_8)
+(decl sink_load_16_to_gpr_mem_16 (SinkableLoad16) GprMem16)
+(extern constructor sink_load_16_to_gpr_mem_16 sink_load_16_to_gpr_mem_16)
+(decl sink_load_32_to_gpr_mem_32 (SinkableLoad32) GprMem32)
+(extern constructor sink_load_32_to_gpr_mem_32 sink_load_32_to_gpr_mem_32)
+(decl sink_load_64_to_gpr_mem_64 (SinkableLoad64) GprMem64)
+(extern constructor sink_load_64_to_gpr_mem_64 sink_load_64_to_gpr_mem_64)
+
+(decl sink_load_8_to_xmm_mem_8 (SinkableLoad8) XmmMem8)
+(extern constructor sink_load_8_to_xmm_mem_8 sink_load_8_to_xmm_mem_8)
+(decl sink_load_16_to_xmm_mem_16 (SinkableLoad16) XmmMem16)
+(extern constructor sink_load_16_to_xmm_mem_16 sink_load_16_to_xmm_mem_16)
+(decl sink_load_32_to_xmm_mem_32 (SinkableLoad32) XmmMem32)
+(extern constructor sink_load_32_to_xmm_mem_32 sink_load_32_to_xmm_mem_32)
+(decl sink_load_64_to_xmm_mem_64 (SinkableLoad64) XmmMem64)
+(extern constructor sink_load_64_to_xmm_mem_64 sink_load_64_to_xmm_mem_64)
+(decl sink_load_128_to_xmm_mem_128 (SinkableLoad128) XmmMem128)
+(extern constructor sink_load_128_to_xmm_mem_128 sink_load_128_to_xmm_mem_128)
+
(decl sink_load_to_reg_mem (SinkableLoad) RegMem)
(rule (sink_load_to_reg_mem load) (RegMem.Mem load))
@@ -1290,7 +1596,7 @@
(rule (vec_int_type (multi_lane 64 2)) $I64X2)
;; Performs an xor operation of the two operands specified.
-(decl x64_xor_vector (Type Xmm XmmMem) Xmm)
+(decl x64_xor_vector (Type Xmm XmmMem128) Xmm)
(rule 1 (x64_xor_vector $F16 x y) (x64_xorps x y))
(rule 1 (x64_xor_vector $F32 x y) (x64_xorps x y))
(rule 1 (x64_xor_vector $F64 x y) (x64_xorpd x y))
@@ -1370,10 +1676,10 @@
(decl x64_load_xmm (Type SyntheticAmode) Xmm)
(rule 1 (x64_load_xmm $F32 addr) (x64_movss_load addr))
(rule 1 (x64_load_xmm $F64 addr) (x64_movsd_load addr))
-(rule 1 (x64_load_xmm $F128 addr) (x64_movdqu_load addr))
+(rule 1 (x64_load_xmm $F128 addr) (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 addr)))
(rule 1 (x64_load_xmm $F32X4 addr) (x64_movups_load addr))
(rule 1 (x64_load_xmm $F64X2 addr) (x64_movupd_load addr))
-(rule 0 (x64_load_xmm (multi_lane _bits _lanes) addr) (x64_movdqu_load addr))
+(rule 0 (x64_load_xmm (multi_lane _bits _lanes) addr) (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 addr)))
(decl x64_mov (SyntheticAmode) Reg)
(spec (x64_mov addr)
@@ -1402,23 +1708,24 @@
(= mode (ExtMode.WL))
(= mode (ExtMode.WQ))
(= mode (ExtMode.LQ)))))
-(rule (x64_movzx (ExtMode.BL) src) (x64_movzbl_rm src))
-(rule (x64_movzx (ExtMode.BQ) src) (x64_movzbq_rm src))
-(rule (x64_movzx (ExtMode.WL) src) (x64_movzwl_rm src))
-(rule (x64_movzx (ExtMode.WQ) src) (x64_movzwq_rm src))
+
+(rule (x64_movzx (ExtMode.BL) src) (x64_movzbl_rm (gpr_mem_as_gpr_mem_8 src)))
+(rule (x64_movzx (ExtMode.BQ) src) (x64_movzbq_rm (gpr_mem_as_gpr_mem_8 src)))
+(rule (x64_movzx (ExtMode.WL) src) (x64_movzwl_rm (gpr_mem_as_gpr_mem_16 src)))
+(rule (x64_movzx (ExtMode.WQ) src) (x64_movzwq_rm (gpr_mem_as_gpr_mem_16 src)))
;; This instruction selection may seem strange but is correct in 64-bit mode:
;; section 3.4.1.1 of the Intel manual says that "32-bit operands generate a
;; 32-bit result, zero-extended to a 64-bit result in the destination
;; general-purpose register." This is applicable beyond `mov` but we use this
;; fact to zero-extend `src` into `dst`.
-(rule (x64_movzx (ExtMode.LQ) src) (x64_movl_rm src))
+(rule (x64_movzx (ExtMode.LQ) src) (x64_movl_rm (gpr_mem_as_gpr_mem_32 src)))
(decl x64_movsx (ExtMode GprMem) Gpr)
-(rule (x64_movsx (ExtMode.BL) src) (x64_movsbl_rm src))
-(rule (x64_movsx (ExtMode.BQ) src) (x64_movsbq_rm src))
-(rule (x64_movsx (ExtMode.WL) src) (x64_movswl_rm src))
-(rule (x64_movsx (ExtMode.WQ) src) (x64_movswq_rm src))
-(rule (x64_movsx (ExtMode.LQ) src) (x64_movslq_rm src))
+(rule (x64_movsx (ExtMode.BL) src) (x64_movsbl_rm (gpr_mem_as_gpr_mem_8 src)))
+(rule (x64_movsx (ExtMode.BQ) src) (x64_movsbq_rm (gpr_mem_as_gpr_mem_8 src)))
+(rule (x64_movsx (ExtMode.WL) src) (x64_movswl_rm (gpr_mem_as_gpr_mem_16 src)))
+(rule (x64_movsx (ExtMode.WQ) src) (x64_movswq_rm (gpr_mem_as_gpr_mem_16 src)))
+(rule (x64_movsx (ExtMode.LQ) src) (x64_movslq_rm (gpr_mem_as_gpr_mem_32 src)))
(decl x64_movss_load (SyntheticAmode) Xmm)
(rule (x64_movss_load from) (x64_movss_a_m_or_avx from))
@@ -1452,14 +1759,14 @@
(x64_vmovd_b from))
;; Helper for creating `movd` instructions.
-(decl x64_movd_to_xmm (GprMem) Xmm)
+(decl x64_movd_to_xmm (GprMem32) Xmm)
(rule (x64_movd_to_xmm from) (x64_movd_a from))
(rule 1 (x64_movd_to_xmm from)
(if-let true (has_avx))
(x64_vmovd_a from))
;; Helper for creating `movq` instructions.
-(decl x64_movq_to_xmm (GprMem) Xmm)
+(decl x64_movq_to_xmm (GprMem64) Xmm)
(rule (x64_movq_to_xmm src) (x64_movq_a src))
(rule 1 (x64_movq_to_xmm from)
(if-let true (has_avx))
@@ -1472,28 +1779,30 @@
(if-let true (has_avx))
(x64_vmovq_b from))
-(decl x64_movdqu_load (XmmMem) Xmm)
+(decl x64_movdqu_load (XmmMem128) Xmm)
(rule (x64_movdqu_load from) (x64_movdqu_a_or_avx from))
(decl x64_movdqu_store (SyntheticAmode Xmm) SideEffectNoResult)
(rule (x64_movdqu_store addr data) (x64_movdqu_b_mem_or_avx addr data))
-(decl x64_pmovsxbw (XmmMem) Xmm)
+;; Packed sign/zero-extend helpers. All of these read 64 bits from memory
+;; (`xmm_m64`) and extend up into a full XMM register.
+(decl x64_pmovsxbw (XmmMem64) Xmm)
(rule (x64_pmovsxbw from) (x64_pmovsxbw_a_or_avx from))
-(decl x64_pmovzxbw (XmmMem) Xmm)
+(decl x64_pmovzxbw (XmmMem64) Xmm)
(rule (x64_pmovzxbw from) (x64_pmovzxbw_a_or_avx from))
-(decl x64_pmovsxwd (XmmMem) Xmm)
+(decl x64_pmovsxwd (XmmMem64) Xmm)
(rule (x64_pmovsxwd from) (x64_pmovsxwd_a_or_avx from))
-(decl x64_pmovzxwd (XmmMem) Xmm)
+(decl x64_pmovzxwd (XmmMem64) Xmm)
(rule (x64_pmovzxwd from) (x64_pmovzxwd_a_or_avx from))
-(decl x64_pmovsxdq (XmmMem) Xmm)
+(decl x64_pmovsxdq (XmmMem64) Xmm)
(rule (x64_pmovsxdq from) (x64_pmovsxdq_a_or_avx from))
-(decl x64_pmovzxdq (XmmMem) Xmm)
+(decl x64_pmovzxdq (XmmMem64) Xmm)
(rule (x64_pmovzxdq from) (x64_pmovzxdq_a_or_avx from))
(decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult)
@@ -1600,10 +1909,10 @@
(rule 1 (x64_add_raw $I64 src1 (is_simm32 src2)) (x64_addq_mi_sxl_raw src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_add_raw $I8 src1 (is_gpr_mem src2)) (x64_addb_rm_raw src1 src2))
-(rule 0 (x64_add_raw $I16 src1 (is_gpr_mem src2)) (x64_addw_rm_raw src1 src2))
-(rule 0 (x64_add_raw $I32 src1 (is_gpr_mem src2)) (x64_addl_rm_raw src1 src2))
-(rule 0 (x64_add_raw $I64 src1 (is_gpr_mem src2)) (x64_addq_rm_raw src1 src2))
+(rule 0 (x64_add_raw ty @ $I8 src1 (is_gpr_mem src2)) (x64_addb_rm_raw src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_add_raw ty @ $I16 src1 (is_gpr_mem src2)) (x64_addw_rm_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_add_raw ty @ $I32 src1 (is_gpr_mem src2)) (x64_addl_rm_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_add_raw ty @ $I64 src1 (is_gpr_mem src2)) (x64_addq_rm_raw src1 (gpr_mem_64_for_ty ty src2)))
;; When the overflow flag is not considered, we can use wider instructions than
;; necessary for 8/16-bit register-to-register operations to avoid CPU false
@@ -1633,7 +1942,7 @@
(decl x64_adc_raw (Type Gpr GprMemImm) AssemblerOutputs)
(rule 2 (x64_adc_raw $I64 src1 (is_simm8 src2)) (x64_adcq_mi_sxb_raw src1 src2))
(rule 1 (x64_adc_raw $I64 src1 (is_simm32 src2)) (x64_adcq_mi_sxl_raw src1 src2))
-(rule 0 (x64_adc_raw $I64 src1 (is_gpr_mem src2)) (x64_adcq_rm_raw src1 src2))
+(rule 0 (x64_adc_raw ty @ $I64 src1 (is_gpr_mem src2)) (x64_adcq_rm_raw src1 (gpr_mem_64_for_ty ty src2)))
;; Normal use of the `adc` instruction consumes a previously-produced flag.
(decl x64_adc_paired (Type Gpr GprMemImm) ConsumesFlags)
@@ -1661,10 +1970,10 @@
(rule 2 (x64_sub_raw $I64 src1 (is_simm32 src2)) (x64_subq_mi_sxl_raw src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_sub_raw $I8 src1 (is_gpr_mem src2)) (x64_subb_rm_raw src1 src2))
-(rule 0 (x64_sub_raw $I16 src1 (is_gpr_mem src2)) (x64_subw_rm_raw src1 src2))
-(rule 0 (x64_sub_raw $I32 src1 (is_gpr_mem src2)) (x64_subl_rm_raw src1 src2))
-(rule 0 (x64_sub_raw $I64 src1 (is_gpr_mem src2)) (x64_subq_rm_raw src1 src2))
+(rule 0 (x64_sub_raw ty @ $I8 src1 (is_gpr_mem src2)) (x64_subb_rm_raw src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_sub_raw ty @ $I16 src1 (is_gpr_mem src2)) (x64_subw_rm_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_sub_raw ty @ $I32 src1 (is_gpr_mem src2)) (x64_subl_rm_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_sub_raw ty @ $I64 src1 (is_gpr_mem src2)) (x64_subq_rm_raw src1 (gpr_mem_64_for_ty ty src2)))
;; When the overflow flag is not considered, we can use wider instructions than
;; necessary for 8/16-bit register-to-register operations to avoid CPU false
@@ -1702,10 +2011,10 @@
(rule 1 (x64_sbb_raw $I64 src1 (is_simm32 src2)) (x64_sbbq_mi_sxl_raw src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_sbb_raw $I8 src1 (is_gpr_mem src2)) (x64_sbbb_rm_raw src1 src2))
-(rule 0 (x64_sbb_raw $I16 src1 (is_gpr_mem src2)) (x64_sbbw_rm_raw src1 src2))
-(rule 0 (x64_sbb_raw $I32 src1 (is_gpr_mem src2)) (x64_sbbl_rm_raw src1 src2))
-(rule 0 (x64_sbb_raw $I64 src1 (is_gpr_mem src2)) (x64_sbbq_rm_raw src1 src2))
+(rule 0 (x64_sbb_raw ty @ $I8 src1 (is_gpr_mem src2)) (x64_sbbb_rm_raw src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_sbb_raw ty @ $I16 src1 (is_gpr_mem src2)) (x64_sbbw_rm_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_sbb_raw ty @ $I32 src1 (is_gpr_mem src2)) (x64_sbbl_rm_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_sbb_raw ty @ $I64 src1 (is_gpr_mem src2)) (x64_sbbq_rm_raw src1 (gpr_mem_64_for_ty ty src2)))
;; When the overflow flag is not considered, we can use wider instructions than
;; necessary for 8/16-bit register-to-register operations to avoid CPU false
@@ -1736,12 +2045,12 @@
;; Helper for creating `mul` instructions or `imul` instructions (depending
;; on `signed`). For the 8-bit rules, see `x64_mul8`.
(decl x64_mul_raw (Type bool Gpr GprMem) AssemblerOutputs)
-(rule (x64_mul_raw $I16 false src1 src2) (x64_mulw_m_raw src1 src2))
-(rule (x64_mul_raw $I32 false src1 src2) (x64_mull_m_raw src1 src2))
-(rule (x64_mul_raw $I64 false src1 src2) (x64_mulq_m_raw src1 src2))
-(rule (x64_mul_raw $I16 true src1 src2) (x64_imulw_m_raw src1 src2))
-(rule (x64_mul_raw $I32 true src1 src2) (x64_imull_m_raw src1 src2))
-(rule (x64_mul_raw $I64 true src1 src2) (x64_imulq_m_raw src1 src2))
+(rule (x64_mul_raw ty @ $I16 false src1 src2) (x64_mulw_m_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule (x64_mul_raw ty @ $I32 false src1 src2) (x64_mull_m_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule (x64_mul_raw ty @ $I64 false src1 src2) (x64_mulq_m_raw src1 (gpr_mem_64_for_ty ty src2)))
+(rule (x64_mul_raw ty @ $I16 true src1 src2) (x64_imulw_m_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule (x64_mul_raw ty @ $I32 true src1 src2) (x64_imull_m_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule (x64_mul_raw ty @ $I64 true src1 src2) (x64_imulq_m_raw src1 (gpr_mem_64_for_ty ty src2)))
(decl x64_mul (Type bool Gpr GprMem) ValueRegs)
(rule 0 (x64_mul ty signed src1 src2)
@@ -1752,18 +2061,18 @@
;; Note that mulx returns the high bits in the first result and the low bits in
;; the second result, so here the result registers are swapped to match
;; `x64_mul` above.
-(rule 1 (x64_mul $I32 false src1 src2)
+(rule 1 (x64_mul ty @ $I32 false src1 src2)
(if-let true (has_bmi2))
- (let ((regs ValueRegs (x64_mulxl_rvm src2 src1)))
+ (let ((regs ValueRegs (x64_mulxl_rvm (gpr_mem_32_for_ty ty src2) src1)))
(value_regs (value_regs_get regs 1) (value_regs_get regs 0))))
-(rule 1 (x64_mul $I64 false src1 src2)
+(rule 1 (x64_mul ty @ $I64 false src1 src2)
(if-let true (has_bmi2))
- (let ((regs ValueRegs (x64_mulxq_rvm src2 src1)))
+ (let ((regs ValueRegs (x64_mulxq_rvm (gpr_mem_64_for_ty ty src2) src1)))
(value_regs (value_regs_get regs 1) (value_regs_get regs 0))))
(decl x64_mulx_hi (Type Gpr GprMem) Gpr)
-(rule (x64_mulx_hi $I32 src1 src2) (x64_mulxl_rvm_hi src2 src1))
-(rule (x64_mulx_hi $I64 src1 src2) (x64_mulxq_rvm_hi src2 src1))
+(rule (x64_mulx_hi ty @ $I32 src1 src2) (x64_mulxl_rvm_hi (gpr_mem_32_for_ty ty src2) src1))
+(rule (x64_mulx_hi ty @ $I64 src1 src2) (x64_mulxq_rvm_hi (gpr_mem_64_for_ty ty src2) src1))
(decl x64_mulxl_rvm_hi (GprMem Gpr) Gpr)
(extern constructor x64_mulxl_rvm_hi x64_mulxl_rvm_hi)
@@ -1780,25 +2089,26 @@
;; Helper for creating `imul` instructions.
(decl x64_imul (Type Gpr GprMem) Gpr)
-(rule (x64_imul $I16 src1 src2) (x64_imulw_rm src1 src2))
-(rule (x64_imul $I32 src1 src2) (x64_imull_rm src1 src2))
-(rule (x64_imul $I64 src1 src2) (x64_imulq_rm src1 src2))
+(rule (x64_imul ty @ $I16 src1 src2) (x64_imulw_rm src1 (gpr_mem_16_for_ty ty src2)))
+(rule (x64_imul ty @ $I32 src1 src2) (x64_imull_rm src1 (gpr_mem_32_for_ty ty src2)))
+(rule (x64_imul ty @ $I64 src1 src2) (x64_imulq_rm src1 (gpr_mem_64_for_ty ty src2)))
;; Helper for creating `imul` instructions with an immediate operand. Match
;; 8-bit immediates first to allow a smaller instruction encoding.
(decl x64_imul_imm (Type GprMem i32) Gpr)
-(rule 2 (x64_imul_imm $I16 src1 (i8_from_i32 src2)) (x64_imulw_rmi_sxb src1 src2))
-(rule 2 (x64_imul_imm $I32 src1 (i8_from_i32 src2)) (x64_imull_rmi_sxb src1 src2))
-(rule 2 (x64_imul_imm $I64 src1 (i8_from_i32 src2)) (x64_imulq_rmi_sxb src1 src2))
-(rule 1 (x64_imul_imm $I16 src1 (i16_from_i32 src2)) (x64_imulw_rmi src1 (i16_cast_unsigned src2)))
-(rule 1 (x64_imul_imm $I32 src1 src2) (x64_imull_rmi src1 (i32_cast_unsigned src2)))
-(rule 1 (x64_imul_imm $I64 src1 src2) (x64_imulq_rmi_sxl src1 src2))
+(rule 2 (x64_imul_imm ty @ $I16 src1 (i8_from_i32 src2)) (x64_imulw_rmi_sxb (gpr_mem_16_for_ty ty src1) src2))
+(rule 2 (x64_imul_imm ty @ $I32 src1 (i8_from_i32 src2)) (x64_imull_rmi_sxb (gpr_mem_32_for_ty ty src1) src2))
+(rule 2 (x64_imul_imm ty @ $I64 src1 (i8_from_i32 src2)) (x64_imulq_rmi_sxb (gpr_mem_64_for_ty ty src1) src2))
+(rule 1 (x64_imul_imm ty @ $I16 src1 (i16_from_i32 src2)) (x64_imulw_rmi (gpr_mem_16_for_ty ty src1) (i16_cast_unsigned src2)))
+(rule 1 (x64_imul_imm ty @ $I32 src1 src2) (x64_imull_rmi (gpr_mem_32_for_ty ty src1) (i32_cast_unsigned src2)))
+(rule 1 (x64_imul_imm ty @ $I64 src1 src2) (x64_imulq_rmi_sxl (gpr_mem_64_for_ty ty src1) src2))
;; Helper for creating `mul` instructions or `imul` instructions (depending
;; on `signed`) for 8-bit operands.
(decl x64_mul8_raw (bool Gpr GprMem) AssemblerOutputs)
-(rule (x64_mul8_raw false src1 src2) (x64_mulb_m_raw src1 src2))
-(rule (x64_mul8_raw true src1 src2) (x64_imulb_m_raw src1 src2))
+;; Fixed 8-bit: no `Type` in scope, so use the unchecked narrow.
+(rule (x64_mul8_raw false src1 src2) (x64_mulb_m_raw src1 (gpr_mem_as_gpr_mem_8 src2)))
+(rule (x64_mul8_raw true src1 src2) (x64_imulb_m_raw src1 (gpr_mem_as_gpr_mem_8 src2)))
(decl x64_mul8 (bool Gpr GprMem) Gpr)
(rule (x64_mul8 signed src1 src2)
@@ -1829,10 +2139,10 @@
(rule 1 (x64_and $I16 src1 (is_gpr src2)) (x64_andl_rm src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_and $I8 src1 (is_gpr_mem src2)) (x64_andb_rm src1 src2))
-(rule 0 (x64_and $I16 src1 (is_gpr_mem src2)) (x64_andw_rm src1 src2))
-(rule 0 (x64_and $I32 src1 (is_gpr_mem src2)) (x64_andl_rm src1 src2))
-(rule 0 (x64_and $I64 src1 (is_gpr_mem src2)) (x64_andq_rm src1 src2))
+(rule 0 (x64_and ty @ $I8 src1 (is_gpr_mem src2)) (x64_andb_rm src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_and ty @ $I16 src1 (is_gpr_mem src2)) (x64_andw_rm src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_and ty @ $I32 src1 (is_gpr_mem src2)) (x64_andl_rm src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_and ty @ $I64 src1 (is_gpr_mem src2)) (x64_andq_rm src1 (gpr_mem_64_for_ty ty src2)))
@@ -1850,10 +2160,10 @@
(rule 1 (x64_or_raw $I64 src1 (is_simm32 src2)) (x64_orq_mi_sxl_raw src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_or_raw $I8 src1 (is_gpr_mem src2)) (x64_orb_rm_raw src1 src2))
-(rule 0 (x64_or_raw $I16 src1 (is_gpr_mem src2)) (x64_orw_rm_raw src1 src2))
-(rule 0 (x64_or_raw $I32 src1 (is_gpr_mem src2)) (x64_orl_rm_raw src1 src2))
-(rule 0 (x64_or_raw $I64 src1 (is_gpr_mem src2)) (x64_orq_rm_raw src1 src2))
+(rule 0 (x64_or_raw ty @ $I8 src1 (is_gpr_mem src2)) (x64_orb_rm_raw src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_or_raw ty @ $I16 src1 (is_gpr_mem src2)) (x64_orw_rm_raw src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_or_raw ty @ $I32 src1 (is_gpr_mem src2)) (x64_orl_rm_raw src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_or_raw ty @ $I64 src1 (is_gpr_mem src2)) (x64_orq_rm_raw src1 (gpr_mem_64_for_ty ty src2)))
;; When flags are not considered, we can use wider instructions than necessary
;; for 8/16-bit register-to-register operations to avoid CPU false dependencies.
@@ -1894,10 +2204,10 @@
(rule 1 (x64_xor $I16 src1 (is_gpr src2)) (x64_xorl_rm src1 src2))
;; Match the operand size to the instruction width.
-(rule 0 (x64_xor $I8 src1 (is_gpr_mem src2)) (x64_xorb_rm src1 src2))
-(rule 0 (x64_xor $I16 src1 (is_gpr_mem src2)) (x64_xorw_rm src1 src2))
-(rule 0 (x64_xor $I32 src1 (is_gpr_mem src2)) (x64_xorl_rm src1 src2))
-(rule 0 (x64_xor $I64 src1 (is_gpr_mem src2)) (x64_xorq_rm src1 src2))
+(rule 0 (x64_xor ty @ $I8 src1 (is_gpr_mem src2)) (x64_xorb_rm src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_xor ty @ $I16 src1 (is_gpr_mem src2)) (x64_xorw_rm src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_xor ty @ $I32 src1 (is_gpr_mem src2)) (x64_xorl_rm src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_xor ty @ $I64 src1 (is_gpr_mem src2)) (x64_xorq_rm src1 (gpr_mem_64_for_ty ty src2)))
;; Helper for `andn` instructions
;;
@@ -1905,10 +2215,10 @@
;; those bit-widths the 32-bit version of the instruction is used which has the
;; desired semantics for the lower bits of the register.
(decl x64_andn (Type Gpr GprMem) Gpr)
-(rule (x64_andn $I8 src1 src2) (x64_andnl_rvm src1 src2))
-(rule (x64_andn $I16 src1 src2) (x64_andnl_rvm src1 src2))
-(rule (x64_andn $I32 src1 src2) (x64_andnl_rvm src1 src2))
-(rule (x64_andn $I64 src1 src2) (x64_andnq_rvm src1 src2))
+(rule (x64_andn ty @ $I8 src1 src2) (x64_andnl_rvm src1 (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty src2)))
+(rule (x64_andn ty @ $I16 src1 src2) (x64_andnl_rvm src1 (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty src2)))
+(rule (x64_andn ty @ $I32 src1 src2) (x64_andnl_rvm src1 (gpr_mem_32_for_ty ty src2)))
+(rule (x64_andn ty @ $I64 src1 src2) (x64_andnq_rvm src1 (gpr_mem_64_for_ty ty src2)))
;; Helper for emitting immediates with an `i64` value. Note that
;; integer constants in ISLE are always parsed as `i128`s; this enables
@@ -2076,8 +2386,8 @@
;; Helper for creating zeroing-of-high-bits instructions bzhi
(decl x64_bzhi (Type GprMem Gpr) Gpr)
-(rule (x64_bzhi $I32 src1 src2) (x64_bzhil_rmv src1 src2))
-(rule (x64_bzhi $I64 src1 src2) (x64_bzhiq_rmv src1 src2))
+(rule (x64_bzhi ty @ $I32 src1 src2) (x64_bzhil_rmv (gpr_mem_32_for_ty ty src1) src2))
+(rule (x64_bzhi ty @ $I64 src1 src2) (x64_bzhiq_rmv (gpr_mem_64_for_ty ty src1) src2))
;; Helper for creating byteswap instructions.
;; In x64, 32- and 64-bit registers use BSWAP instruction, and
@@ -2101,13 +2411,18 @@
(rule 1 (x64_cmp $I64 src1 (is_simm32 src2)) (x64_cmpq_mi src1 src2))
;; Base case: rhs is a GprMem operand.
-(rule 0 (x64_cmp $I8 src1 (is_gpr_mem src2)) (x64_cmpb_rm src1 src2))
-(rule 0 (x64_cmp $I16 src1 (is_gpr_mem src2)) (x64_cmpw_rm src1 src2))
-(rule 0 (x64_cmp $I32 src1 (is_gpr_mem src2)) (x64_cmpl_rm src1 src2))
-(rule 0 (x64_cmp $I64 src1 (is_gpr_mem src2)) (x64_cmpq_rm src1 src2))
+(rule 0 (x64_cmp ty @ $I8 src1 (is_gpr_mem src2)) (x64_cmpb_rm src1 (gpr_mem_8_for_ty ty src2)))
+(rule 0 (x64_cmp ty @ $I16 src1 (is_gpr_mem src2)) (x64_cmpw_rm src1 (gpr_mem_16_for_ty ty src2)))
+(rule 0 (x64_cmp ty @ $I32 src1 (is_gpr_mem src2)) (x64_cmpl_rm src1 (gpr_mem_32_for_ty ty src2)))
+(rule 0 (x64_cmp ty @ $I64 src1 (is_gpr_mem src2)) (x64_cmpq_rm src1 (gpr_mem_64_for_ty ty src2)))
;; Helper for creating floating-point comparison instructions (`UCOMIS[S|D]`).
-(decl x64_ucomis (Type Xmm XmmMem) ProducesFlags)
+;;
+;; `src2` takes a CLIF `Value`, not a sized operand, so the dispatch on
+;; `Type` pins the width via the `Value -> XmmMem{N}` auto-conversion
+;; (which release-asserts the value's CLIF width matches N). No trust
+;; cast is needed.
+(decl x64_ucomis (Type Xmm Value) ProducesFlags)
(rule (x64_ucomis $F32 src1 src2) (x64_ucomiss_a_or_avx src1 src2))
(rule (x64_ucomis $F64 src1 src2) (x64_ucomisd_a_or_avx src1 src2))
@@ -2119,13 +2434,13 @@
(rule 1 (x64_test $I32 src1 (is_imm32 src2)) (x64_testl_mi src1 src2))
(rule 1 (x64_test $I64 src1 (is_simm32 src2)) (x64_testq_mi src1 src2))
-(rule 0 (x64_test $I8 src1 (is_gpr_mem src2)) (x64_testb_mr src2 src1))
-(rule 0 (x64_test $I16 src1 (is_gpr_mem src2)) (x64_testw_mr src2 src1))
-(rule 0 (x64_test $I32 src1 (is_gpr_mem src2)) (x64_testl_mr src2 src1))
-(rule 0 (x64_test $I64 src1 (is_gpr_mem src2)) (x64_testq_mr src2 src1))
+(rule 0 (x64_test ty @ $I8 src1 (is_gpr_mem src2)) (x64_testb_mr (gpr_mem_8_for_ty ty src2) src1))
+(rule 0 (x64_test ty @ $I16 src1 (is_gpr_mem src2)) (x64_testw_mr (gpr_mem_16_for_ty ty src2) src1))
+(rule 0 (x64_test ty @ $I32 src1 (is_gpr_mem src2)) (x64_testl_mr (gpr_mem_32_for_ty ty src2) src1))
+(rule 0 (x64_test ty @ $I64 src1 (is_gpr_mem src2)) (x64_testq_mr (gpr_mem_64_for_ty ty src2) src1))
;; Helper for creating `ptest` instructions.
-(decl x64_ptest (Xmm XmmMem) ProducesFlags)
+(decl x64_ptest (Xmm XmmMem128) ProducesFlags)
(rule (x64_ptest src1 src2) (x64_ptest_rm_or_avx src1 src2))
;; Helper for creating `cmove` instructions. Note that these instructions do not
@@ -2142,38 +2457,38 @@
;; same instruction but disassemblers only print one mnemonic and that's the
;; name used here.
(decl cmove (Type CC GprMem Gpr) ConsumesFlags)
-(rule 0 (cmove (fits_in_32 _) (CC.O) c a) (x64_cmovol_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NO) c a) (x64_cmovnol_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.B) c a) (x64_cmovbl_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NB) c a) (x64_cmovael_rm a c)) ;; nb == ae
-(rule 0 (cmove (fits_in_32 _) (CC.Z) c a) (x64_cmovel_rm a c)) ;; z == e
-(rule 0 (cmove (fits_in_32 _) (CC.NZ) c a) (x64_cmovnel_rm a c)) ;; nz == ne
-(rule 0 (cmove (fits_in_32 _) (CC.BE) c a) (x64_cmovbel_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NBE) c a) (x64_cmoval_rm a c)) ;; nbe == a
-(rule 0 (cmove (fits_in_32 _) (CC.S) c a) (x64_cmovsl_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NS) c a) (x64_cmovnsl_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.L) c a) (x64_cmovll_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NL) c a) (x64_cmovgel_rm a c)) ;; nl == ge
-(rule 0 (cmove (fits_in_32 _) (CC.LE) c a) (x64_cmovlel_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NLE) c a) (x64_cmovgl_rm a c)) ;; nle == g
-(rule 0 (cmove (fits_in_32 _) (CC.P) c a) (x64_cmovpl_rm a c))
-(rule 0 (cmove (fits_in_32 _) (CC.NP) c a) (x64_cmovnpl_rm a c))
-(rule 1 (cmove $I64 (CC.O) c a) (x64_cmovoq_rm a c))
-(rule 1 (cmove $I64 (CC.NO) c a) (x64_cmovnoq_rm a c))
-(rule 1 (cmove $I64 (CC.B) c a) (x64_cmovbq_rm a c))
-(rule 1 (cmove $I64 (CC.NB) c a) (x64_cmovaeq_rm a c)) ;; nb == ae
-(rule 1 (cmove $I64 (CC.Z) c a) (x64_cmoveq_rm a c)) ;; z == e
-(rule 1 (cmove $I64 (CC.NZ) c a) (x64_cmovneq_rm a c)) ;; nz == ne
-(rule 1 (cmove $I64 (CC.BE) c a) (x64_cmovbeq_rm a c))
-(rule 1 (cmove $I64 (CC.NBE) c a) (x64_cmovaq_rm a c)) ;; nbe == a
-(rule 1 (cmove $I64 (CC.S) c a) (x64_cmovsq_rm a c))
-(rule 1 (cmove $I64 (CC.NS) c a) (x64_cmovnsq_rm a c))
-(rule 1 (cmove $I64 (CC.L) c a) (x64_cmovlq_rm a c))
-(rule 1 (cmove $I64 (CC.NL) c a) (x64_cmovgeq_rm a c)) ;; nl == ge
-(rule 1 (cmove $I64 (CC.LE) c a) (x64_cmovleq_rm a c))
-(rule 1 (cmove $I64 (CC.NLE) c a) (x64_cmovgq_rm a c)) ;; nle == g
-(rule 1 (cmove $I64 (CC.P) c a) (x64_cmovpq_rm a c))
-(rule 1 (cmove $I64 (CC.NP) c a) (x64_cmovnpq_rm a c))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.O) c a) (x64_cmovol_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NO) c a) (x64_cmovnol_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.B) c a) (x64_cmovbl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NB) c a) (x64_cmovael_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; nb == ae
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.Z) c a) (x64_cmovel_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; z == e
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NZ) c a) (x64_cmovnel_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; nz == ne
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.BE) c a) (x64_cmovbel_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NBE) c a) (x64_cmoval_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; nbe == a
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.S) c a) (x64_cmovsl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NS) c a) (x64_cmovnsl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.L) c a) (x64_cmovll_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NL) c a) (x64_cmovgel_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; nl == ge
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.LE) c a) (x64_cmovlel_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NLE) c a) (x64_cmovgl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c))) ;; nle == g
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.P) c a) (x64_cmovpl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 0 (cmove ty @ (fits_in_32 _) (CC.NP) c a) (x64_cmovnpl_rm a (gpr_mem_as_gpr_mem_32_or_widened_in_reg ty c)))
+(rule 1 (cmove ty @ $I64 (CC.O) c a) (x64_cmovoq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NO) c a) (x64_cmovnoq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.B) c a) (x64_cmovbq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NB) c a) (x64_cmovaeq_rm a (gpr_mem_64_for_ty ty c))) ;; nb == ae
+(rule 1 (cmove ty @ $I64 (CC.Z) c a) (x64_cmoveq_rm a (gpr_mem_64_for_ty ty c))) ;; z == e
+(rule 1 (cmove ty @ $I64 (CC.NZ) c a) (x64_cmovneq_rm a (gpr_mem_64_for_ty ty c))) ;; nz == ne
+(rule 1 (cmove ty @ $I64 (CC.BE) c a) (x64_cmovbeq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NBE) c a) (x64_cmovaq_rm a (gpr_mem_64_for_ty ty c))) ;; nbe == a
+(rule 1 (cmove ty @ $I64 (CC.S) c a) (x64_cmovsq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NS) c a) (x64_cmovnsq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.L) c a) (x64_cmovlq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NL) c a) (x64_cmovgeq_rm a (gpr_mem_64_for_ty ty c))) ;; nl == ge
+(rule 1 (cmove ty @ $I64 (CC.LE) c a) (x64_cmovleq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NLE) c a) (x64_cmovgq_rm a (gpr_mem_64_for_ty ty c))) ;; nle == g
+(rule 1 (cmove ty @ $I64 (CC.P) c a) (x64_cmovpq_rm a (gpr_mem_64_for_ty ty c)))
+(rule 1 (cmove ty @ $I64 (CC.NP) c a) (x64_cmovnpq_rm a (gpr_mem_64_for_ty ty c)))
(decl cmove_xmm (Type CC Xmm Xmm) ConsumesFlags)
(rule (cmove_xmm ty cc consequent alternative)
@@ -2217,236 +2532,236 @@
;; Helpers for creating vector `add` instructions.
-(decl x64_addss (Xmm XmmMem) Xmm)
+(decl x64_addss (Xmm XmmMem32) Xmm)
(rule (x64_addss src1 src2) (x64_addss_a_or_avx src1 src2))
-(decl x64_addsd (Xmm XmmMem) Xmm)
+(decl x64_addsd (Xmm XmmMem64) Xmm)
(rule (x64_addsd src1 src2) (x64_addsd_a_or_avx src1 src2))
-(decl x64_addps (Xmm XmmMem) Xmm)
+(decl x64_addps (Xmm XmmMem128) Xmm)
(rule (x64_addps src1 src2) (x64_addps_a_or_avx src1 src2))
-(decl x64_addpd (Xmm XmmMem) Xmm)
+(decl x64_addpd (Xmm XmmMem128) Xmm)
(rule (x64_addpd src1 src2) (x64_addpd_a_or_avx src1 src2))
-(decl x64_paddb (Xmm XmmMem) Xmm)
+(decl x64_paddb (Xmm XmmMem128) Xmm)
(rule (x64_paddb src1 src2) (x64_paddb_a_or_avx src1 src2))
-(decl x64_paddw (Xmm XmmMem) Xmm)
+(decl x64_paddw (Xmm XmmMem128) Xmm)
(rule (x64_paddw src1 src2) (x64_paddw_a_or_avx src1 src2))
-(decl x64_paddd (Xmm XmmMem) Xmm)
+(decl x64_paddd (Xmm XmmMem128) Xmm)
(rule (x64_paddd src1 src2) (x64_paddd_a_or_avx src1 src2))
-(decl x64_paddq (Xmm XmmMem) Xmm)
+(decl x64_paddq (Xmm XmmMem128) Xmm)
(rule (x64_paddq src1 src2) (x64_paddq_a_or_avx src1 src2))
-(decl x64_paddsb (Xmm XmmMem) Xmm)
+(decl x64_paddsb (Xmm XmmMem128) Xmm)
(rule (x64_paddsb src1 src2) (x64_paddsb_a_or_avx src1 src2))
-(decl x64_paddsw (Xmm XmmMem) Xmm)
+(decl x64_paddsw (Xmm XmmMem128) Xmm)
(rule (x64_paddsw src1 src2) (x64_paddsw_a_or_avx src1 src2))
-(decl x64_phaddw (Xmm XmmMem) Xmm)
+(decl x64_phaddw (Xmm XmmMem128) Xmm)
(rule (x64_phaddw src1 src2) (x64_phaddw_a_or_avx src1 src2))
-(decl x64_phaddd (Xmm XmmMem) Xmm)
+(decl x64_phaddd (Xmm XmmMem128) Xmm)
(rule (x64_phaddd src1 src2) (x64_phaddd_a_or_avx src1 src2))
-(decl x64_paddusb (Xmm XmmMem) Xmm)
+(decl x64_paddusb (Xmm XmmMem128) Xmm)
(rule (x64_paddusb src1 src2) (x64_paddusb_a_or_avx src1 src2))
-(decl x64_paddusw (Xmm XmmMem) Xmm)
+(decl x64_paddusw (Xmm XmmMem128) Xmm)
(rule (x64_paddusw src1 src2) (x64_paddusw_a_or_avx src1 src2))
;; Helpers for creating vector `sub` instructions.
-(decl x64_subss (Xmm XmmMem) Xmm)
+(decl x64_subss (Xmm XmmMem32) Xmm)
(rule (x64_subss src1 src2) (x64_subss_a_or_avx src1 src2))
-(decl x64_subsd (Xmm XmmMem) Xmm)
+(decl x64_subsd (Xmm XmmMem64) Xmm)
(rule (x64_subsd src1 src2) (x64_subsd_a_or_avx src1 src2))
-(decl x64_subps (Xmm XmmMem) Xmm)
+(decl x64_subps (Xmm XmmMem128) Xmm)
(rule (x64_subps src1 src2) (x64_subps_a_or_avx src1 src2))
-(decl x64_subpd (Xmm XmmMem) Xmm)
+(decl x64_subpd (Xmm XmmMem128) Xmm)
(rule (x64_subpd src1 src2) (x64_subpd_a_or_avx src1 src2))
-(decl x64_psubb (Xmm XmmMem) Xmm)
+(decl x64_psubb (Xmm XmmMem128) Xmm)
(rule (x64_psubb src1 src2) (x64_psubb_a_or_avx src1 src2))
-(decl x64_psubw (Xmm XmmMem) Xmm)
+(decl x64_psubw (Xmm XmmMem128) Xmm)
(rule (x64_psubw src1 src2) (x64_psubw_a_or_avx src1 src2))
-(decl x64_psubd (Xmm XmmMem) Xmm)
+(decl x64_psubd (Xmm XmmMem128) Xmm)
(rule (x64_psubd src1 src2) (x64_psubd_a_or_avx src1 src2))
-(decl x64_psubq (Xmm XmmMem) Xmm)
+(decl x64_psubq (Xmm XmmMem128) Xmm)
(rule (x64_psubq src1 src2) (x64_psubq_a_or_avx src1 src2))
-(decl x64_psubsb (Xmm XmmMem) Xmm)
+(decl x64_psubsb (Xmm XmmMem128) Xmm)
(rule (x64_psubsb src1 src2) (x64_psubsb_a_or_avx src1 src2))
-(decl x64_psubsw (Xmm XmmMem) Xmm)
+(decl x64_psubsw (Xmm XmmMem128) Xmm)
(rule (x64_psubsw src1 src2) (x64_psubsw_a_or_avx src1 src2))
-(decl x64_psubusb (Xmm XmmMem) Xmm)
+(decl x64_psubusb (Xmm XmmMem128) Xmm)
(rule (x64_psubusb src1 src2) (x64_psubusb_a_or_avx src1 src2))
-(decl x64_psubusw (Xmm XmmMem) Xmm)
+(decl x64_psubusw (Xmm XmmMem128) Xmm)
(rule (x64_psubusw src1 src2) (x64_psubusw_a_or_avx src1 src2))
-;; Helpers for creating `pavg*` instructions.
-(decl x64_pavgb (Xmm XmmMem) Xmm)
+;; Helpers for creating `pavg*` instructions (128-bit packed).
+(decl x64_pavgb (Xmm XmmMem128) Xmm)
(rule (x64_pavgb src1 src2) (x64_pavgb_a_or_avx src1 src2))
-(decl x64_pavgw (Xmm XmmMem) Xmm)
+(decl x64_pavgw (Xmm XmmMem128) Xmm)
(rule (x64_pavgw src1 src2) (x64_pavgw_a_or_avx src1 src2))
;; Helpers for creating vector `and` instructions.
-(decl x64_pand (Xmm XmmMem) Xmm)
+(decl x64_pand (Xmm XmmMem128) Xmm)
(rule (x64_pand src1 src2) (x64_pand_a_or_avx src1 src2))
-(decl x64_andps (Xmm XmmMem) Xmm)
+(decl x64_andps (Xmm XmmMem128) Xmm)
(rule (x64_andps src1 src2) (x64_andps_a_or_avx src1 src2))
-(decl x64_andpd (Xmm XmmMem) Xmm)
+(decl x64_andpd (Xmm XmmMem128) Xmm)
(rule (x64_andpd src1 src2) (x64_andpd_a_or_avx src1 src2))
;; Helpers for creating vector `or` instructions.
-(decl x64_por (Xmm XmmMem) Xmm)
+(decl x64_por (Xmm XmmMem128) Xmm)
(rule (x64_por src1 src2) (x64_por_a_or_avx src1 src2))
-(decl x64_orps (Xmm XmmMem) Xmm)
+(decl x64_orps (Xmm XmmMem128) Xmm)
(rule (x64_orps src1 src2) (x64_orps_a_or_avx src1 src2))
-(decl x64_orpd (Xmm XmmMem) Xmm)
+(decl x64_orpd (Xmm XmmMem128) Xmm)
(rule (x64_orpd src1 src2) (x64_orpd_a_or_avx src1 src2))
;; Helpers for creating vector `xor` instructions.
-(decl x64_pxor (Xmm XmmMem) Xmm)
+(decl x64_pxor (Xmm XmmMem128) Xmm)
(rule (x64_pxor src1 src2) (x64_pxor_a_or_avx src1 src2))
-(decl x64_xorps (Xmm XmmMem) Xmm)
+(decl x64_xorps (Xmm XmmMem128) Xmm)
(rule (x64_xorps src1 src2) (x64_xorps_a_or_avx src1 src2))
-(decl x64_xorpd (Xmm XmmMem) Xmm)
+(decl x64_xorpd (Xmm XmmMem128) Xmm)
(rule (x64_xorpd src1 src2) (x64_xorpd_a_or_avx src1 src2))
;; Helpers for creating vector `andn` instructions.
-(decl x64_andnps (Xmm XmmMem) Xmm)
+(decl x64_andnps (Xmm XmmMem128) Xmm)
(rule (x64_andnps src1 src2) (x64_andnps_a_or_avx src1 src2))
-(decl x64_andnpd (Xmm XmmMem) Xmm)
+(decl x64_andnpd (Xmm XmmMem128) Xmm)
(rule (x64_andnpd src1 src2) (x64_andnpd_a_or_avx src1 src2))
-(decl x64_pandn (Xmm XmmMem) Xmm)
+(decl x64_pandn (Xmm XmmMem128) Xmm)
(rule (x64_pandn src1 src2) (x64_pandn_a_or_avx src1 src2))
;; Helper for creating vector `mul` instructions.
-(decl x64_mulss (Xmm XmmMem) Xmm)
+(decl x64_mulss (Xmm XmmMem32) Xmm)
(rule (x64_mulss src1 src2) (x64_mulss_a_or_avx src1 src2))
-(decl x64_mulsd (Xmm XmmMem) Xmm)
+(decl x64_mulsd (Xmm XmmMem64) Xmm)
(rule (x64_mulsd src1 src2) (x64_mulsd_a_or_avx src1 src2))
-(decl x64_mulps (Xmm XmmMem) Xmm)
+(decl x64_mulps (Xmm XmmMem128) Xmm)
(rule (x64_mulps src1 src2) (x64_mulps_a_or_avx src1 src2))
-(decl x64_mulpd (Xmm XmmMem) Xmm)
+(decl x64_mulpd (Xmm XmmMem128) Xmm)
(rule (x64_mulpd src1 src2) (x64_mulpd_a_or_avx src1 src2))
-(decl x64_pmullw (Xmm XmmMem) Xmm)
+(decl x64_pmullw (Xmm XmmMem128) Xmm)
(rule (x64_pmullw src1 src2) (x64_pmullw_a_or_avx src1 src2))
-(decl x64_pmulld (Xmm XmmMem) Xmm)
+(decl x64_pmulld (Xmm XmmMem128) Xmm)
(rule (x64_pmulld src1 src2) (x64_pmulld_a_or_avx src1 src2))
-(decl x64_pmulhw (Xmm XmmMem) Xmm)
+(decl x64_pmulhw (Xmm XmmMem128) Xmm)
(rule (x64_pmulhw src1 src2) (x64_pmulhw_a_or_avx src1 src2))
-(decl x64_pmulhrsw (Xmm XmmMem) Xmm)
+(decl x64_pmulhrsw (Xmm XmmMem128) Xmm)
(rule (x64_pmulhrsw src1 src2) (x64_pmulhrsw_a_or_avx src1 src2))
-(decl x64_pmulhuw (Xmm XmmMem) Xmm)
+(decl x64_pmulhuw (Xmm XmmMem128) Xmm)
(rule (x64_pmulhuw src1 src2) (x64_pmulhuw_a_or_avx src1 src2))
-(decl x64_pmuldq (Xmm XmmMem) Xmm)
+(decl x64_pmuldq (Xmm XmmMem128) Xmm)
(rule (x64_pmuldq src1 src2) (x64_pmuldq_a_or_avx src1 src2))
-(decl x64_pmuludq (Xmm XmmMem) Xmm)
+(decl x64_pmuludq (Xmm XmmMem128) Xmm)
(rule (x64_pmuludq src1 src2) (x64_pmuludq_a_or_avx src1 src2))
;; Helpers for creating vector `div` instructions.
-(decl x64_divss (Xmm XmmMem) Xmm)
+(decl x64_divss (Xmm XmmMem32) Xmm)
(rule (x64_divss src1 src2) (x64_divss_a_or_avx src1 src2))
-(decl x64_divsd (Xmm XmmMem) Xmm)
+(decl x64_divsd (Xmm XmmMem64) Xmm)
(rule (x64_divsd src1 src2) (x64_divsd_a_or_avx src1 src2))
-(decl x64_divps (Xmm XmmMem) Xmm)
+(decl x64_divps (Xmm XmmMem128) Xmm)
(rule (x64_divps src1 src2) (x64_divps_a_or_avx src1 src2))
-(decl x64_divpd (Xmm XmmMem) Xmm)
+(decl x64_divpd (Xmm XmmMem128) Xmm)
(rule (x64_divpd src1 src2) (x64_divpd_a_or_avx src1 src2))
;; Helpers for creating `unpack` instructions.
-(decl x64_punpckhwd (Xmm XmmMem) Xmm)
+(decl x64_punpckhwd (Xmm XmmMem128) Xmm)
(rule (x64_punpckhwd src1 src2) (x64_punpckhwd_a_or_avx src1 src2))
-(decl x64_punpcklwd (Xmm XmmMem) Xmm)
+(decl x64_punpcklwd (Xmm XmmMem128) Xmm)
(rule (x64_punpcklwd src1 src2) (x64_punpcklwd_a_or_avx src1 src2))
-(decl x64_punpckldq (Xmm XmmMem) Xmm)
+(decl x64_punpckldq (Xmm XmmMem128) Xmm)
(rule (x64_punpckldq src1 src2) (x64_punpckldq_a_or_avx src1 src2))
-(decl x64_punpckhdq (Xmm XmmMem) Xmm)
+(decl x64_punpckhdq (Xmm XmmMem128) Xmm)
(rule (x64_punpckhdq src1 src2) (x64_punpckhdq_a_or_avx src1 src2))
-(decl x64_punpcklqdq (Xmm XmmMem) Xmm)
+(decl x64_punpcklqdq (Xmm XmmMem128) Xmm)
(rule (x64_punpcklqdq src1 src2) (x64_punpcklqdq_a_or_avx src1 src2))
-(decl x64_punpckhqdq (Xmm XmmMem) Xmm)
+(decl x64_punpckhqdq (Xmm XmmMem128) Xmm)
(rule (x64_punpckhqdq src1 src2) (x64_punpckhqdq_a_or_avx src1 src2))
-(decl x64_unpcklps (Xmm XmmMem) Xmm)
+(decl x64_unpcklps (Xmm XmmMem128) Xmm)
(rule (x64_unpcklps src1 src2) (x64_unpcklps_a_or_avx src1 src2))
-(decl x64_unpcklpd (Xmm XmmMem) Xmm)
+(decl x64_unpcklpd (Xmm XmmMem128) Xmm)
(rule (x64_unpcklpd src1 src2) (x64_unpcklpd_a_or_avx src1 src2))
-(decl x64_unpckhps (Xmm XmmMem) Xmm)
+(decl x64_unpckhps (Xmm XmmMem128) Xmm)
(rule (x64_unpckhps src1 src2) (x64_unpckhps_a_or_avx src1 src2))
-(decl x64_punpcklbw (Xmm XmmMem) Xmm)
+(decl x64_punpcklbw (Xmm XmmMem128) Xmm)
(rule (x64_punpcklbw src1 src2) (x64_punpcklbw_a_or_avx src1 src2))
-(decl x64_punpckhbw (Xmm XmmMem) Xmm)
+(decl x64_punpckhbw (Xmm XmmMem128) Xmm)
(rule (x64_punpckhbw src1 src2) (x64_punpckhbw_a_or_avx src1 src2))
;; Helper for creating `blendvpd` instructions.
-(decl x64_blendvpd (Xmm XmmMem Xmm) Xmm)
+(decl x64_blendvpd (Xmm XmmMem128 Xmm) Xmm)
(rule 0 (x64_blendvpd src1 src2 mask) (x64_blendvpd_rm0 src1 src2 mask))
(rule 1 (x64_blendvpd src1 src2 mask)
(if-let true (has_avx))
(x64_vblendvpd_rvmr src1 src2 mask))
;; Helper for creating `blendvps` instructions.
-(decl x64_blendvps (Xmm XmmMem Xmm) Xmm)
+(decl x64_blendvps (Xmm XmmMem128 Xmm) Xmm)
(rule 0 (x64_blendvps src1 src2 mask) (x64_blendvps_rm0 src1 src2 mask))
(rule 1 (x64_blendvps src1 src2 mask)
(if-let true (has_avx))
(x64_vblendvps_rvmr src1 src2 mask))
;; Helper for creating `pblendvb` instructions.
-(decl x64_pblendvb (Xmm XmmMem Xmm) Xmm)
+(decl x64_pblendvb (Xmm XmmMem128 Xmm) Xmm)
(rule 0 (x64_pblendvb src1 src2 mask) (x64_pblendvb_rm src1 src2 mask))
(rule 1 (x64_pblendvb src1 src2 mask)
(if-let true (has_avx))
(x64_vpblendvb_rvmr src1 src2 mask))
;; Helper for creating `pblendw` instructions.
-(decl x64_pblendw (Xmm XmmMem u8) Xmm)
+(decl x64_pblendw (Xmm XmmMem128 u8) Xmm)
(rule (x64_pblendw src1 src2 imm) (x64_pblendw_rmi_or_avx src1 src2 imm))
;; Helper for creating `movsd`/`movss` instructions which create a new vector
@@ -2468,59 +2783,59 @@
(rule (x64_movlhps src1 src2) (x64_movlhps_rm_or_avx src1 src2))
;; Helpers for creating `pmaxs*` instructions.
-(decl x64_pmaxs (Type Xmm XmmMem) Xmm)
+(decl x64_pmaxs (Type Xmm XmmMem128) Xmm)
(rule (x64_pmaxs $I8X16 x y) (x64_pmaxsb_a_or_avx x y))
(rule (x64_pmaxs $I16X8 x y) (x64_pmaxsw_a_or_avx x y))
(rule (x64_pmaxs $I32X4 x y) (x64_pmaxsd_a_or_avx x y))
;; No $I64X2 version (PMAXSQ) in SSE4.1.
;; Helpers for creating `pmins*` instructions.
-(decl x64_pmins (Type Xmm XmmMem) Xmm)
+(decl x64_pmins (Type Xmm XmmMem128) Xmm)
(rule (x64_pmins $I8X16 x y) (x64_pminsb_a_or_avx x y))
(rule (x64_pmins $I16X8 x y) (x64_pminsw_a_or_avx x y))
(rule (x64_pmins $I32X4 x y) (x64_pminsd_a_or_avx x y))
;; No $I64X2 version (PMINSQ) in SSE4.1.
;; Helpers for creating `pmaxu*` instructions.
-(decl x64_pmaxu (Type Xmm XmmMem) Xmm)
+(decl x64_pmaxu (Type Xmm XmmMem128) Xmm)
(rule (x64_pmaxu $I8X16 x y) (x64_pmaxub_a_or_avx x y))
(rule (x64_pmaxu $I16X8 x y) (x64_pmaxuw_a_or_avx x y))
(rule (x64_pmaxu $I32X4 x y) (x64_pmaxud_a_or_avx x y))
;; No $I64X2 version (PMAXUQ) in SSE4.1.
;; Helper for creating `pminu*` instructions.
-(decl x64_pminu (Type Xmm XmmMem) Xmm)
+(decl x64_pminu (Type Xmm XmmMem128) Xmm)
(rule (x64_pminu $I8X16 x y) (x64_pminub_a_or_avx x y))
(rule (x64_pminu $I16X8 x y) (x64_pminuw_a_or_avx x y))
(rule (x64_pminu $I32X4 x y) (x64_pminud_a_or_avx x y))
;; No $I64X2 version (PMINUQ) in SSE4.1.
;; Helper for creating `packsswb` instructions.
-(decl x64_packsswb (Xmm XmmMem) Xmm)
+(decl x64_packsswb (Xmm XmmMem128) Xmm)
(rule (x64_packsswb src1 src2) (x64_packsswb_a_or_avx src1 src2))
;; Helper for creating `packssdw` instructions.
-(decl x64_packssdw (Xmm XmmMem) Xmm)
+(decl x64_packssdw (Xmm XmmMem128) Xmm)
(rule (x64_packssdw src1 src2) (x64_packssdw_a_or_avx src1 src2))
;; Helper for creating `packuswb` instructions.
-(decl x64_packuswb (Xmm XmmMem) Xmm)
+(decl x64_packuswb (Xmm XmmMem128) Xmm)
(rule (x64_packuswb src1 src2) (x64_packuswb_a_or_avx src1 src2))
;; Helper for creating `packusdw` instructions.
-(decl x64_packusdw (Xmm XmmMem) Xmm)
+(decl x64_packusdw (Xmm XmmMem128) Xmm)
(rule (x64_packusdw src1 src2) (x64_packusdw_a_or_avx src1 src2))
;; Helper for creating `palignr` instructions.
-(decl x64_palignr (Xmm XmmMem u8) Xmm)
+(decl x64_palignr (Xmm XmmMem128 u8) Xmm)
(rule (x64_palignr src1 src2 imm) (x64_palignr_a_or_avx src1 src2 imm))
;; Helpers for creating `cmpp*` instructions.
-(decl x64_cmpp (Type Xmm XmmMem FcmpImm) Xmm)
+(decl x64_cmpp (Type Xmm XmmMem128 FcmpImm) Xmm)
(rule (x64_cmpp $F32X4 x y imm) (x64_cmpps x y imm))
(rule (x64_cmpp $F64X2 x y imm) (x64_cmppd x y imm))
-(decl x64_cmpps (Xmm XmmMem FcmpImm) Xmm)
+(decl x64_cmpps (Xmm XmmMem128 FcmpImm) Xmm)
(rule 1 (x64_cmpps src1 src2 imm)
(if-let true (has_avx))
(x64_vcmpps_b src1 src2 (encode_fcmp_imm imm)))
@@ -2529,42 +2844,39 @@
;; Note that `Size32` is intentional despite this being used for 64-bit
;; operations, since this presumably induces the correct encoding of the
;; instruction.
-(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm)
+(decl x64_cmppd (Xmm XmmMem128 FcmpImm) Xmm)
(rule 1 (x64_cmppd src1 src2 imm)
(if-let true (has_avx))
(x64_vcmppd_b src1 src2 (encode_fcmp_imm imm)))
(rule 0 (x64_cmppd src1 src2 imm) (x64_cmppd_a src1 src2 (encode_fcmp_imm imm)))
-;; Helper for creating `pinsrb` instructions.
+;; Helpers for creating `pinsr*` instructions.
(decl x64_pinsrb (Xmm GprMem u8) Xmm)
(rule 1 (x64_pinsrb src1 src2 lane)
(if-let true (has_avx))
- (x64_vpinsrb_b src1 src2 lane))
-(rule 0 (x64_pinsrb src1 src2 lane) (x64_pinsrb_a src1 src2 lane))
+ (x64_vpinsrb_b src1 (gpr_mem_as_gpr_mem_32 src2) lane))
+(rule 0 (x64_pinsrb src1 src2 lane) (x64_pinsrb_a src1 (gpr_mem_as_gpr_mem_32 src2) lane))
-;; Helper for creating `pinsrw` instructions.
(decl x64_pinsrw (Xmm GprMem u8) Xmm)
(rule 1 (x64_pinsrw src1 src2 lane)
(if-let true (has_avx))
- (x64_vpinsrw_b src1 src2 lane))
-(rule 0 (x64_pinsrw src1 src2 lane) (x64_pinsrw_a src1 src2 lane))
+ (x64_vpinsrw_b src1 (gpr_mem_as_gpr_mem_32 src2) lane))
+(rule 0 (x64_pinsrw src1 src2 lane) (x64_pinsrw_a src1 (gpr_mem_as_gpr_mem_32 src2) lane))
-;; Helper for creating `pinsrd` instructions.
(decl x64_pinsrd (Xmm GprMem u8) Xmm)
(rule 1 (x64_pinsrd src1 src2 lane)
(if-let true (has_avx))
- (x64_vpinsrd_b src1 src2 lane))
-(rule 0 (x64_pinsrd src1 src2 lane) (x64_pinsrd_a src1 src2 lane))
+ (x64_vpinsrd_b src1 (gpr_mem_as_gpr_mem_32 src2) lane))
+(rule 0 (x64_pinsrd src1 src2 lane) (x64_pinsrd_a src1 (gpr_mem_as_gpr_mem_32 src2) lane))
-;; Helper for creating `pinsrq` instructions.
(decl x64_pinsrq (Xmm GprMem u8) Xmm)
(rule 1 (x64_pinsrq src1 src2 lane)
(if-let true (has_avx))
- (x64_vpinsrq_b src1 src2 lane))
-(rule 0 (x64_pinsrq src1 src2 lane) (x64_pinsrq_a src1 src2 lane))
+ (x64_vpinsrq_b src1 (gpr_mem_as_gpr_mem_64 src2) lane))
+(rule 0 (x64_pinsrq src1 src2 lane) (x64_pinsrq_a src1 (gpr_mem_as_gpr_mem_64 src2) lane))
;; Helper for creating `roundss` instructions.
-(decl x64_roundss (XmmMem RoundImm) Xmm)
+(decl x64_roundss (XmmMem32 RoundImm) Xmm)
(rule 1 (x64_roundss src1 round)
(if-let true (has_avx))
(x64_vroundss_rvmi (xmm_zero $F32X4) src1 (encode_round_imm round)))
@@ -2572,7 +2884,7 @@
(x64_roundss_rmi src1 (encode_round_imm round)))
;; Helper for creating `roundsd` instructions.
-(decl x64_roundsd (XmmMem RoundImm) Xmm)
+(decl x64_roundsd (XmmMem64 RoundImm) Xmm)
(rule 1 (x64_roundsd src1 round)
(if-let true (has_avx))
(x64_vroundsd_rvmi (xmm_zero $F64X2) src1 (encode_round_imm round)))
@@ -2580,7 +2892,7 @@
(x64_roundsd_rmi src1 (encode_round_imm round)))
;; Helper for creating `roundps` instructions.
-(decl x64_roundps (XmmMem RoundImm) Xmm)
+(decl x64_roundps (XmmMem128 RoundImm) Xmm)
(rule 1 (x64_roundps src1 round)
(if-let true (has_avx))
(x64_vroundps_rmi src1 (encode_round_imm round)))
@@ -2588,52 +2900,52 @@
(x64_roundps_rmi src1 (encode_round_imm round)))
;; Helper for creating `roundpd` instructions.
-(decl x64_roundpd (XmmMem RoundImm) Xmm)
+(decl x64_roundpd (XmmMem128 RoundImm) Xmm)
(rule 1 (x64_roundpd src1 round)
(if-let true (has_avx))
(x64_vroundpd_rmi src1 (encode_round_imm round)))
(rule 0 (x64_roundpd src1 round)
(x64_roundpd_rmi src1 (encode_round_imm round)))
-;; Helper for creating `pmaddwd` instructions.
-(decl x64_pmaddwd (Xmm XmmMem) Xmm)
+;; Helpers for creating `pmadd*` instructions.
+(decl x64_pmaddwd (Xmm XmmMem128) Xmm)
(rule 0 (x64_pmaddwd src1 src2) (x64_pmaddwd_a_or_avx src1 src2))
-(decl x64_pmaddubsw (Xmm XmmMem) Xmm)
+(decl x64_pmaddubsw (Xmm XmmMem128) Xmm)
(rule (x64_pmaddubsw src1 src2) (x64_pmaddubsw_a_or_avx src1 src2))
;; Helper for creating `insertps` instructions.
-(decl x64_insertps (Xmm XmmMem u8) Xmm)
+(decl x64_insertps (Xmm XmmMem32 u8) Xmm)
(rule 0 (x64_insertps src1 src2 lane) (x64_insertps_a_or_avx src1 src2 lane))
;; Helper for creating `pshufd` instructions.
-(decl x64_pshufd (XmmMem u8) Xmm)
+(decl x64_pshufd (XmmMem128 u8) Xmm)
(rule (x64_pshufd src imm) (x64_pshufd_a src imm))
(rule 1 (x64_pshufd src imm)
(if-let true (has_avx))
(x64_vpshufd_a src imm))
;; Helper for creating `pshufb` instructions.
-(decl x64_pshufb (Xmm XmmMem) Xmm)
+(decl x64_pshufb (Xmm XmmMem128) Xmm)
(rule (x64_pshufb src1 src2) (x64_pshufb_a_or_avx src1 src2))
;; Helper for creating `shufpd` instructions.
-(decl x64_shufpd (Xmm XmmMem u8) Xmm)
+(decl x64_shufpd (Xmm XmmMem128 u8) Xmm)
(rule (x64_shufpd src1 src2 byte) (x64_shufpd_a_or_avx src1 src2 byte))
;; Helper for creating `shufps` instructions.
-(decl x64_shufps (Xmm XmmMem u8) Xmm)
+(decl x64_shufps (Xmm XmmMem128 u8) Xmm)
(rule (x64_shufps src1 src2 byte) (x64_shufps_a_or_avx src1 src2 byte))
;; Helper for creating `pshuflw` instructions.
-(decl x64_pshuflw (XmmMem u8) Xmm)
+(decl x64_pshuflw (XmmMem128 u8) Xmm)
(rule (x64_pshuflw src imm) (x64_pshuflw_a src imm))
(rule 1 (x64_pshuflw src imm)
(if-let true (has_avx))
(x64_vpshuflw_a src imm))
;; Helper for creating `pshufhw` instructions.
-(decl x64_pshufhw (XmmMem u8) Xmm)
+(decl x64_pshufhw (XmmMem128 u8) Xmm)
(rule (x64_pshufhw src imm) (x64_pshufhw_a src imm))
(rule 1 (x64_pshufhw src imm)
(if-let true (has_avx))
@@ -2642,68 +2954,64 @@
;; Helper for creating `vcvtudq2ps` instructions.
-(decl x64_vcvtudq2ps (XmmMem) Xmm)
+(decl x64_vcvtudq2ps (XmmMem128) Xmm)
(rule (x64_vcvtudq2ps src) (x64_vcvtudq2ps_a src))
;; Helper for creating `vpabsq` instructions.
-(decl x64_vpabsq (XmmMem) Xmm)
+(decl x64_vpabsq (XmmMem128) Xmm)
(rule (x64_vpabsq src) (x64_vpabsq_c src))
;; Helper for creating `vpopcntb` instructions.
-(decl x64_vpopcntb (XmmMem) Xmm)
+(decl x64_vpopcntb (XmmMem128) Xmm)
(rule (x64_vpopcntb src) (x64_vpopcntb_a src))
;; Helper for creating `vpmullq` instructions.
-;;
-;; Requires AVX-512 vl and dq.
-(decl x64_vpmullq (Xmm XmmMem) Xmm)
+(decl x64_vpmullq (Xmm XmmMem128) Xmm)
(rule (x64_vpmullq src1 src2) (x64_vpmullq_c src1 src2))
;; Helper for creating `vpermi2b` instructions.
-;;
-;; Requires AVX-512 vl and vbmi extensions.
-(decl x64_vpermi2b (Xmm Xmm XmmMem) Xmm)
+(decl x64_vpermi2b (Xmm Xmm XmmMem128) Xmm)
(rule (x64_vpermi2b src1 src2 src3) (x64_vpermi2b_a src1 src2 src3))
;; Helpers for creating vector `shift` instructions.
(decl x64_psllw (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psllw src1 (is_xmm_mem src2)) (x64_psllw_a_or_avx src1 src2))
+(rule 1 (x64_psllw src1 (is_xmm_mem src2)) (x64_psllw_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psllw src1 (is_imm8_xmm src2)) (x64_psllw_b_or_avx src1 src2))
(decl x64_pslld (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_pslld src1 (is_xmm_mem src2)) (x64_pslld_a_or_avx src1 src2))
+(rule 1 (x64_pslld src1 (is_xmm_mem src2)) (x64_pslld_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_pslld src1 (is_imm8_xmm src2)) (x64_pslld_b_or_avx src1 src2))
(decl x64_psllq (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psllq src1 (is_xmm_mem src2)) (x64_psllq_a_or_avx src1 src2))
+(rule 1 (x64_psllq src1 (is_xmm_mem src2)) (x64_psllq_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psllq src1 (is_imm8_xmm src2)) (x64_psllq_b_or_avx src1 src2))
(decl x64_psrlw (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psrlw src1 (is_xmm_mem src2)) (x64_psrlw_a_or_avx src1 src2))
+(rule 1 (x64_psrlw src1 (is_xmm_mem src2)) (x64_psrlw_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psrlw src1 (is_imm8_xmm src2)) (x64_psrlw_b_or_avx src1 src2))
(decl x64_psrld (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psrld src1 (is_xmm_mem src2)) (x64_psrld_a_or_avx src1 src2))
+(rule 1 (x64_psrld src1 (is_xmm_mem src2)) (x64_psrld_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psrld src1 (is_imm8_xmm src2)) (x64_psrld_b_or_avx src1 src2))
(decl x64_psrlq (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psrlq src1 (is_xmm_mem src2)) (x64_psrlq_a_or_avx src1 src2))
+(rule 1 (x64_psrlq src1 (is_xmm_mem src2)) (x64_psrlq_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psrlq src1 (is_imm8_xmm src2)) (x64_psrlq_b_or_avx src1 src2))
(decl x64_psraw (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psraw src1 (is_xmm_mem src2)) (x64_psraw_a_or_avx src1 src2))
+(rule 1 (x64_psraw src1 (is_xmm_mem src2)) (x64_psraw_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psraw src1 (is_imm8_xmm src2)) (x64_psraw_b_or_avx src1 src2))
(decl x64_psrad (Xmm XmmMemImm) Xmm)
-(rule 1 (x64_psrad src1 (is_xmm_mem src2)) (x64_psrad_a_or_avx src1 src2))
+(rule 1 (x64_psrad src1 (is_xmm_mem src2)) (x64_psrad_a_or_avx src1 (xmm_mem_as_xmm_mem_128 src2)))
(rule 0 (x64_psrad src1 (is_imm8_xmm src2)) (x64_psrad_b_or_avx src1 src2))
;; Helper for creating `vpsraq` instructions.
-(decl x64_vpsraq (Xmm XmmMem) Xmm)
+(decl x64_vpsraq (Xmm XmmMem128) Xmm)
(rule (x64_vpsraq src1 src2) (x64_vpsraq_g src1 src2))
;; Helper for creating `vpsraq` instructions.
-(decl x64_vpsraq_imm (XmmMem u8) Xmm)
+(decl x64_vpsraq_imm (XmmMem128 u8) Xmm)
(rule (x64_vpsraq_imm src imm) (x64_vpsraq_f src imm))
;; Helper for creating `pextr*` instructions.
@@ -2784,21 +3092,21 @@
;; Helper for creating `lzcnt` instructions.
(decl x64_lzcnt (Type GprMem) Gpr)
-(rule (x64_lzcnt $I16 src) (x64_lzcntw_rm src))
-(rule (x64_lzcnt $I32 src) (x64_lzcntl_rm src))
-(rule (x64_lzcnt $I64 src) (x64_lzcntq_rm src))
+(rule (x64_lzcnt ty @ $I16 src) (x64_lzcntw_rm (gpr_mem_16_for_ty ty src)))
+(rule (x64_lzcnt ty @ $I32 src) (x64_lzcntl_rm (gpr_mem_32_for_ty ty src)))
+(rule (x64_lzcnt ty @ $I64 src) (x64_lzcntq_rm (gpr_mem_64_for_ty ty src)))
;; Helper for creating `tzcnt` instructions.
(decl x64_tzcnt (Type GprMem) Gpr)
-(rule (x64_tzcnt $I16 src) (x64_tzcntw_a src))
-(rule (x64_tzcnt $I32 src) (x64_tzcntl_a src))
-(rule (x64_tzcnt $I64 src) (x64_tzcntq_a src))
+(rule (x64_tzcnt ty @ $I16 src) (x64_tzcntw_a (gpr_mem_16_for_ty ty src)))
+(rule (x64_tzcnt ty @ $I32 src) (x64_tzcntl_a (gpr_mem_32_for_ty ty src)))
+(rule (x64_tzcnt ty @ $I64 src) (x64_tzcntq_a (gpr_mem_64_for_ty ty src)))
;; Helper for creating `bsr` instructions.
(decl x64_bsr (Type GprMem) ProducesFlags)
-(rule (x64_bsr $I16 src) (asm_produce_flags (x64_bsrw_rm_raw src)))
-(rule (x64_bsr $I32 src) (asm_produce_flags (x64_bsrl_rm_raw src)))
-(rule (x64_bsr $I64 src) (asm_produce_flags (x64_bsrq_rm_raw src)))
+(rule (x64_bsr ty @ $I16 src) (asm_produce_flags (x64_bsrw_rm_raw (gpr_mem_16_for_ty ty src))))
+(rule (x64_bsr ty @ $I32 src) (asm_produce_flags (x64_bsrl_rm_raw (gpr_mem_32_for_ty ty src))))
+(rule (x64_bsr ty @ $I64 src) (asm_produce_flags (x64_bsrq_rm_raw (gpr_mem_64_for_ty ty src))))
;; Helper for creating `bsr + cmov` instruction pairs that produce the
;; result of the `bsr`, or `alt` if the input was zero.
@@ -2814,9 +3122,9 @@
;; Helper for creating `bsf` instructions.
(decl x64_bsf (Type GprMem) ProducesFlags)
-(rule (x64_bsf $I16 src) (asm_produce_flags (x64_bsfw_rm_raw src)))
-(rule (x64_bsf $I32 src) (asm_produce_flags (x64_bsfl_rm_raw src)))
-(rule (x64_bsf $I64 src) (asm_produce_flags (x64_bsfq_rm_raw src)))
+(rule (x64_bsf ty @ $I16 src) (asm_produce_flags (x64_bsfw_rm_raw (gpr_mem_16_for_ty ty src))))
+(rule (x64_bsf ty @ $I32 src) (asm_produce_flags (x64_bsfl_rm_raw (gpr_mem_32_for_ty ty src))))
+(rule (x64_bsf ty @ $I64 src) (asm_produce_flags (x64_bsfq_rm_raw (gpr_mem_64_for_ty ty src))))
;; Helper for creating `bsf + cmov` instruction pairs that produce the
;; result of the `bsf`, or `alt` if the input was zero.
@@ -2832,137 +3140,212 @@
;; Helper for creating `blsi` instructions.
(decl x64_blsi (Type GprMem) Gpr)
-(rule (x64_blsi $I32 src) (x64_blsil_vm src))
-(rule (x64_blsi $I64 src) (x64_blsiq_vm src))
+(rule (x64_blsi ty @ $I32 src) (x64_blsil_vm (gpr_mem_32_for_ty ty src)))
+(rule (x64_blsi ty @ $I64 src) (x64_blsiq_vm (gpr_mem_64_for_ty ty src)))
;; Helper for creating `blsmsk` instructions.
(decl x64_blsmsk (Type GprMem) Gpr)
-(rule (x64_blsmsk $I32 src) (x64_blsmskl_vm src))
-(rule (x64_blsmsk $I64 src) (x64_blsmskq_vm src))
+(rule (x64_blsmsk ty @ $I32 src) (x64_blsmskl_vm (gpr_mem_32_for_ty ty src)))
+(rule (x64_blsmsk ty @ $I64 src) (x64_blsmskq_vm (gpr_mem_64_for_ty ty src)))
;; Helper for creating `blsr` instructions.
(decl x64_blsr (Type GprMem) Gpr)
-(rule (x64_blsr $I32 src) (x64_blsrl_vm src))
-(rule (x64_blsr $I64 src) (x64_blsrq_vm src))
+(rule (x64_blsr ty @ $I32 src) (x64_blsrl_vm (gpr_mem_32_for_ty ty src)))
+(rule (x64_blsr ty @ $I64 src) (x64_blsrq_vm (gpr_mem_64_for_ty ty src)))
;; Helper for creating `bt` instructions.
(decl x64_bt (Type GprMem Gpr) ProducesFlags)
-(rule (x64_bt $I16 src1 src2) (x64_btw_mr src1 src2))
-(rule (x64_bt $I32 src1 src2) (x64_btl_mr src1 src2))
-(rule (x64_bt $I64 src1 src2) (x64_btq_mr src1 src2))
+(rule (x64_bt ty @ $I16 src1 src2) (x64_btw_mr (gpr_mem_16_for_ty ty src1) src2))
+(rule (x64_bt ty @ $I32 src1 src2) (x64_btl_mr (gpr_mem_32_for_ty ty src1) src2))
+(rule (x64_bt ty @ $I64 src1 src2) (x64_btq_mr (gpr_mem_64_for_ty ty src1) src2))
;; Helper for creating `bt` instructions.
(decl x64_bt_imm (Type GprMem u8) ProducesFlags)
-(rule (x64_bt_imm $I16 src imm) (x64_btw_mi src imm))
-(rule (x64_bt_imm $I32 src imm) (x64_btl_mi src imm))
-(rule (x64_bt_imm $I64 src imm) (x64_btq_mi src imm))
+(rule (x64_bt_imm ty @ $I16 src imm) (x64_btw_mi (gpr_mem_16_for_ty ty src) imm))
+(rule (x64_bt_imm ty @ $I32 src imm) (x64_btl_mi (gpr_mem_32_for_ty ty src) imm))
+(rule (x64_bt_imm ty @ $I64 src imm) (x64_btq_mi (gpr_mem_64_for_ty ty src) imm))
;; Helper for creating `sarx` instructions.
(decl x64_sarx (Type GprMem Gpr) Gpr)
-(rule (x64_sarx $I32 val amt) (x64_sarxl_rmv val amt))
-(rule (x64_sarx $I64 val amt) (x64_sarxq_rmv val amt))
+(rule (x64_sarx ty @ $I32 val amt) (x64_sarxl_rmv (gpr_mem_32_for_ty ty val) amt))
+(rule (x64_sarx ty @ $I64 val amt) (x64_sarxq_rmv (gpr_mem_64_for_ty ty val) amt))
;; Helper for creating `shrx` instructions.
(decl x64_shrx (Type GprMem Gpr) Gpr)
-(rule (x64_shrx $I32 val amt) (x64_shrxl_rmv val amt))
-(rule (x64_shrx $I64 val amt) (x64_shrxq_rmv val amt))
+(rule (x64_shrx ty @ $I32 val amt) (x64_shrxl_rmv (gpr_mem_32_for_ty ty val) amt))
+(rule (x64_shrx ty @ $I64 val amt) (x64_shrxq_rmv (gpr_mem_64_for_ty ty val) amt))
;; Helper for creating `shlx` instructions.
(decl x64_shlx (Type GprMem Gpr) Gpr)
-(rule (x64_shlx $I32 val amt) (x64_shlxl_rmv val amt))
-(rule (x64_shlx $I64 val amt) (x64_shlxq_rmv val amt))
+(rule (x64_shlx ty @ $I32 val amt) (x64_shlxl_rmv (gpr_mem_32_for_ty ty val) amt))
+(rule (x64_shlx ty @ $I64 val amt) (x64_shlxq_rmv (gpr_mem_64_for_ty ty val) amt))
;; Helper for creating `rorx` instructions.
(decl x64_rorx (Type GprMem u8) Gpr)
-(rule (x64_rorx $I32 src imm) (x64_rorxl_rmi src imm))
-(rule (x64_rorx $I64 src imm) (x64_rorxq_rmi src imm))
+(rule (x64_rorx ty @ $I32 src imm) (x64_rorxl_rmi (gpr_mem_32_for_ty ty src) imm))
+(rule (x64_rorx ty @ $I64 src imm) (x64_rorxq_rmi (gpr_mem_64_for_ty ty src) imm))
;; Helper for creating `popcnt` instructions.
(decl x64_popcnt (Type GprMem) Gpr)
-(rule (x64_popcnt $I16 src) (x64_popcntw_rm src))
-(rule (x64_popcnt $I32 src) (x64_popcntl_rm src))
-(rule (x64_popcnt $I64 src) (x64_popcntq_rm src))
+(rule (x64_popcnt ty @ $I16 src) (x64_popcntw_rm (gpr_mem_16_for_ty ty src)))
+(rule (x64_popcnt ty @ $I32 src) (x64_popcntl_rm (gpr_mem_32_for_ty ty src)))
+(rule (x64_popcnt ty @ $I64 src) (x64_popcntq_rm (gpr_mem_64_for_ty ty src)))
-;; Helpers for creating `min*` instructions.
-(decl x64_minss (Xmm XmmMem) Xmm)
+;; Helpers for creating `min*` instructions. Scalar `minss`/`minsd` read
+;; 32/64 bits; packed `minps`/`minpd` are 128-bit.
+(decl x64_minss (Xmm XmmMem32) Xmm)
(rule (x64_minss src1 src2) (x64_minss_a_or_avx src1 src2))
-(decl x64_minsd (Xmm XmmMem) Xmm)
+(decl x64_minsd (Xmm XmmMem64) Xmm)
(rule (x64_minsd src1 src2) (x64_minsd_a_or_avx src1 src2))
-(decl x64_minps (Xmm XmmMem) Xmm)
+(decl x64_minps (Xmm XmmMem128) Xmm)
(rule (x64_minps src1 src2) (x64_minps_a_or_avx src1 src2))
-(decl x64_minpd (Xmm XmmMem) Xmm)
+(decl x64_minpd (Xmm XmmMem128) Xmm)
(rule (x64_minpd src1 src2) (x64_minpd_a_or_avx src1 src2))
-(decl x64_maxss (Xmm XmmMem) Xmm)
+(decl x64_maxss (Xmm XmmMem32) Xmm)
(rule (x64_maxss src1 src2) (x64_maxss_a_or_avx src1 src2))
-(decl x64_maxsd (Xmm XmmMem) Xmm)
+(decl x64_maxsd (Xmm XmmMem64) Xmm)
(rule (x64_maxsd src1 src2) (x64_maxsd_a_or_avx src1 src2))
-(decl x64_maxps (Xmm XmmMem) Xmm)
+(decl x64_maxps (Xmm XmmMem128) Xmm)
(rule (x64_maxps src1 src2) (x64_maxps_a_or_avx src1 src2))
-(decl x64_maxpd (Xmm XmmMem) Xmm)
+(decl x64_maxpd (Xmm XmmMem128) Xmm)
(rule (x64_maxpd src1 src2) (x64_maxpd_a_or_avx src1 src2))
-;; Helper for creating `vfmadd213*` instructions
-(decl x64_vfmadd213 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfmadd213 $F32 a b c) (x64_vfmadd213ss_a a b c))
-(rule (x64_vfmadd213 $F64 a b c) (x64_vfmadd213sd_a a b c))
-(rule (x64_vfmadd213 $F32X4 a b c) (x64_vfmadd213ps_a a b c))
-(rule (x64_vfmadd213 $F64X2 a b c) (x64_vfmadd213pd_a a b c))
-
-;; Helper for creating `vfmadd132*` instructions
-(decl x64_vfmadd132 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfmadd132 $F32 a b c) (x64_vfmadd132ss_a a b c))
-(rule (x64_vfmadd132 $F64 a b c) (x64_vfmadd132sd_a a b c))
-(rule (x64_vfmadd132 $F32X4 a b c) (x64_vfmadd132ps_a a b c))
-(rule (x64_vfmadd132 $F64X2 a b c) (x64_vfmadd132pd_a a b c))
-
-;; Helper for creating `vfnmadd213*` instructions
-(decl x64_vfnmadd213 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfnmadd213 $F32 a b c) (x64_vfnmadd213ss_a a b c))
-(rule (x64_vfnmadd213 $F64 a b c) (x64_vfnmadd213sd_a a b c))
-(rule (x64_vfnmadd213 $F32X4 a b c) (x64_vfnmadd213ps_a a b c))
-(rule (x64_vfnmadd213 $F64X2 a b c) (x64_vfnmadd213pd_a a b c))
-
-;; Helper for creating `vfnmadd132*` instructions
-(decl x64_vfnmadd132 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfnmadd132 $F32 a b c) (x64_vfnmadd132ss_a a b c))
-(rule (x64_vfnmadd132 $F64 a b c) (x64_vfnmadd132sd_a a b c))
-(rule (x64_vfnmadd132 $F32X4 a b c) (x64_vfnmadd132ps_a a b c))
-(rule (x64_vfnmadd132 $F64X2 a b c) (x64_vfnmadd132pd_a a b c))
-
-;; Helper for creating `vfmsub213*` instructions
-(decl x64_vfmsub213 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfmsub213 $F32 a b c) (x64_vfmsub213ss_a a b c))
-(rule (x64_vfmsub213 $F64 a b c) (x64_vfmsub213sd_a a b c))
-(rule (x64_vfmsub213 $F32X4 a b c) (x64_vfmsub213ps_a a b c))
-(rule (x64_vfmsub213 $F64X2 a b c) (x64_vfmsub213pd_a a b c))
-
-;; Helper for creating `vfmsub132*` instructions
-(decl x64_vfmsub132 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfmsub132 $F32 a b c) (x64_vfmsub132ss_a a b c))
-(rule (x64_vfmsub132 $F64 a b c) (x64_vfmsub132sd_a a b c))
-(rule (x64_vfmsub132 $F32X4 a b c) (x64_vfmsub132ps_a a b c))
-(rule (x64_vfmsub132 $F64X2 a b c) (x64_vfmsub132pd_a a b c))
-
-;; Helper for creating `vfnmsub213*` instructions
-(decl x64_vfnmsub213 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfnmsub213 $F32 a b c) (x64_vfnmsub213ss_a a b c))
-(rule (x64_vfnmsub213 $F64 a b c) (x64_vfnmsub213sd_a a b c))
-(rule (x64_vfnmsub213 $F32X4 a b c) (x64_vfnmsub213ps_a a b c))
-(rule (x64_vfnmsub213 $F64X2 a b c) (x64_vfnmsub213pd_a a b c))
-
-;; Helper for creating `vfnmsub132*` instructions
-(decl x64_vfnmsub132 (Type Xmm Xmm XmmMem) Xmm)
-(rule (x64_vfnmsub132 $F32 a b c) (x64_vfnmsub132ss_a a b c))
-(rule (x64_vfnmsub132 $F64 a b c) (x64_vfnmsub132sd_a a b c))
-(rule (x64_vfnmsub132 $F32X4 a b c) (x64_vfnmsub132ps_a a b c))
-(rule (x64_vfnmsub132 $F64X2 a b c) (x64_vfnmsub132pd_a a b c))
+;; FMA helpers.
+
+;; --- vfmadd213 ---
+(decl x64_vfmadd213 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfmadd213 $F32 a b c) (x64_vfmadd213_f32 a b c))
+(rule (x64_vfmadd213 $F64 a b c) (x64_vfmadd213_f64 a b c))
+(rule (x64_vfmadd213 $F32X4 a b c) (x64_vfmadd213_f32x4 a b c))
+(rule (x64_vfmadd213 $F64X2 a b c) (x64_vfmadd213_f64x2 a b c))
+
+(decl x64_vfmadd213_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfmadd213_f32 a b c) (x64_vfmadd213ss_a a b c))
+(decl x64_vfmadd213_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfmadd213_f64 a b c) (x64_vfmadd213sd_a a b c))
+(decl x64_vfmadd213_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmadd213_f32x4 a b c) (x64_vfmadd213ps_a a b c))
+(decl x64_vfmadd213_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmadd213_f64x2 a b c) (x64_vfmadd213pd_a a b c))
+
+;; --- vfmadd132 ---
+(decl x64_vfmadd132 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfmadd132 $F32 a b c) (x64_vfmadd132_f32 a b c))
+(rule (x64_vfmadd132 $F64 a b c) (x64_vfmadd132_f64 a b c))
+(rule (x64_vfmadd132 $F32X4 a b c) (x64_vfmadd132_f32x4 a b c))
+(rule (x64_vfmadd132 $F64X2 a b c) (x64_vfmadd132_f64x2 a b c))
+
+(decl x64_vfmadd132_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfmadd132_f32 a b c) (x64_vfmadd132ss_a a b c))
+(decl x64_vfmadd132_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfmadd132_f64 a b c) (x64_vfmadd132sd_a a b c))
+(decl x64_vfmadd132_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmadd132_f32x4 a b c) (x64_vfmadd132ps_a a b c))
+(decl x64_vfmadd132_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmadd132_f64x2 a b c) (x64_vfmadd132pd_a a b c))
+
+;; --- vfnmadd213 ---
+(decl x64_vfnmadd213 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfnmadd213 $F32 a b c) (x64_vfnmadd213_f32 a b c))
+(rule (x64_vfnmadd213 $F64 a b c) (x64_vfnmadd213_f64 a b c))
+(rule (x64_vfnmadd213 $F32X4 a b c) (x64_vfnmadd213_f32x4 a b c))
+(rule (x64_vfnmadd213 $F64X2 a b c) (x64_vfnmadd213_f64x2 a b c))
+
+(decl x64_vfnmadd213_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfnmadd213_f32 a b c) (x64_vfnmadd213ss_a a b c))
+(decl x64_vfnmadd213_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfnmadd213_f64 a b c) (x64_vfnmadd213sd_a a b c))
+(decl x64_vfnmadd213_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmadd213_f32x4 a b c) (x64_vfnmadd213ps_a a b c))
+(decl x64_vfnmadd213_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmadd213_f64x2 a b c) (x64_vfnmadd213pd_a a b c))
+
+;; --- vfnmadd132 ---
+(decl x64_vfnmadd132 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfnmadd132 $F32 a b c) (x64_vfnmadd132_f32 a b c))
+(rule (x64_vfnmadd132 $F64 a b c) (x64_vfnmadd132_f64 a b c))
+(rule (x64_vfnmadd132 $F32X4 a b c) (x64_vfnmadd132_f32x4 a b c))
+(rule (x64_vfnmadd132 $F64X2 a b c) (x64_vfnmadd132_f64x2 a b c))
+
+(decl x64_vfnmadd132_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfnmadd132_f32 a b c) (x64_vfnmadd132ss_a a b c))
+(decl x64_vfnmadd132_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfnmadd132_f64 a b c) (x64_vfnmadd132sd_a a b c))
+(decl x64_vfnmadd132_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmadd132_f32x4 a b c) (x64_vfnmadd132ps_a a b c))
+(decl x64_vfnmadd132_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmadd132_f64x2 a b c) (x64_vfnmadd132pd_a a b c))
+
+;; --- vfmsub213 ---
+(decl x64_vfmsub213 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfmsub213 $F32 a b c) (x64_vfmsub213_f32 a b c))
+(rule (x64_vfmsub213 $F64 a b c) (x64_vfmsub213_f64 a b c))
+(rule (x64_vfmsub213 $F32X4 a b c) (x64_vfmsub213_f32x4 a b c))
+(rule (x64_vfmsub213 $F64X2 a b c) (x64_vfmsub213_f64x2 a b c))
+
+(decl x64_vfmsub213_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfmsub213_f32 a b c) (x64_vfmsub213ss_a a b c))
+(decl x64_vfmsub213_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfmsub213_f64 a b c) (x64_vfmsub213sd_a a b c))
+(decl x64_vfmsub213_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmsub213_f32x4 a b c) (x64_vfmsub213ps_a a b c))
+(decl x64_vfmsub213_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmsub213_f64x2 a b c) (x64_vfmsub213pd_a a b c))
+
+;; --- vfmsub132 ---
+(decl x64_vfmsub132 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfmsub132 $F32 a b c) (x64_vfmsub132_f32 a b c))
+(rule (x64_vfmsub132 $F64 a b c) (x64_vfmsub132_f64 a b c))
+(rule (x64_vfmsub132 $F32X4 a b c) (x64_vfmsub132_f32x4 a b c))
+(rule (x64_vfmsub132 $F64X2 a b c) (x64_vfmsub132_f64x2 a b c))
+
+(decl x64_vfmsub132_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfmsub132_f32 a b c) (x64_vfmsub132ss_a a b c))
+(decl x64_vfmsub132_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfmsub132_f64 a b c) (x64_vfmsub132sd_a a b c))
+(decl x64_vfmsub132_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmsub132_f32x4 a b c) (x64_vfmsub132ps_a a b c))
+(decl x64_vfmsub132_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfmsub132_f64x2 a b c) (x64_vfmsub132pd_a a b c))
+
+;; --- vfnmsub213 ---
+(decl x64_vfnmsub213 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfnmsub213 $F32 a b c) (x64_vfnmsub213_f32 a b c))
+(rule (x64_vfnmsub213 $F64 a b c) (x64_vfnmsub213_f64 a b c))
+(rule (x64_vfnmsub213 $F32X4 a b c) (x64_vfnmsub213_f32x4 a b c))
+(rule (x64_vfnmsub213 $F64X2 a b c) (x64_vfnmsub213_f64x2 a b c))
+
+(decl x64_vfnmsub213_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfnmsub213_f32 a b c) (x64_vfnmsub213ss_a a b c))
+(decl x64_vfnmsub213_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfnmsub213_f64 a b c) (x64_vfnmsub213sd_a a b c))
+(decl x64_vfnmsub213_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmsub213_f32x4 a b c) (x64_vfnmsub213ps_a a b c))
+(decl x64_vfnmsub213_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmsub213_f64x2 a b c) (x64_vfnmsub213pd_a a b c))
+
+;; --- vfnmsub132 ---
+(decl x64_vfnmsub132 (Type Xmm Xmm Value) Xmm)
+(rule (x64_vfnmsub132 $F32 a b c) (x64_vfnmsub132_f32 a b c))
+(rule (x64_vfnmsub132 $F64 a b c) (x64_vfnmsub132_f64 a b c))
+(rule (x64_vfnmsub132 $F32X4 a b c) (x64_vfnmsub132_f32x4 a b c))
+(rule (x64_vfnmsub132 $F64X2 a b c) (x64_vfnmsub132_f64x2 a b c))
+
+(decl x64_vfnmsub132_f32 (Xmm Xmm XmmMem32) Xmm)
+(rule (x64_vfnmsub132_f32 a b c) (x64_vfnmsub132ss_a a b c))
+(decl x64_vfnmsub132_f64 (Xmm Xmm XmmMem64) Xmm)
+(rule (x64_vfnmsub132_f64 a b c) (x64_vfnmsub132sd_a a b c))
+(decl x64_vfnmsub132_f32x4 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmsub132_f32x4 a b c) (x64_vfnmsub132ps_a a b c))
+(decl x64_vfnmsub132_f64x2 (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vfnmsub132_f64x2 a b c) (x64_vfnmsub132pd_a a b c))
;; Note, the `vfmsub231` and `vfnmsub231*` instructions are omitted, because
;; instruction selection happens before register allocation and therefore there
@@ -2976,54 +3359,54 @@
;; operation on the low bits of the second register. This introduces
;; a data-dependency on the contents of the first register which is modeled
;; here.
-(decl x64_sqrtss (Xmm XmmMem) Xmm)
+(decl x64_sqrtss (Xmm XmmMem32) Xmm)
(rule (x64_sqrtss x y) (x64_sqrtss_a_or_avx x y))
;; Helper for creating `sqrtsd` instructions.
;;
;; NB: see `x64_sqrtss` for explanation of why this has two args.
-(decl x64_sqrtsd (Xmm XmmMem) Xmm)
+(decl x64_sqrtsd (Xmm XmmMem64) Xmm)
(rule 0 (x64_sqrtsd x y) (x64_sqrtsd_a_or_avx x y))
;; Helper for creating `sqrtps` instructions.
-(decl x64_sqrtps (XmmMem) Xmm)
+(decl x64_sqrtps (XmmMem128) Xmm)
(rule (x64_sqrtps x) (x64_sqrtps_a_or_avx x))
;; Helper for creating `sqrtpd` instructions.
-(decl x64_sqrtpd (XmmMem) Xmm)
+(decl x64_sqrtpd (XmmMem128) Xmm)
(rule (x64_sqrtpd x) (x64_sqrtpd_a_or_avx x))
;; Helper for creating `reciprocal` instructions.
;;
;; Helper for creating `rcpps` instructions.
-(decl x64_rcpps (XmmMem) Xmm)
+(decl x64_rcpps (XmmMem128) Xmm)
(rule (x64_rcpps x) (x64_rcpps_rm_or_avx x))
;; Helper for creating `rcpss` instructions.
-(decl x64_rcpss (XmmMem) Xmm)
+(decl x64_rcpss (XmmMem32) Xmm)
(rule (x64_rcpss x) (x64_rcpss_rm x))
;; Helper for creating `vrsqrtss` instructions.
-(decl x64_vrcpss (Xmm XmmMem) Xmm)
+(decl x64_vrcpss (Xmm XmmMem32) Xmm)
(rule (x64_vrcpss x y) (x64_vrcpss_rvm x y))
;; Helper for creating `rsqrtps` instructions.
-(decl x64_rsqrtps (XmmMem) Xmm)
+(decl x64_rsqrtps (XmmMem128) Xmm)
(rule (x64_rsqrtps x) (x64_rsqrtps_rm_or_avx x))
;; Helper for creating `rsqrtss` instructions.
-(decl x64_rsqrtss (XmmMem) Xmm)
+(decl x64_rsqrtss (XmmMem32) Xmm)
(rule (x64_rsqrtss x) (x64_rsqrtss_rm x))
;; Helper for creating `vrsqrtss` instructions.
-(decl x64_vrsqrtss (Xmm XmmMem) Xmm)
+(decl x64_vrsqrtss (Xmm XmmMem32) Xmm)
(rule (x64_vrsqrtss x y) (x64_vrsqrtss_rvm x y))
;; Helper for creating `cvtss2sd` instructions.
;;
;; NB: see `x64_sqrtss` for why this has two args (same reasoning, different op)
-(decl x64_cvtss2sd (Xmm XmmMem) Xmm)
+(decl x64_cvtss2sd (Xmm XmmMem32) Xmm)
(rule 1 (x64_cvtss2sd x y)
(if-let true (has_avx))
(x64_vcvtss2sd_b x y))
@@ -3032,35 +3415,35 @@
;; Helper for creating `cvtsd2ss` instructions.
;;
;; NB: see `x64_sqrtss` for why this has two args (same reasoning, different op)
-(decl x64_cvtsd2ss (Xmm XmmMem) Xmm)
+(decl x64_cvtsd2ss (Xmm XmmMem64) Xmm)
(rule 1 (x64_cvtsd2ss x y)
(if-let true (has_avx))
(x64_vcvtsd2ss_b x y))
(rule 0 (x64_cvtsd2ss x y) (x64_cvtsd2ss_a x y))
;; Helper for creating `cvtdq2ps` instructions.
-(decl x64_cvtdq2ps (XmmMem) Xmm)
+(decl x64_cvtdq2ps (XmmMem128) Xmm)
(rule 1 (x64_cvtdq2ps x)
(if-let true (has_avx))
(x64_vcvtdq2ps_a x))
(rule (x64_cvtdq2ps x) (x64_cvtdq2ps_a x))
;; Helper for creating `cvtps2pd` instructions.
-(decl x64_cvtps2pd (XmmMem) Xmm)
+(decl x64_cvtps2pd (XmmMem64) Xmm)
(rule 1 (x64_cvtps2pd x)
(if-let true (has_avx))
(x64_vcvtps2pd_a x))
(rule 0 (x64_cvtps2pd x) (x64_cvtps2pd_a x))
;; Helper for creating `cvtpd2ps` instructions.
-(decl x64_cvtpd2ps (XmmMem) Xmm)
+(decl x64_cvtpd2ps (XmmMem128) Xmm)
(rule 1 (x64_cvtpd2ps x)
(if-let true (has_avx))
(x64_vcvtpd2ps_a x))
(rule 0 (x64_cvtpd2ps x) (x64_cvtpd2ps_a x))
;; Helper for creating `cvtdq2pd` instructions.
-(decl x64_cvtdq2pd (XmmMem) Xmm)
+(decl x64_cvtdq2pd (XmmMem64) Xmm)
(rule 1 (x64_cvtdq2pd x)
(if-let true (has_avx))
(x64_vcvtdq2pd_a x))
@@ -3068,42 +3451,43 @@
;; Helper for creating `cvtsi2ss` instructions.
(decl x64_cvtsi2ss (Type Xmm GprMem) Xmm)
-(rule 1 (x64_cvtsi2ss $I32 x y)
+(rule 1 (x64_cvtsi2ss ty @ $I32 x y)
(if-let true (has_avx))
- (x64_vcvtsi2ssl_b x y))
-(rule 1 (x64_cvtsi2ss $I64 x y)
+ (x64_vcvtsi2ssl_b x (gpr_mem_32_for_ty ty y)))
+(rule 1 (x64_cvtsi2ss ty @ $I64 x y)
(if-let true (has_avx))
- (x64_vcvtsi2ssq_b x y))
-(rule 0 (x64_cvtsi2ss $I32 x y) (x64_cvtsi2ssl_a x y))
-(rule 0 (x64_cvtsi2ss $I64 x y) (x64_cvtsi2ssq_a x y))
+ (x64_vcvtsi2ssq_b x (gpr_mem_64_for_ty ty y)))
+(rule 0 (x64_cvtsi2ss ty @ $I32 x y) (x64_cvtsi2ssl_a x (gpr_mem_32_for_ty ty y)))
+(rule 0 (x64_cvtsi2ss ty @ $I64 x y) (x64_cvtsi2ssq_a x (gpr_mem_64_for_ty ty y)))
;; Helper for creating `cvtsi2sd` instructions.
(decl x64_cvtsi2sd (Type Xmm GprMem) Xmm)
-(rule 1 (x64_cvtsi2sd $I32 x y)
+(rule 1 (x64_cvtsi2sd ty @ $I32 x y)
(if-let true (has_avx))
- (x64_vcvtsi2sdl_b x y))
-(rule 1 (x64_cvtsi2sd $I64 x y)
+ (x64_vcvtsi2sdl_b x (gpr_mem_32_for_ty ty y)))
+(rule 1 (x64_cvtsi2sd ty @ $I64 x y)
(if-let true (has_avx))
- (x64_vcvtsi2sdq_b x y))
-(rule 0 (x64_cvtsi2sd $I32 x y) (x64_cvtsi2sdl_a x y))
-(rule 0 (x64_cvtsi2sd $I64 x y) (x64_cvtsi2sdq_a x y))
+ (x64_vcvtsi2sdq_b x (gpr_mem_64_for_ty ty y)))
+(rule 0 (x64_cvtsi2sd ty @ $I32 x y) (x64_cvtsi2sdl_a x (gpr_mem_32_for_ty ty y)))
+(rule 0 (x64_cvtsi2sd ty @ $I64 x y) (x64_cvtsi2sdq_a x (gpr_mem_64_for_ty ty y)))
;; Helper for creating `cvttps2dq` instructions.
-(decl x64_cvttps2dq (XmmMem) Xmm)
+(decl x64_cvttps2dq (XmmMem128) Xmm)
(rule 1 (x64_cvttps2dq x)
(if-let true (has_avx))
(x64_vcvttps2dq_a x))
(rule 0 (x64_cvttps2dq x) (x64_cvttps2dq_a x))
;; Helper for creating `cvttpd2dq` instructions.
-(decl x64_cvttpd2dq (XmmMem) Xmm)
+(decl x64_cvttpd2dq (XmmMem128) Xmm)
(rule 1 (x64_cvttpd2dq x)
(if-let true (has_avx))
(x64_vcvttpd2dq_a x))
(rule 0 (x64_cvttpd2dq x) (x64_cvttpd2dq_a x))
;; Helpers for creating `pcmpeq*` instructions.
-(decl x64_pcmpeq (Type Xmm XmmMem) Xmm)
+;; 128-bit packed compare-for-equal, dispatched by lane type.
+(decl x64_pcmpeq (Type Xmm XmmMem128) Xmm)
(rule (x64_pcmpeq $I8X16 x y) (x64_pcmpeqb x y))
(rule (x64_pcmpeq $I16X8 x y) (x64_pcmpeqw x y))
(rule (x64_pcmpeq $I32X4 x y) (x64_pcmpeqd x y))
@@ -3121,21 +3505,22 @@
(cmp32_swapped Xmm (x64_pshufd cmp32 0b10_11_00_01)))
(x64_pand cmp32 cmp32_swapped)))
-;; Helpers for creating `pcmpeq*` instructions.
-(decl x64_pcmpeqb (Xmm XmmMem) Xmm)
+;; Helpers for creating `pcmpeq*` instructions (all 128-bit packed).
+(decl x64_pcmpeqb (Xmm XmmMem128) Xmm)
(rule (x64_pcmpeqb x y) (x64_pcmpeqb_a_or_avx x y))
-(decl x64_pcmpeqw (Xmm XmmMem) Xmm)
+(decl x64_pcmpeqw (Xmm XmmMem128) Xmm)
(rule (x64_pcmpeqw x y) (x64_pcmpeqw_a_or_avx x y))
-(decl x64_pcmpeqd (Xmm XmmMem) Xmm)
+(decl x64_pcmpeqd (Xmm XmmMem128) Xmm)
(rule (x64_pcmpeqd x y) (x64_pcmpeqd_a_or_avx x y))
-(decl x64_pcmpeqq (Xmm XmmMem) Xmm)
+(decl x64_pcmpeqq (Xmm XmmMem128) Xmm)
(rule (x64_pcmpeqq x y) (x64_pcmpeqq_a_or_avx x y))
-;; Helpers for creating `pcmpgt*` instructions.
-(decl x64_pcmpgt (Type Xmm XmmMem) Xmm)
+;; Helpers for creating `pcmpgt*` instructions. These are 128-bit packed
+;; compares regardless of lane width.
+(decl x64_pcmpgt (Type Xmm XmmMem128) Xmm)
(rule (x64_pcmpgt $I8X16 x y) (x64_pcmpgtb_a_or_avx x y))
(rule (x64_pcmpgt $I16X8 x y) (x64_pcmpgtw_a_or_avx x y))
(rule (x64_pcmpgt $I32X4 x y) (x64_pcmpgtd_a_or_avx x y))
@@ -3178,7 +3563,7 @@
;; 32-bits are equal then the lower 32-bits comparison is used instead.
(rule 0 (x64_pcmpgt $I64X2 x y)
(let (
- (mask Xmm (x64_movdqu_load (emit_u128_le_const 0x00000000_80000000_00000000_80000000)))
+ (mask Xmm (x64_movdqu_load (emit_xmm_mem_128_le_const 0x00000000_80000000_00000000_80000000)))
(x_masked Xmm (x64_pxor mask x))
(y_masked Xmm (x64_pxor mask y))
(cmp32 Xmm (x64_pcmpgtd_a x_masked y_masked))
@@ -3302,25 +3687,25 @@
(ConsumesFlags.ConsumesFlagsSideEffect (MInst.TrapIfOr cc1 cc2 tc)))
;; Helper for creating `movddup` instructions
-(decl x64_movddup (XmmMem) Xmm)
+(decl x64_movddup (XmmMem64) Xmm)
(rule (x64_movddup src) (x64_movddup_a src))
(rule 1 (x64_movddup src)
(if-let true (has_avx))
(x64_vmovddup_a src))
;; Helpers for creating `vpbroadcast*` instructions.
-(decl x64_vpbroadcastb (XmmMem) Xmm)
+(decl x64_vpbroadcastb (XmmMem8) Xmm)
(rule (x64_vpbroadcastb src) (x64_vpbroadcastb_a src))
-(decl x64_vpbroadcastw (XmmMem) Xmm)
+(decl x64_vpbroadcastw (XmmMem16) Xmm)
(rule (x64_vpbroadcastw src) (x64_vpbroadcastw_a src))
-(decl x64_vpbroadcastd (XmmMem) Xmm)
+(decl x64_vpbroadcastd (XmmMem32) Xmm)
(rule (x64_vpbroadcastd src) (x64_vpbroadcastd_a src))
-(decl x64_vbroadcastss (XmmMem) Xmm)
-(rule 1 (x64_vbroadcastss (is_xmm src)) (x64_vbroadcastss_a_r src))
-(rule 0 (x64_vbroadcastss (is_mem src)) (x64_vbroadcastss_a_m src))
+(decl x64_vbroadcastss (XmmMem32) Xmm)
+(rule 1 (x64_vbroadcastss (is_xmm_32 src)) (x64_vbroadcastss_a_r src))
+(rule 0 (x64_vbroadcastss (is_mem_32 src)) (x64_vbroadcastss_a_m src))
;;;; Jumps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3839,14 +4224,14 @@
;; Two registers are returned through `ValueRegs` where the first is the
;; quotient and the second is the remainder.
(decl x64_div (Type Gpr Gpr GprMem TrapCode) ValueRegs)
-(rule (x64_div $I16 lo hi divisor code) (x64_divw_m lo hi divisor code))
-(rule (x64_div $I32 lo hi divisor code) (x64_divl_m lo hi divisor code))
-(rule (x64_div $I64 lo hi divisor code) (x64_divq_m lo hi divisor code))
+(rule (x64_div ty @ $I16 lo hi divisor code) (x64_divw_m lo hi (gpr_mem_16_for_ty ty divisor) code))
+(rule (x64_div ty @ $I32 lo hi divisor code) (x64_divl_m lo hi (gpr_mem_32_for_ty ty divisor) code))
+(rule (x64_div ty @ $I64 lo hi divisor code) (x64_divq_m lo hi (gpr_mem_64_for_ty ty divisor) code))
(decl x64_idiv (Type Gpr Gpr GprMem TrapCode) ValueRegs)
-(rule (x64_idiv $I16 lo hi divisor code) (x64_idivw_m lo hi divisor code))
-(rule (x64_idiv $I32 lo hi divisor code) (x64_idivl_m lo hi divisor code))
-(rule (x64_idiv $I64 lo hi divisor code) (x64_idivq_m lo hi divisor code))
+(rule (x64_idiv ty @ $I16 lo hi divisor code) (x64_idivw_m lo hi (gpr_mem_16_for_ty ty divisor) code))
+(rule (x64_idiv ty @ $I32 lo hi divisor code) (x64_idivl_m lo hi (gpr_mem_32_for_ty ty divisor) code))
+(rule (x64_idiv ty @ $I64 lo hi divisor code) (x64_idivq_m lo hi (gpr_mem_64_for_ty ty divisor) code))
;;;; Pinned Register ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3950,6 +4335,39 @@
(convert WritableGpr GprMemImm writable_gpr_to_gpr_mem_imm)
(convert WritableGpr ValueRegs writable_gpr_to_value_regs)
+;; Sized GPR auto-conversions. `Value` -> `GprMem{N}` / `GprMemImm{N}`
+;; asserts that the value's CLIF width is N bits; `Gpr` -> sized is
+;; infallible (Gpr carries no width0.
+;;
+;; `RegMem{,Imm}` -> sized is intentionally not provided:
+;; those carry no width, so callers must use an explicit width cast.
+
+(convert Value GprMem8 put_in_gpr_mem_8)
+(convert Value GprMem16 put_in_gpr_mem_16)
+(convert Value GprMem32 put_in_gpr_mem_32)
+(convert Value GprMem64 put_in_gpr_mem_64)
+(convert Value GprMemImm8 put_in_gpr_mem_imm_8)
+(convert Value GprMemImm16 put_in_gpr_mem_imm_16)
+(convert Value GprMemImm32 put_in_gpr_mem_imm_32)
+(convert Value GprMemImm64 put_in_gpr_mem_imm_64)
+(convert Gpr GprMem8 gpr_to_gpr_mem_8)
+(convert Gpr GprMem16 gpr_to_gpr_mem_16)
+(convert Gpr GprMem32 gpr_to_gpr_mem_32)
+(convert Gpr GprMem64 gpr_to_gpr_mem_64)
+(convert Gpr GprMemImm8 gpr_to_gpr_mem_imm_8)
+(convert Gpr GprMemImm16 gpr_to_gpr_mem_imm_16)
+(convert Gpr GprMemImm32 gpr_to_gpr_mem_imm_32)
+(convert Gpr GprMemImm64 gpr_to_gpr_mem_imm_64)
+
+(convert GprMem8 GprMem gpr_mem_8_to_gpr_mem)
+(convert GprMem16 GprMem gpr_mem_16_to_gpr_mem)
+(convert GprMem32 GprMem gpr_mem_32_to_gpr_mem)
+(convert GprMem64 GprMem gpr_mem_64_to_gpr_mem)
+(convert GprMemImm8 GprMemImm gpr_mem_imm_8_to_gpr_mem_imm)
+(convert GprMemImm16 GprMemImm gpr_mem_imm_16_to_gpr_mem_imm)
+(convert GprMemImm32 GprMemImm gpr_mem_imm_32_to_gpr_mem_imm)
+(convert GprMemImm64 GprMemImm gpr_mem_imm_64_to_gpr_mem_imm)
+
(convert Xmm InstOutput output_xmm)
(convert Value Xmm put_in_xmm)
(convert Value XmmMem put_in_xmm_mem)
@@ -3983,6 +4401,92 @@
(convert XmmMem XmmMemAligned xmm_mem_to_xmm_mem_aligned)
(convert XmmMemImm XmmMemAlignedImm xmm_mem_imm_to_xmm_mem_aligned_imm)
+(rule (xmm_mem_to_xmm_mem_aligned_from_sized_8 x)
+ (xmm_mem_to_xmm_mem_aligned (xmm_mem_8_to_xmm_mem x)))
+(rule (xmm_mem_to_xmm_mem_aligned_from_sized_16 x)
+ (xmm_mem_to_xmm_mem_aligned (xmm_mem_16_to_xmm_mem x)))
+(rule (xmm_mem_to_xmm_mem_aligned_from_sized_32 x)
+ (xmm_mem_to_xmm_mem_aligned (xmm_mem_32_to_xmm_mem x)))
+(rule (xmm_mem_to_xmm_mem_aligned_from_sized_64 x)
+ (xmm_mem_to_xmm_mem_aligned (xmm_mem_64_to_xmm_mem x)))
+(rule (xmm_mem_to_xmm_mem_aligned_from_sized_128 x)
+ (xmm_mem_to_xmm_mem_aligned (xmm_mem_128_to_xmm_mem x)))
+
+(decl xmm_mem_to_xmm_mem_aligned_from_sized_8 (XmmMem8) XmmMemAligned)
+(decl xmm_mem_to_xmm_mem_aligned_from_sized_16 (XmmMem16) XmmMemAligned)
+(decl xmm_mem_to_xmm_mem_aligned_from_sized_32 (XmmMem32) XmmMemAligned)
+(decl xmm_mem_to_xmm_mem_aligned_from_sized_64 (XmmMem64) XmmMemAligned)
+(decl xmm_mem_to_xmm_mem_aligned_from_sized_128 (XmmMem128) XmmMemAligned)
+
+(convert XmmMem8 XmmMemAligned xmm_mem_to_xmm_mem_aligned_from_sized_8)
+(convert XmmMem16 XmmMemAligned xmm_mem_to_xmm_mem_aligned_from_sized_16)
+(convert XmmMem32 XmmMemAligned xmm_mem_to_xmm_mem_aligned_from_sized_32)
+(convert XmmMem64 XmmMemAligned xmm_mem_to_xmm_mem_aligned_from_sized_64)
+(convert XmmMem128 XmmMemAligned xmm_mem_to_xmm_mem_aligned_from_sized_128)
+
+(convert Value XmmMem8 put_in_xmm_mem_8)
+(convert Value XmmMem16 put_in_xmm_mem_16)
+(convert Value XmmMem32 put_in_xmm_mem_32)
+(convert Value XmmMem64 put_in_xmm_mem_64)
+(convert Value XmmMem128 put_in_xmm_mem_128)
+(convert Value XmmMemAligned8 put_in_xmm_mem_aligned_8)
+(convert Value XmmMemAligned16 put_in_xmm_mem_aligned_16)
+(convert Value XmmMemAligned32 put_in_xmm_mem_aligned_32)
+(convert Value XmmMemAligned64 put_in_xmm_mem_aligned_64)
+(convert Value XmmMemAligned128 put_in_xmm_mem_aligned_128)
+(convert Value XmmMemImm8 put_in_xmm_mem_imm_8)
+(convert Value XmmMemImm16 put_in_xmm_mem_imm_16)
+(convert Value XmmMemImm32 put_in_xmm_mem_imm_32)
+(convert Value XmmMemImm64 put_in_xmm_mem_imm_64)
+(convert Value XmmMemImm128 put_in_xmm_mem_imm_128)
+(convert Value XmmMemAlignedImm8 put_in_xmm_mem_aligned_imm_8)
+(convert Value XmmMemAlignedImm16 put_in_xmm_mem_aligned_imm_16)
+(convert Value XmmMemAlignedImm32 put_in_xmm_mem_aligned_imm_32)
+(convert Value XmmMemAlignedImm64 put_in_xmm_mem_aligned_imm_64)
+(convert Value XmmMemAlignedImm128 put_in_xmm_mem_aligned_imm_128)
+
+(convert Xmm XmmMem8 xmm_to_xmm_mem_8)
+(convert Xmm XmmMem16 xmm_to_xmm_mem_16)
+(convert Xmm XmmMem32 xmm_to_xmm_mem_32)
+(convert Xmm XmmMem64 xmm_to_xmm_mem_64)
+(convert Xmm XmmMem128 xmm_to_xmm_mem_128)
+(convert Xmm XmmMemAligned8 xmm_to_xmm_mem_aligned_8)
+(convert Xmm XmmMemAligned16 xmm_to_xmm_mem_aligned_16)
+(convert Xmm XmmMemAligned32 xmm_to_xmm_mem_aligned_32)
+(convert Xmm XmmMemAligned64 xmm_to_xmm_mem_aligned_64)
+(convert Xmm XmmMemAligned128 xmm_to_xmm_mem_aligned_128)
+(convert Xmm XmmMemImm8 xmm_to_xmm_mem_imm_8)
+(convert Xmm XmmMemImm16 xmm_to_xmm_mem_imm_16)
+(convert Xmm XmmMemImm32 xmm_to_xmm_mem_imm_32)
+(convert Xmm XmmMemImm64 xmm_to_xmm_mem_imm_64)
+(convert Xmm XmmMemImm128 xmm_to_xmm_mem_imm_128)
+(convert Xmm XmmMemAlignedImm8 xmm_to_xmm_mem_aligned_imm_8)
+(convert Xmm XmmMemAlignedImm16 xmm_to_xmm_mem_aligned_imm_16)
+(convert Xmm XmmMemAlignedImm32 xmm_to_xmm_mem_aligned_imm_32)
+(convert Xmm XmmMemAlignedImm64 xmm_to_xmm_mem_aligned_imm_64)
+(convert Xmm XmmMemAlignedImm128 xmm_to_xmm_mem_aligned_imm_128)
+
+(convert XmmMem8 XmmMem xmm_mem_8_to_xmm_mem)
+(convert XmmMem16 XmmMem xmm_mem_16_to_xmm_mem)
+(convert XmmMem32 XmmMem xmm_mem_32_to_xmm_mem)
+(convert XmmMem64 XmmMem xmm_mem_64_to_xmm_mem)
+(convert XmmMem128 XmmMem xmm_mem_128_to_xmm_mem)
+(convert XmmMemAligned8 XmmMemAligned xmm_mem_aligned_8_to_xmm_mem_aligned)
+(convert XmmMemAligned16 XmmMemAligned xmm_mem_aligned_16_to_xmm_mem_aligned)
+(convert XmmMemAligned32 XmmMemAligned xmm_mem_aligned_32_to_xmm_mem_aligned)
+(convert XmmMemAligned64 XmmMemAligned xmm_mem_aligned_64_to_xmm_mem_aligned)
+(convert XmmMemAligned128 XmmMemAligned xmm_mem_aligned_128_to_xmm_mem_aligned)
+(convert XmmMemImm8 XmmMemImm xmm_mem_imm_8_to_xmm_mem_imm)
+(convert XmmMemImm16 XmmMemImm xmm_mem_imm_16_to_xmm_mem_imm)
+(convert XmmMemImm32 XmmMemImm xmm_mem_imm_32_to_xmm_mem_imm)
+(convert XmmMemImm64 XmmMemImm xmm_mem_imm_64_to_xmm_mem_imm)
+(convert XmmMemImm128 XmmMemImm xmm_mem_imm_128_to_xmm_mem_imm)
+(convert XmmMemAlignedImm8 XmmMemAlignedImm xmm_mem_aligned_imm_8_to_xmm_mem_aligned_imm)
+(convert XmmMemAlignedImm16 XmmMemAlignedImm xmm_mem_aligned_imm_16_to_xmm_mem_aligned_imm)
+(convert XmmMemAlignedImm32 XmmMemAlignedImm xmm_mem_aligned_imm_32_to_xmm_mem_aligned_imm)
+(convert XmmMemAlignedImm64 XmmMemAlignedImm xmm_mem_aligned_imm_64_to_xmm_mem_aligned_imm)
+(convert XmmMemAlignedImm128 XmmMemAlignedImm xmm_mem_aligned_imm_128_to_xmm_mem_aligned_imm)
+
(convert Gpr Imm8Gpr gpr_to_imm8_gpr)
(convert Amode SyntheticAmode amode_to_synthetic_amode)
@@ -4003,8 +4507,116 @@
(convert SinkableLoad RegMemImm sink_load_to_reg_mem_imm)
(convert SinkableLoad GprMemImm sink_load_to_gpr_mem_imm)
(convert SinkableLoad XmmMem sink_load_to_xmm_mem)
+
+;; Width-tagged sinkable-load conversions. Each is guaranteed safe by the
+;; fact that the source `SinkableLoadN` newtype can only be obtained from
+;; an extractor that checked the Value's CLIF type width matches N.
+(convert SinkableLoad8 GprMem8 sink_load_8_to_gpr_mem_8)
+(convert SinkableLoad16 GprMem16 sink_load_16_to_gpr_mem_16)
+(convert SinkableLoad32 GprMem32 sink_load_32_to_gpr_mem_32)
+(convert SinkableLoad64 GprMem64 sink_load_64_to_gpr_mem_64)
+(convert SinkableLoad8 XmmMem8 sink_load_8_to_xmm_mem_8)
+(convert SinkableLoad16 XmmMem16 sink_load_16_to_xmm_mem_16)
+(convert SinkableLoad32 XmmMem32 sink_load_32_to_xmm_mem_32)
+(convert SinkableLoad64 XmmMem64 sink_load_64_to_xmm_mem_64)
+(convert SinkableLoad128 XmmMem128 sink_load_128_to_xmm_mem_128)
(convert SinkableLoad SyntheticAmode sink_load)
+;; Auto-conversions from `Amode` / `SyntheticAmode` to the sized
+;; `GprMem{N}` / `XmmMem{N}` / `XmmMemAligned{N}` operand types.
+;;
+;; Note that these are unchecked casts that allow a bare amode to
+;; become a GprMem of any width; but we've judged this to be fine,
+;; because when feeding a bare amode into a potentially
+;; memory-accessing instruction, one is already opting into accessing
+;; memory of the width that the instruction implies. The
+;; specific-width operand types enforce safety one level up, when we
+;; autoconvert a `Value` to a `GprMem{N}`.
+;;
+;; And in addition to that, these autoconversions are actually really
+;; quite necessary for ergonomics: without them, we would need to
+;; update a *bunch* of `to_amode`-invoking callsites to have explicit
+;; casts that would serve no real purpose since the caller is already
+;; typically type-specific and is doing low-level construction of
+;; memory accesses.
+(convert SyntheticAmode GprMem8 synthetic_amode_to_gpr_mem_8)
+(convert SyntheticAmode GprMem16 synthetic_amode_to_gpr_mem_16)
+(convert SyntheticAmode GprMem32 synthetic_amode_to_gpr_mem_32)
+(convert SyntheticAmode GprMem64 synthetic_amode_to_gpr_mem_64)
+(convert SyntheticAmode XmmMem8 synthetic_amode_to_xmm_mem_8)
+(convert SyntheticAmode XmmMem16 synthetic_amode_to_xmm_mem_16)
+(convert SyntheticAmode XmmMem32 synthetic_amode_to_xmm_mem_32)
+(convert SyntheticAmode XmmMem64 synthetic_amode_to_xmm_mem_64)
+(convert SyntheticAmode XmmMem128 synthetic_amode_to_xmm_mem_128)
+(convert SyntheticAmode XmmMemAligned128 synthetic_amode_to_xmm_mem_aligned_128)
+
+(convert Amode GprMem8 amode_to_gpr_mem_8)
+(convert Amode GprMem16 amode_to_gpr_mem_16)
+(convert Amode GprMem32 amode_to_gpr_mem_32)
+(convert Amode GprMem64 amode_to_gpr_mem_64)
+(convert Amode XmmMem8 amode_to_xmm_mem_8)
+(convert Amode XmmMem16 amode_to_xmm_mem_16)
+(convert Amode XmmMem32 amode_to_xmm_mem_32)
+(convert Amode XmmMem64 amode_to_xmm_mem_64)
+(convert Amode XmmMem128 amode_to_xmm_mem_128)
+(convert Amode XmmMemAligned128 amode_to_xmm_mem_aligned_128)
+
+
+(convert Reg GprMem8 reg_to_gpr_mem_8)
+(convert Reg GprMem16 reg_to_gpr_mem_16)
+(convert Reg GprMem32 reg_to_gpr_mem_32)
+(convert Reg GprMem64 reg_to_gpr_mem_64)
+
+(convert WritableGpr GprMem8 writable_gpr_to_gpr_mem_8)
+(convert WritableGpr GprMem16 writable_gpr_to_gpr_mem_16)
+(convert WritableGpr GprMem32 writable_gpr_to_gpr_mem_32)
+(convert WritableGpr GprMem64 writable_gpr_to_gpr_mem_64)
+(convert WritableXmm XmmMem128 writable_xmm_to_xmm_mem_128)
+(convert WritableXmm XmmMemAligned128 writable_xmm_to_xmm_mem_aligned_128)
+
+;; Sized `XmmMem{N} -> XmmMemAligned{N}` bridges: if the operand is a
+;; register or aligned memory, pass through; otherwise emit an unaligned
+;; 16-byte load into an XMM register. Only N=128 is actually used by the
+;; asm DSL (the `align(xmm_mN)` qualifier is only applied to xmm_m128),
+;; but we provide the other sizes for completeness.
+(convert XmmMem32 XmmMemAligned32 xmm_mem_32_to_xmm_mem_aligned_32)
+(convert XmmMem64 XmmMemAligned64 xmm_mem_64_to_xmm_mem_aligned_64)
+(convert XmmMem128 XmmMemAligned128 xmm_mem_128_to_xmm_mem_aligned_128)
+
+;; Explicit `Type`-parameterized narrowing helpers. Used where a
+;; `Type`-parameterized helper (e.g. `x64_add_raw`, `x64_div`,
+;; `cmove`) has an un-sized `GprMem` / `XmmMem` / `XmmMemAligned`
+;; operand in scope and needs to pass it to a now-sized asm helper.
+;;
+;; The `Type` argument is the helper's 's own `Type` parameter,
+;; to be cross-checked against the asserted size.
+(decl gpr_mem_8_for_ty (Type GprMem) GprMem8)
+(extern constructor gpr_mem_8_for_ty gpr_mem_8_for_ty)
+(decl gpr_mem_16_for_ty (Type GprMem) GprMem16)
+(extern constructor gpr_mem_16_for_ty gpr_mem_16_for_ty)
+(decl gpr_mem_32_for_ty (Type GprMem) GprMem32)
+(extern constructor gpr_mem_32_for_ty gpr_mem_32_for_ty)
+(decl gpr_mem_64_for_ty (Type GprMem) GprMem64)
+(extern constructor gpr_mem_64_for_ty gpr_mem_64_for_ty)
+
+;; Narrow a `GprMem` to `GprMem32` for use at a 32-bit asm helper that is
+;; being applied to a `Type` of 8, 16, or 32 bits. At `Type` width 32 any
+;; memory operand is safe (we read the exact source width). At narrower
+;; widths only a register source or a pool-emitted constant reference is
+;; permitted. Note that the sinkable-load infrastructure explicitly
+;; avoids sinking loads narrower than 32 bits; so this is an assert
+;; local to the use-site that verifies that that logic works.
+(decl gpr_mem_as_gpr_mem_32_or_widened_in_reg (Type GprMem) GprMem32)
+(extern constructor gpr_mem_as_gpr_mem_32_or_widened_in_reg gpr_mem_as_gpr_mem_32_or_widened_in_reg)
+(decl xmm_mem_32_for_ty (Type XmmMem) XmmMem32)
+(extern constructor xmm_mem_32_for_ty xmm_mem_32_for_ty)
+(decl xmm_mem_64_for_ty (Type XmmMem) XmmMem64)
+(extern constructor xmm_mem_64_for_ty xmm_mem_64_for_ty)
+(decl xmm_mem_128_for_ty (Type XmmMem) XmmMem128)
+(extern constructor xmm_mem_128_for_ty xmm_mem_128_for_ty)
+(decl xmm_mem_aligned_128_for_ty (Type XmmMemAligned) XmmMemAligned128)
+(extern constructor xmm_mem_aligned_128_for_ty xmm_mem_aligned_128_for_ty)
+
(decl reg_to_xmm_mem (Reg) XmmMem)
(rule (reg_to_xmm_mem r)
(xmm_to_xmm_mem (xmm_new r)))
@@ -4059,6 +4671,135 @@
(decl const_to_reg_mem (VCodeConstant) RegMem)
(rule (const_to_reg_mem c) (RegMem.Mem (const_to_synthetic_amode c)))
+;; Sized `VCodeConstant / SyntheticAmode / Amode -> XmmMem{N}`
+;; helpers. These are explicit (not auto-`convert`) because the
+;; source types don't carry width: using the wrong helper for the
+;; actual pool entry or load size is a bug, so the call site should
+;; specify the intended width. Use these at the point of emission of a
+;; known-width constant (e.g. paired with `emit_u128_le_const`) or
+;; when synthesizing a load from an `Amode` that's known to be 128
+;; bits wide.
+(decl const_to_xmm_mem_128 (VCodeConstant) XmmMem128)
+(extern constructor const_to_xmm_mem_128 const_to_xmm_mem_128)
+(decl synthetic_amode_to_xmm_mem_8 (SyntheticAmode) XmmMem8)
+(extern constructor synthetic_amode_to_xmm_mem_8 synthetic_amode_to_xmm_mem_8)
+(decl synthetic_amode_to_xmm_mem_16 (SyntheticAmode) XmmMem16)
+(extern constructor synthetic_amode_to_xmm_mem_16 synthetic_amode_to_xmm_mem_16)
+(decl synthetic_amode_to_xmm_mem_32 (SyntheticAmode) XmmMem32)
+(extern constructor synthetic_amode_to_xmm_mem_32 synthetic_amode_to_xmm_mem_32)
+(decl synthetic_amode_to_xmm_mem_64 (SyntheticAmode) XmmMem64)
+(extern constructor synthetic_amode_to_xmm_mem_64 synthetic_amode_to_xmm_mem_64)
+(decl synthetic_amode_to_xmm_mem_128 (SyntheticAmode) XmmMem128)
+(extern constructor synthetic_amode_to_xmm_mem_128 synthetic_amode_to_xmm_mem_128)
+;; Aligned 128-bit memory: go through the un-sized alignment check, which
+;; copies unaligned memory to a register via a 16-byte `movdqu` load.
+(decl synthetic_amode_to_xmm_mem_aligned_128 (SyntheticAmode) XmmMemAligned128)
+(rule (synthetic_amode_to_xmm_mem_aligned_128 amode)
+ (xmm_mem_128_to_xmm_mem_aligned_128 (synthetic_amode_to_xmm_mem_128 amode)))
+
+;; Amode -> sized bridges. Each routes through the `SyntheticAmode.Real`
+;; wrapping and the matching sized conversion above.
+(decl amode_to_gpr_mem_8 (Amode) GprMem8)
+(rule (amode_to_gpr_mem_8 a) (synthetic_amode_to_gpr_mem_8 (amode_to_synthetic_amode a)))
+(decl amode_to_gpr_mem_16 (Amode) GprMem16)
+(rule (amode_to_gpr_mem_16 a) (synthetic_amode_to_gpr_mem_16 (amode_to_synthetic_amode a)))
+(decl amode_to_gpr_mem_32 (Amode) GprMem32)
+(rule (amode_to_gpr_mem_32 a) (synthetic_amode_to_gpr_mem_32 (amode_to_synthetic_amode a)))
+(decl amode_to_gpr_mem_64 (Amode) GprMem64)
+(rule (amode_to_gpr_mem_64 a) (synthetic_amode_to_gpr_mem_64 (amode_to_synthetic_amode a)))
+(decl amode_to_xmm_mem_8 (Amode) XmmMem8)
+(rule (amode_to_xmm_mem_8 a) (synthetic_amode_to_xmm_mem_8 (amode_to_synthetic_amode a)))
+(decl amode_to_xmm_mem_16 (Amode) XmmMem16)
+(rule (amode_to_xmm_mem_16 a) (synthetic_amode_to_xmm_mem_16 (amode_to_synthetic_amode a)))
+(decl amode_to_xmm_mem_32 (Amode) XmmMem32)
+(rule (amode_to_xmm_mem_32 a) (synthetic_amode_to_xmm_mem_32 (amode_to_synthetic_amode a)))
+(decl amode_to_xmm_mem_64 (Amode) XmmMem64)
+(rule (amode_to_xmm_mem_64 a) (synthetic_amode_to_xmm_mem_64 (amode_to_synthetic_amode a)))
+;; (amode_to_xmm_mem_128 already exists below; keep a single source of
+;; truth by routing through the SyntheticAmode variant.)
+(decl amode_to_xmm_mem_aligned_128 (Amode) XmmMemAligned128)
+(rule (amode_to_xmm_mem_aligned_128 a)
+ (synthetic_amode_to_xmm_mem_aligned_128 (amode_to_synthetic_amode a)))
+(decl synthetic_amode_to_gpr_mem_8 (SyntheticAmode) GprMem8)
+(extern constructor synthetic_amode_to_gpr_mem_8 synthetic_amode_to_gpr_mem_8)
+(decl synthetic_amode_to_gpr_mem_16 (SyntheticAmode) GprMem16)
+(extern constructor synthetic_amode_to_gpr_mem_16 synthetic_amode_to_gpr_mem_16)
+(decl synthetic_amode_to_gpr_mem_32 (SyntheticAmode) GprMem32)
+(extern constructor synthetic_amode_to_gpr_mem_32 synthetic_amode_to_gpr_mem_32)
+(decl synthetic_amode_to_gpr_mem_64 (SyntheticAmode) GprMem64)
+(extern constructor synthetic_amode_to_gpr_mem_64 synthetic_amode_to_gpr_mem_64)
+
+(decl reg_to_gpr_mem_8 (Reg) GprMem8)
+(extern constructor reg_to_gpr_mem_8 reg_to_gpr_mem_8)
+(decl reg_to_gpr_mem_16 (Reg) GprMem16)
+(extern constructor reg_to_gpr_mem_16 reg_to_gpr_mem_16)
+(decl reg_to_gpr_mem_32 (Reg) GprMem32)
+(extern constructor reg_to_gpr_mem_32 reg_to_gpr_mem_32)
+(decl reg_to_gpr_mem_64 (Reg) GprMem64)
+(extern constructor reg_to_gpr_mem_64 reg_to_gpr_mem_64)
+
+(decl writable_gpr_to_gpr_mem_8 (WritableGpr) GprMem8)
+(extern constructor writable_gpr_to_gpr_mem_8 writable_gpr_to_gpr_mem_8)
+(decl writable_gpr_to_gpr_mem_16 (WritableGpr) GprMem16)
+(extern constructor writable_gpr_to_gpr_mem_16 writable_gpr_to_gpr_mem_16)
+(decl writable_gpr_to_gpr_mem_32 (WritableGpr) GprMem32)
+(extern constructor writable_gpr_to_gpr_mem_32 writable_gpr_to_gpr_mem_32)
+(decl writable_gpr_to_gpr_mem_64 (WritableGpr) GprMem64)
+(extern constructor writable_gpr_to_gpr_mem_64 writable_gpr_to_gpr_mem_64)
+(decl writable_xmm_to_xmm_mem_128 (WritableXmm) XmmMem128)
+(extern constructor writable_xmm_to_xmm_mem_128 writable_xmm_to_xmm_mem_128)
+(decl writable_xmm_to_xmm_mem_aligned_128 (WritableXmm) XmmMemAligned128)
+(extern constructor writable_xmm_to_xmm_mem_aligned_128 writable_xmm_to_xmm_mem_aligned_128)
+
+(decl xmm_mem_32_to_xmm_mem_aligned_32 (XmmMem32) XmmMemAligned32)
+(extern constructor xmm_mem_32_to_xmm_mem_aligned_32 xmm_mem_32_to_xmm_mem_aligned_32)
+(decl xmm_mem_64_to_xmm_mem_aligned_64 (XmmMem64) XmmMemAligned64)
+(extern constructor xmm_mem_64_to_xmm_mem_aligned_64 xmm_mem_64_to_xmm_mem_aligned_64)
+(decl xmm_mem_128_to_xmm_mem_aligned_128 (XmmMem128) XmmMemAligned128)
+(extern constructor xmm_mem_128_to_xmm_mem_aligned_128 xmm_mem_128_to_xmm_mem_aligned_128)
+
+;; Unchecked narrowing from un-sized memory-operand types to their sized
+;; counterparts. Use these at callsites where the helper's own name
+;; pins the width (so no `Type` is in scope to double-check against) --
+;; e.g., `x64_movzx` / `x64_movsx` dispatch on an `ExtMode` literal,
+;; `x64_movd_to_xmm` is intrinsically 32-bit, vector shifts like
+;; `x64_pslld` are intrinsically 128-bit. For `Type`-parameterized
+;; dispatchers, prefer the `..._for_ty` variants above, which also
+;; assert consistency between the dispatcher's `Type` and the target
+;; width.
+(decl gpr_mem_as_gpr_mem_8 (GprMem) GprMem8)
+(extern constructor gpr_mem_as_gpr_mem_8 gpr_mem_as_gpr_mem_8)
+(decl gpr_mem_as_gpr_mem_16 (GprMem) GprMem16)
+(extern constructor gpr_mem_as_gpr_mem_16 gpr_mem_as_gpr_mem_16)
+(decl gpr_mem_as_gpr_mem_32 (GprMem) GprMem32)
+(extern constructor gpr_mem_as_gpr_mem_32 gpr_mem_as_gpr_mem_32)
+(decl gpr_mem_as_gpr_mem_64 (GprMem) GprMem64)
+(extern constructor gpr_mem_as_gpr_mem_64 gpr_mem_as_gpr_mem_64)
+
+(decl xmm_mem_as_xmm_mem_32 (XmmMem) XmmMem32)
+(extern constructor xmm_mem_as_xmm_mem_32 xmm_mem_as_xmm_mem_32)
+(decl xmm_mem_as_xmm_mem_64 (XmmMem) XmmMem64)
+(extern constructor xmm_mem_as_xmm_mem_64 xmm_mem_as_xmm_mem_64)
+(decl xmm_mem_as_xmm_mem_128 (XmmMem) XmmMem128)
+(extern constructor xmm_mem_as_xmm_mem_128 xmm_mem_as_xmm_mem_128)
+
+(decl xmm_mem_aligned_as_xmm_mem_aligned_32 (XmmMemAligned) XmmMemAligned32)
+(extern constructor xmm_mem_aligned_as_xmm_mem_aligned_32 xmm_mem_aligned_as_xmm_mem_aligned_32)
+(decl xmm_mem_aligned_as_xmm_mem_aligned_64 (XmmMemAligned) XmmMemAligned64)
+(extern constructor xmm_mem_aligned_as_xmm_mem_aligned_64 xmm_mem_aligned_as_xmm_mem_aligned_64)
+(decl xmm_mem_aligned_as_xmm_mem_aligned_128 (XmmMemAligned) XmmMemAligned128)
+(extern constructor xmm_mem_aligned_as_xmm_mem_aligned_128 xmm_mem_aligned_as_xmm_mem_aligned_128)
+(decl amode_to_xmm_mem_128 (Amode) XmmMem128)
+(rule (amode_to_xmm_mem_128 amode)
+ (synthetic_amode_to_xmm_mem_128 (amode_to_synthetic_amode amode)))
+
+;; Emit a 128-bit little-endian constant into the pool and return it as a
+;; sized `XmmMem128` operand -- the normal path for 128-bit masks built
+;; in-line at the callsite.
+(decl emit_xmm_mem_128_le_const (u128) XmmMem128)
+(rule (emit_xmm_mem_128_le_const c)
+ (const_to_xmm_mem_128 (emit_u128_le_const c)))
+
(decl xmm_to_xmm_mem_aligned (Xmm) XmmMemAligned)
(rule (xmm_to_xmm_mem_aligned reg) (xmm_mem_to_xmm_mem_aligned reg))
(decl amode_to_xmm_mem_aligned (Amode) XmmMemAligned)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 7e3b7394b9..6ed0608de4 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -24,13 +24,25 @@ pub trait FromWritableReg: Sized {
/// A macro for defining a newtype of `Reg` that enforces some invariant about
/// the wrapped `Reg` (such as that it is of a particular register class).
+///
+/// Each `reg_mem` / `reg_mem_imm` entry can optionally carry:
+/// * `aligned:BOOL` -- require aligned memory operands (used for SSE
+/// non-VEX encodings).
+/// * `size:BITS` -- the bit width of the value the operand holds. This
+/// is purely a type-level distinction (a sized newtype is a different
+/// Rust type than an unsized one of the same "family"), used to prevent
+/// cross-width operand mix-ups at compile time.
macro_rules! newtype_of_reg {
(
$newtype_reg:ident,
$newtype_writable_reg:ident,
$newtype_option_writable_reg:ident,
- reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
- reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
+ reg_mem: ($($newtype_reg_mem:ident
+ $(size:$size:literal)?
+ $(aligned:$aligned:ident)?),*),
+ reg_mem_imm: ($($newtype_reg_mem_imm:ident
+ $(size:$size_imm:literal)?
+ $(aligned:$aligned_imm:ident)?),*),
|$check_reg:ident| $check:expr
) => {
/// A newtype wrapper around `Reg`.
@@ -146,6 +158,13 @@ macro_rules! newtype_of_reg {
}
impl $newtype_reg_mem {
+ $(
+ /// The bit width of the value this operand holds. This is
+ /// a type-level distinction only; the underlying `RegMem`
+ /// does not record the width.
+ pub const SIZE_BITS: u32 = $size;
+ )?
+
/// Construct a `RegMem` newtype from the given `RegMem`, or return
/// `None` if the `RegMem` is not a valid instance of this `RegMem`
/// newtype.
@@ -227,6 +246,13 @@ macro_rules! newtype_of_reg {
}
impl $newtype_reg_mem_imm {
+ $(
+ /// The bit width of the value this operand holds. This is
+ /// a type-level distinction only; the underlying
+ /// `RegMemImm` does not record the width.
+ pub const SIZE_BITS: u32 = $size_imm;
+ )?
+
/// Construct this newtype from the given `RegMemImm`, or return
/// `None` if the `RegMemImm` is not a valid instance of this
/// newtype.
@@ -294,12 +320,24 @@ macro_rules! newtype_of_reg {
}
// Define a newtype of `Reg` for general-purpose registers.
+//
+// The sized `GprMemN` / `GprMemImmN` variants indicate the width of
+// the register or memory operand that an instruction operates on; for
+// a memory operand, the width of the load or store in particular.
newtype_of_reg!(
Gpr,
WritableGpr,
OptionWritableGpr,
- reg_mem: (GprMem),
- reg_mem_imm: (GprMemImm),
+ reg_mem: (GprMem,
+ GprMem8 size:8,
+ GprMem16 size:16,
+ GprMem32 size:32,
+ GprMem64 size:64),
+ reg_mem_imm: (GprMemImm,
+ GprMemImm8 size:8,
+ GprMemImm16 size:16,
+ GprMemImm32 size:32,
+ GprMemImm64 size:64),
|reg| reg.class() == RegClass::Int
);
@@ -324,12 +362,38 @@ impl Gpr {
}
// Define a newtype of `Reg` for XMM registers.
+//
+// Specific-width newtypes are, as for the GPR case above, used to
+// distinguish the width of the operation particularly when a memory
+// load or store occurs.
newtype_of_reg!(
Xmm,
WritableXmm,
OptionWritableXmm,
- reg_mem: (XmmMem, XmmMemAligned aligned:true),
- reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
+ reg_mem: (XmmMem,
+ XmmMem8 size:8,
+ XmmMem16 size:16,
+ XmmMem32 size:32,
+ XmmMem64 size:64,
+ XmmMem128 size:128,
+ XmmMemAligned aligned:true,
+ XmmMemAligned8 size:8 aligned:true,
+ XmmMemAligned16 size:16 aligned:true,
+ XmmMemAligned32 size:32 aligned:true,
+ XmmMemAligned64 size:64 aligned:true,
+ XmmMemAligned128 size:128 aligned:true),
+ reg_mem_imm: (XmmMemImm,
+ XmmMemImm8 size:8,
+ XmmMemImm16 size:16,
+ XmmMemImm32 size:32,
+ XmmMemImm64 size:64,
+ XmmMemImm128 size:128,
+ XmmMemAlignedImm aligned:true,
+ XmmMemAlignedImm8 size:8 aligned:true,
+ XmmMemAlignedImm16 size:16 aligned:true,
+ XmmMemAlignedImm32 size:32 aligned:true,
+ XmmMemAlignedImm64 size:64 aligned:true,
+ XmmMemAlignedImm128 size:128 aligned:true),
|reg| reg.class() == RegClass::Float
);
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 456efc8e9f..1a9d4b5692 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -110,7 +110,7 @@
(rule 3 (lower (has_type $I128 (iadd _ x (uextend _ y @ (value_type $I64)))))
(let ((x_regs ValueRegs x))
(iadd128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)
- y (RegMemImm.Imm 0))))
+ y (gpr_mem_imm_64_imm 0))))
;; Specialized lowering rule for `iadd` of two 64-bit unsigned integers, meaning
;; that we can skip the `adc` and instead use `setb`. This is in some sense a
@@ -132,7 +132,7 @@
;; Helper for lowering 128-bit addition with the 64-bit halves of the lhs/rhs
;; already split. The first two arguments are lo/hi for the lhs and the second
;; two are lo/hi for the rhs.
-(decl iadd128 (Gpr Gpr GprMemImm GprMemImm) ValueRegs)
+(decl iadd128 (Gpr Gpr GprMemImm64 GprMemImm64) ValueRegs)
(rule (iadd128 x_lo x_hi y_lo y_hi)
(with_flags (x64_add_with_flags_paired $I64 x_lo y_lo)
(x64_adc_paired $I64 x_hi y_hi)))
@@ -283,12 +283,12 @@
(rule 3 (lower (has_type $I128 (isub _ x (uextend _ y @ (value_type $I64)))))
(let ((x_regs ValueRegs x))
(isub128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)
- y (RegMemImm.Imm 0))))
+ y (gpr_mem_imm_64_imm 0))))
;; Helper for lowering 128-bit subtraction with the 64-bit halves of the lhs/rhs
;; already split. The first two arguments are lo/hi for the lhs and the second
;; two are lo/hi for the rhs.
-(decl isub128 (Gpr Gpr GprMemImm GprMemImm) ValueRegs)
+(decl isub128 (Gpr Gpr GprMemImm64 GprMemImm64) ValueRegs)
(rule (isub128 x_lo x_hi y_lo y_hi)
(with_flags (x64_sub_with_flags_paired $I64 x_lo y_lo)
(x64_sbb_paired $I64 x_hi y_hi)))
@@ -343,7 +343,7 @@
;; SSE.
-(decl sse_and (Type Xmm XmmMem) Xmm)
+(decl sse_and (Type Xmm XmmMem128) Xmm)
(rule (sse_and $F32X4 x y) (x64_andps x y))
(rule (sse_and $F64X2 x y) (x64_andpd x y))
(rule (sse_and $F32 x y) (x64_andps x y))
@@ -374,7 +374,7 @@
;; by Cranelift's `band_not` instruction that is legalized into the simpler
;; forms early on.
-(decl sse_and_not (Type Xmm XmmMem) Xmm)
+(decl sse_and_not (Type Xmm XmmMem128) Xmm)
(rule (sse_and_not $F32X4 x y) (x64_andnps x y))
(rule (sse_and_not $F64X2 x y) (x64_andnpd x y))
(rule -1 (sse_and_not (multi_lane _bits _lanes) x y) (x64_pandn x y))
@@ -470,7 +470,8 @@
;; SSE.
-(decl sse_or (Type Xmm XmmMem) Xmm)
+;; Bitwise SSE OR. Same 128-bit-physical-width story as `sse_and`.
+(decl sse_or (Type Xmm XmmMem128) Xmm)
(rule (sse_or $F32X4 x y) (x64_orps x y))
(rule (sse_or $F64X2 x y) (x64_orpd x y))
(rule (sse_or $F32 x y) (x64_orps x y))
@@ -632,8 +633,8 @@
;; the mask below.
(unmasked Xmm (x64_psllw src (mov_rmi_to_xmm masked_amt)))
(mask_addr SyntheticAmode (ishl_i8x16_mask masked_amt))
- (mask Reg (x64_movdqu_load mask_addr)))
- (x64_pand unmasked (RegMem.Reg mask))))
+ (mask Xmm (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 mask_addr))))
+ (x64_pand unmasked mask)))
;; Get the address of the mask to use when fixing up the lanes that weren't
;; correctly generated by the 16x8 shift.
@@ -737,7 +738,7 @@
;; correct for half of the lanes; the others must be fixed up with
;; the mask below.
(unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt))))
- (x64_pand unmasked (ushr_i8x16_mask masked_amt))))
+ (x64_pand unmasked (synthetic_amode_to_xmm_mem_128 (ushr_i8x16_mask masked_amt)))))
;; Get the address of the mask to use when fixing up the lanes that weren't
;; correctly generated by the 16x8 shift.
@@ -1099,7 +1100,7 @@
;; dst_lo:hi_lolo = mulhi_u x_lo, y_lo
;; dst_hi = add hilo_hilo, hi_lolo
;; return (dst_lo, dst_hi)
-(decl imul128 (Gpr Gpr GprMem GprMem) ValueRegs)
+(decl imul128 (Gpr Gpr GprMem64 GprMem64) ValueRegs)
(rule (imul128 x_lo x_hi y_lo y_hi)
;; Put `x` into registers and unpack its hi/lo halves.
(let (
@@ -1400,10 +1401,10 @@
;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type $F32 (fabs _ x)))
- (x64_andps x (imm $F32 0x7fffffff)))
+ (x64_andps x (xmm_new (imm $F32 0x7fffffff))))
(rule (lower (has_type $F64 (fabs _ x)))
- (x64_andpd x (imm $F64 0x7fffffffffffffff)))
+ (x64_andpd x (xmm_new (imm $F64 0x7fffffffffffffff))))
;; Special case for `f32x4.abs`.
(rule (lower (has_type $F32X4 (fabs _ x)))
@@ -1418,10 +1419,10 @@
;;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type $F32 (fneg _ x)))
- (x64_xorps x (imm $F32 0x80000000)))
+ (x64_xorps x (xmm_new (imm $F32 0x80000000))))
(rule (lower (has_type $F64 (fneg _ x)))
- (x64_xorpd x (imm $F64 0x8000000000000000)))
+ (x64_xorpd x (xmm_new (imm $F64 0x8000000000000000))))
(rule (lower (has_type $F32X4 (fneg _ x)))
(x64_xorps x
@@ -1569,9 +1570,16 @@
(x64_or ty a b)))
(rule 5 (lower (has_type (ty_scalar_float ty) (bitselect _ c t f)))
- (let ((a Xmm (sse_and ty c t))
- (c_neg Xmm (x64_xor_vector ty c (vector_all_ones)))
- (b Xmm (sse_and ty c_neg f)))
+ ;; The SSE bitwise ops act on the full 128 bits of an XMM register, so
+ ;; force each scalar float operand into an XMM register first -- we
+ ;; must not fold a 4/8-byte load into an operand that the instruction
+ ;; will read 16 bytes from.
+ (let ((c_xmm Xmm (put_in_xmm c))
+ (t_xmm Xmm (put_in_xmm t))
+ (f_xmm Xmm (put_in_xmm f))
+ (a Xmm (sse_and ty c_xmm t_xmm))
+ (c_neg Xmm (x64_xor_vector ty c_xmm (vector_all_ones)))
+ (b Xmm (sse_and ty c_neg f_xmm)))
(sse_or ty a b)))
;;;; Rules for `blendv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1625,7 +1633,7 @@
;; `por`'d with the original vec-with-a-hole to produce the final result of the
;; insertion.
(rule (lower (insertlane _ vec @ (value_type $I8X16) val (u8_from_uimm8 n)))
- (let ((vec_with_hole Xmm (x64_pand vec (insert_i8x16_lane_hole n)))
+ (let ((vec_with_hole Xmm (x64_pand vec (const_to_xmm_mem_128 (insert_i8x16_lane_hole n))))
(val Gpr (x64_movzx (ExtMode.BL) val))
(val Gpr (x64_shll_mi val (u8_wrapping_shl (u8_and n 3) 3)))
(val Xmm (x64_movd_to_xmm val))
@@ -1694,7 +1702,7 @@
(if-let true (has_sse41))
(x64_pinsrq (bitcast_gpr_to_xmm 64 lane0) lane1 1))
-(rule 1 (lower (insertlane _ vec @ (value_type $F32X4) (sinkable_load val) (u8_from_uimm8 idx)))
+(rule 1 (lower (insertlane _ vec @ (value_type $F32X4) (sinkable_load_32 val) (u8_from_uimm8 idx)))
(if-let true (has_sse41))
(x64_insertps vec val (sse_insertps_lane_imm idx)))
(rule (lower (insertlane _ vec @ (value_type $F32X4) val (u8_from_uimm8 idx)))
@@ -1868,11 +1876,11 @@
(decl flip_high_bit_mask (Type) Xmm)
(rule (flip_high_bit_mask $I16X8)
- (x64_movdqu_load (emit_u128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000)))
+ (x64_movdqu_load (emit_xmm_mem_128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000)))
(rule (flip_high_bit_mask $I32X4)
- (x64_movdqu_load (emit_u128_le_const 0x80000000_80000000_80000000_80000000)))
+ (x64_movdqu_load (emit_xmm_mem_128_le_const 0x80000000_80000000_80000000_80000000)))
(rule (flip_high_bit_mask $I64X2)
- (x64_movdqu_load (emit_u128_le_const 0x8000000000000000_8000000000000000)))
+ (x64_movdqu_load (emit_xmm_mem_128_le_const 0x8000000000000000_8000000000000000)))
;; SSE `umin`.
@@ -2497,7 +2505,7 @@
(rule 1 (lower (has_type $I8X16 (popcnt _ src)))
(if-let true (has_ssse3))
- (let ((low_mask XmmMem (emit_u128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))
+ (let ((low_mask XmmMem128 (emit_xmm_mem_128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))
(low_nibbles Xmm (x64_pand src low_mask))
;; Note that this is a 16x8 shift, but that's OK; we mask
;; off anything that traverses from one byte to the next
@@ -2512,7 +2520,7 @@
;; A modified version of the popcnt method from Hacker's Delight.
(rule (lower (has_type $I8X16 (popcnt _ src)))
- (let ((mask1 XmmMem (emit_u128_le_const 0x77777777777777777777777777777777))
+ (let ((mask1 XmmMem128 (emit_xmm_mem_128_le_const 0x77777777777777777777777777777777))
(src Xmm src)
(shifted Xmm (x64_pand (x64_psrlq src (xmi_imm 1)) mask1))
(src Xmm (x64_psubb src shifted))
@@ -2521,7 +2529,7 @@
(shifted Xmm (x64_pand (x64_psrlq shifted (xmi_imm 1)) mask1))
(src Xmm (x64_psubb src shifted))
(src Xmm (x64_paddb src (x64_psrlw src (xmi_imm 4)))))
- (x64_pand src (emit_u128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))))
+ (x64_pand src (emit_xmm_mem_128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))))
;; Rules for `bitrev` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2689,13 +2697,13 @@
;; The above rules automatically sink loads for rhs operands, so additionally
;; add rules for sinking loads with lhs operands.
-(rule 1 (lower (has_type $F32 (fadd _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F32 (fadd _ (sinkable_load_32 x) y)))
(x64_addss y x))
-(rule 1 (lower (has_type $F64 (fadd _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F64 (fadd _ (sinkable_load_64 x) y)))
(x64_addsd y x))
-(rule 1 (lower (has_type $F32X4 (fadd _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F32X4 (fadd _ (sinkable_load_128 x) y)))
(x64_addps y x))
-(rule 1 (lower (has_type $F64X2 (fadd _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F64X2 (fadd _ (sinkable_load_128 x) y)))
(x64_addpd y x))
;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2722,13 +2730,13 @@
;; The above rules automatically sink loads for rhs operands, so additionally
;; add rules for sinking loads with lhs operands.
-(rule 1 (lower (has_type $F32 (fmul _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F32 (fmul _ (sinkable_load_32 x) y)))
(x64_mulss y x))
-(rule 1 (lower (has_type $F64 (fmul _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F64 (fmul _ (sinkable_load_64 x) y)))
(x64_mulsd y x))
-(rule 1 (lower (has_type $F32X4 (fmul _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F32X4 (fmul _ (sinkable_load_128 x) y)))
(x64_mulps y x))
-(rule 1 (lower (has_type $F64X2 (fmul _ (sinkable_load x) y)))
+(rule 1 (lower (has_type $F64X2 (fmul _ (sinkable_load_128 x) y)))
(x64_mulpd y x))
;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3008,9 +3016,11 @@
;; Allow sinking loads with one of the two values being multiplied in addition
;; to the value being added. Note that both x and y can be sunk here due to
-;; multiplication being commutative.
-(rule 1 (fmadd ty (sinkable_load x) y z) (x64_vfmadd132 ty y z x))
-(rule 2 (fmadd ty x (sinkable_load y) z) (x64_vfmadd132 ty x z y))
+;; multiplication being commutative. We bind `x`/`y` to the CLIF `Value`
+;; (via `@`) and use `sinkable_load` only as a predicate, so `vfmadd132`'s
+;; `Value -> XmmMem{N}` auto-conversion performs the actual sink.
+(rule 1 (fmadd ty x @ (sinkable_load _) y z) (x64_vfmadd132 ty y z x))
+(rule 2 (fmadd ty x y @ (sinkable_load _) z) (x64_vfmadd132 ty x z y))
;; If one of the values being multiplied is negated then use a `vfnmadd*`
;; instruction instead
@@ -3018,8 +3028,8 @@
(rule 4 (fmadd ty x (fneg _ y) z) (fnmadd ty x y z))
(rule (fnmadd ty x y z) (x64_vfnmadd213 ty x y z))
-(rule 1 (fnmadd ty (sinkable_load x) y z) (x64_vfnmadd132 ty y z x))
-(rule 2 (fnmadd ty x (sinkable_load y) z) (x64_vfnmadd132 ty x z y))
+(rule 1 (fnmadd ty x @ (sinkable_load _) y z) (x64_vfnmadd132 ty y z x))
+(rule 2 (fnmadd ty x y @ (sinkable_load _) z) (x64_vfnmadd132 ty x z y))
(rule 2 (lower (has_type ty (fma _ x y (fneg _ z))))
@@ -3036,8 +3046,8 @@
;; Allow sinking loads with one of the two values being multiplied in addition
;; to the value being subtracted. Note that both x and y can be sunk here due to
;; multiplication being commutative.
-(rule 1 (fmsub ty (sinkable_load x) y z) (x64_vfmsub132 ty y z x))
-(rule 2 (fmsub ty x (sinkable_load y) z) (x64_vfmsub132 ty x z y))
+(rule 1 (fmsub ty x @ (sinkable_load _) y z) (x64_vfmsub132 ty y z x))
+(rule 2 (fmsub ty x y @ (sinkable_load _) z) (x64_vfmsub132 ty x z y))
;; If one of the values being multiplied is negated then use a `vfnmsub*`
;; instruction instead
@@ -3045,8 +3055,8 @@
(rule 4 (fmsub ty x (fneg _ y) z) (fnmsub ty x y z))
(rule (fnmsub ty x y z) (x64_vfnmsub213 ty x y z))
-(rule 1 (fnmsub ty (sinkable_load x) y z) (x64_vfnmsub132 ty y z x))
-(rule 2 (fnmsub ty x (sinkable_load y) z) (x64_vfnmsub132 ty x z y))
+(rule 1 (fnmsub ty x @ (sinkable_load _) y z) (x64_vfnmsub132 ty y z x))
+(rule 2 (fnmsub ty x y @ (sinkable_load _) z) (x64_vfnmsub132 ty x z y))
;; Rules for `load*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3095,7 +3105,7 @@
(rule 1 (lower (has_type $F64X2 (load _ (little_or_native_endian flags) address offset)))
(x64_movupd_load (to_amode flags address offset)))
(rule 0 (lower (has_type (is_xmm_type (ty_128 _)) (load _ (little_or_native_endian flags) address offset)))
- (x64_movdqu_load (to_amode flags address offset)))
+ (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 (to_amode flags address offset))))
;; We can load an I128 by doing two 64-bit loads.
(rule -3 (lower (has_type $I128
@@ -3110,22 +3120,22 @@
;; to the next wider width (e.g., 16x4 -> 32x4).
(rule 1 (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovsxbw (to_amode flags address offset)))
+ (x64_pmovsxbw (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule 1 (lower (has_type $I16X8 (uload8x8 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovzxbw (to_amode flags address offset)))
+ (x64_pmovzxbw (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule 1 (lower (has_type $I32X4 (sload16x4 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovsxwd (to_amode flags address offset)))
+ (x64_pmovsxwd (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule 1 (lower (has_type $I32X4 (uload16x4 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovzxwd (to_amode flags address offset)))
+ (x64_pmovzxwd (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule 1 (lower (has_type $I64X2 (sload32x2 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovsxdq (to_amode flags address offset)))
+ (x64_pmovsxdq (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule 1 (lower (has_type $I64X2 (uload32x2 _ (little_or_native_endian flags) address offset)))
(if-let true (has_sse41))
- (x64_pmovzxdq (to_amode flags address offset)))
+ (x64_pmovzxdq (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))
(rule (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))
(lower_swiden_low $I16X8 (x64_movq_to_xmm (to_amode flags address offset))))
@@ -3723,7 +3733,10 @@
(x64_unpcklpd f0 f1)))
(rule 1 (lower (has_type $F64X2 (fcvt_from_sint _ (swiden_low _ a @ (value_type $I32X4)))))
- (x64_cvtdq2pd a))
+ ;; `cvtdq2pd` only reads the low 8 bytes of its source operand; force
+ ;; `a` into an XMM register so we don't fold a 128-bit load into an
+ ;; instruction that only wants 64 bits.
+ (x64_cvtdq2pd (put_in_xmm a)))
;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3765,10 +3778,10 @@
;; leading `1.` at the start of the float that we constructed, one for the low
;; half and one for the upper half.
(rule -1 (lower (has_type $F64X2 (fcvt_from_uint _ val @ (value_type $I64X2))))
- (let ((low32_mask XmmMem (emit_u128_le_const 0x00000000ffffffff_00000000ffffffff))
- (float_1p52 XmmMem (emit_u128_le_const 0x4330000000000000_4330000000000000))
- (float_1p84 XmmMem (emit_u128_le_const 0x4530000000000000_4530000000000000))
- (float_1p84_plus_1p52 XmmMem (emit_u128_le_const 0x4530000000100000_4530000000100000))
+ (let ((low32_mask XmmMem128 (emit_xmm_mem_128_le_const 0x00000000ffffffff_00000000ffffffff))
+ (float_1p52 XmmMem128 (emit_xmm_mem_128_le_const 0x4330000000000000_4330000000000000))
+ (float_1p84 XmmMem128 (emit_xmm_mem_128_le_const 0x4530000000000000_4530000000000000))
+ (float_1p84_plus_1p52 XmmMem128 (emit_xmm_mem_128_le_const 0x4530000000100000_4530000000100000))
(low32 Xmm (x64_pand val low32_mask))
(low32_as_float Xmm (x64_por low32 float_1p52))
(high32 Xmm (x64_psrlq val (xmi_imm 32)))
@@ -3780,9 +3793,9 @@
;; every value of the mantissa represents a corresponding uint32 number.
;; When we subtract 0x1.0p52 we are left with double(src).
(rule 1 (lower (has_type $F64X2 (fcvt_from_uint _ (uwiden_low _ val @ (value_type $I32X4)))))
- (let ((uint_mask XmmMem (emit_u128_le_const 0x43300000_43300000))
+ (let ((uint_mask XmmMem128 (emit_xmm_mem_128_le_const 0x43300000_43300000))
(res Xmm (x64_unpcklps val uint_mask))
- (uint_mask_high XmmMem (emit_u128_le_const 0x4330000000000000_4330000000000000)))
+ (uint_mask_high XmmMem128 (emit_xmm_mem_128_le_const 0x4330000000000000_4330000000000000)))
(x64_subpd res uint_mask_high)))
;; When AVX512VL and AVX512F are available,
@@ -3978,7 +3991,7 @@
(rule (lower (has_type $I8X16 (iadd_pairwise _ x y)))
(let (
;; Shuffle all the even lanes of `x` and `y` into one register
- (even_lane_mask Xmm (x64_movdqu_load (emit_u128_le_const 0x00ff_00ff_00ff_00ff_00ff_00ff_00ff_00ff)))
+ (even_lane_mask Xmm (x64_movdqu_load (emit_xmm_mem_128_le_const 0x00ff_00ff_00ff_00ff_00ff_00ff_00ff_00ff)))
(x_evens Xmm (x64_pand x even_lane_mask))
(y_evens Xmm (x64_pand y even_lane_mask))
(evens Xmm (x64_packuswb x_evens y_evens))
@@ -4051,7 +4064,7 @@
(has_type $I32X4 (iadd_pairwise _
(swiden_low _ val @ (value_type $I16X8))
(swiden_high _ val))))
- (let ((mul_const XmmMem (emit_u128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001)))
+ (let ((mul_const XmmMem128 (emit_xmm_mem_128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001)))
(x64_pmaddwd val mul_const)))
;; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction
@@ -4060,7 +4073,7 @@
(uwiden_low _ val @ (value_type $I8X16))
(uwiden_high _ val))))
(if-let true (has_ssse3))
- (let ((mul_const XmmMem (emit_u128_le_const 0x01010101010101010101010101010101)))
+ (let ((mul_const XmmMem128 (emit_xmm_mem_128_le_const 0x01010101010101010101010101010101)))
(x64_pmaddubsw val mul_const)))
;; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction
@@ -4068,13 +4081,13 @@
(has_type $I32X4 (iadd_pairwise _
(uwiden_low _ val @ (value_type $I16X8))
(uwiden_high _ val))))
- (let ((xor_const XmmMem (emit_u128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))
+ (let ((xor_const XmmMem128 (emit_xmm_mem_128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))
(dst Xmm (x64_pxor val xor_const))
- (madd_const XmmMem (emit_u128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001))
+ (madd_const XmmMem128 (emit_xmm_mem_128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001))
(dst Xmm (x64_pmaddwd dst madd_const))
- (addd_const XmmMem (emit_u128_le_const 0x00010000_00010000_00010000_00010000)))
+ (addd_const XmmMem128 (emit_xmm_mem_128_le_const 0x00010000_00010000_00010000_00010000)))
(x64_paddd dst addd_const)))
;; special case for the `i32x4.dot_i16x8_s` wasm instruction
@@ -4086,16 +4099,20 @@
;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; With SSE4.1 use the `pmovsx*` instructions for this
+;; With SSE4.1 use the `pmovsx*` instructions for this. The `pmovsx*`
+;; instructions only read the low 8 bytes of the source (`xmm_m64`), but
+;; `val` here is a 128-bit vector value; force it into an XMM register so
+;; we don't accidentally fold a full 16-byte load in place of the 8-byte
+;; one.
(rule 1 (lower (has_type $I16X8 (swiden_low _ val @ (value_type $I8X16))))
(if-let true (has_sse41))
- (x64_pmovsxbw val))
+ (x64_pmovsxbw (put_in_xmm val)))
(rule 1 (lower (has_type $I32X4 (swiden_low _ val @ (value_type $I16X8))))
(if-let true (has_sse41))
- (x64_pmovsxwd val))
+ (x64_pmovsxwd (put_in_xmm val)))
(rule 1 (lower (has_type $I64X2 (swiden_low _ val @ (value_type $I32X4))))
(if-let true (has_sse41))
- (x64_pmovsxdq val))
+ (x64_pmovsxdq (put_in_xmm val)))
(rule (lower (has_type ty (swiden_low _ val))) (lower_swiden_low ty val))
@@ -4151,16 +4168,17 @@
;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; With SSE4.1 use the `pmovzx*` instructions for this
+;; With SSE4.1 use the `pmovzx*` instructions for this. Same register-forcing
+;; rationale as the `pmovsx*` case above.
(rule 1 (lower (has_type $I16X8 (uwiden_low _ val @ (value_type $I8X16))))
(if-let true (has_sse41))
- (x64_pmovzxbw val))
+ (x64_pmovzxbw (put_in_xmm val)))
(rule 1 (lower (has_type $I32X4 (uwiden_low _ val @ (value_type $I16X8))))
(if-let true (has_sse41))
- (x64_pmovzxwd val))
+ (x64_pmovzxwd (put_in_xmm val)))
(rule 1 (lower (has_type $I64X2 (uwiden_low _ val @ (value_type $I32X4))))
(if-let true (has_sse41))
- (x64_pmovzxdq val))
+ (x64_pmovzxdq (put_in_xmm val)))
(rule (lower (has_type ty (uwiden_low _ val))) (lower_uwiden_low ty val))
@@ -4213,7 +4231,7 @@
(tmp1 Xmm (x64_cmppd a a (FcmpImm.Equal)))
;; 2147483647.0 is equivalent to 0x41DFFFFFFFC00000
- (umax_mask XmmMem (emit_u128_le_const 0x41DFFFFFFFC00000_41DFFFFFFFC00000))
+ (umax_mask XmmMem128 (emit_xmm_mem_128_le_const 0x41DFFFFFFFC00000_41DFFFFFFFC00000))
;; ANDPD xmm_y, [wasm_f64x2_splat(2147483647.0)]
(tmp1 Xmm (x64_andps tmp1 umax_mask))
@@ -4256,7 +4274,7 @@
(val Xmm (x64_pand val val_gt_zero))
;; Next clamp all larger-than-u16-max lanes to u16::MAX.
- (max Xmm (x64_movdqu_load (emit_u128_le_const 0x0000ffff_0000ffff_0000ffff_0000ffff)))
+ (max Xmm (x64_movdqu_load (emit_xmm_mem_128_le_const 0x0000ffff_0000ffff_0000ffff_0000ffff)))
(cmp Xmm (x64_pcmpgtd_a_or_avx max val))
(valid_lanes Xmm (x64_pand val cmp))
(clamped_lanes Xmm (x64_pandn cmp max))
@@ -4322,25 +4340,37 @@
;; Emits either a `round{ss,sd,ps,pd}` instruction, as appropriate, or generates
;; the appropriate libcall and sequence to call that.
-;;
-;; Recursion: at most once to convert memory case into register case.
-(decl rec x64_round (Type RegMem RoundImm) Xmm)
-(rule 1 (x64_round $F32 a imm)
+(decl x64_round (Type Value RoundImm) Xmm)
+(rule (x64_round $F32 a imm) (x64_round_f32 a imm))
+(rule (x64_round $F64 a imm) (x64_round_f64 a imm))
+(rule (x64_round $F32X4 a imm) (x64_round_f32x4 a imm))
+(rule (x64_round $F64X2 a imm) (x64_round_f64x2 a imm))
+
+;; Recursion on the memory fallback is bounded: the loaded register becomes
+;; an `XmmMem32.Reg`, which takes the `is_xmm_32` branch on the next call.
+(decl rec x64_round_f32 (XmmMem32 RoundImm) Xmm)
+(rule 2 (x64_round_f32 a imm)
(if-let true (has_sse41))
(x64_roundss a imm))
-(rule 1 (x64_round $F64 a imm)
+(rule 1 (x64_round_f32 (is_xmm_32 a) imm)
+ (libcall_1 (round_libcall $F32 imm) a))
+(rule 0 (x64_round_f32 (is_mem_32 addr) imm)
+ (x64_round_f32 (x64_movss_load addr) imm))
+
+(decl rec x64_round_f64 (XmmMem64 RoundImm) Xmm)
+(rule 2 (x64_round_f64 a imm)
(if-let true (has_sse41))
(x64_roundsd a imm))
-(rule 1 (x64_round $F32X4 a imm)
+(rule 1 (x64_round_f64 (is_xmm_64 a) imm)
+ (libcall_1 (round_libcall $F64 imm) a))
+(rule 0 (x64_round_f64 (is_mem_64 addr) imm)
+ (x64_round_f64 (x64_movsd_load addr) imm))
+
+(decl rec x64_round_f32x4 (XmmMem128 RoundImm) Xmm)
+(rule 2 (x64_round_f32x4 a imm)
(if-let true (has_sse41))
(x64_roundps a imm))
-(rule 1 (x64_round $F64X2 a imm)
- (if-let true (has_sse41))
- (x64_roundpd a imm))
-
-(rule (x64_round $F32 (RegMem.Reg a) imm) (libcall_1 (round_libcall $F32 imm) a))
-(rule (x64_round $F64 (RegMem.Reg a) imm) (libcall_1 (round_libcall $F64 imm) a))
-(rule (x64_round $F32X4 (RegMem.Reg a) imm)
+(rule 1 (x64_round_f32x4 (is_xmm_128 a) imm)
(let (
(libcall LibCall (round_libcall $F32 imm))
(result Xmm (libcall_1 libcall a))
@@ -4352,15 +4382,22 @@
(result Xmm (f32x4_insertlane result a3 3))
)
result))
-(rule (x64_round $F64X2 (RegMem.Reg a) imm)
+(rule 0 (x64_round_f32x4 (is_mem_128 addr) imm)
+ (x64_round_f32x4 (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 addr)) imm))
+
+(decl rec x64_round_f64x2 (XmmMem128 RoundImm) Xmm)
+(rule 2 (x64_round_f64x2 a imm)
+ (if-let true (has_sse41))
+ (x64_roundpd a imm))
+(rule 1 (x64_round_f64x2 (is_xmm_128 a) imm)
(let (
(libcall LibCall (round_libcall $F64 imm))
(result Xmm (libcall_1 libcall a))
(a1 Xmm (libcall_1 libcall (x64_pshufd a 0b00_00_11_10)))
)
(x64_movlhps result a1)))
-(rule (x64_round ty (RegMem.Mem addr) imm)
- (x64_round ty (RegMem.Reg (x64_load_xmm ty addr)) imm))
+(rule 0 (x64_round_f64x2 (is_mem_128 addr) imm)
+ (x64_round_f64x2 (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 addr)) imm))
(decl round_libcall (Type RoundImm) LibCall)
(rule (round_libcall $F32 (RoundImm.RoundUp)) (LibCall.CeilF32))
@@ -4683,7 +4720,7 @@
;; mask values) and fix the indexes to all point to the `dst` vector.
(rule 3 (lower (shuffle _ a a (vec_mask_from_immediate mask)))
(if-let true (has_ssse3))
- (x64_pshufb a (shuffle_0_31_mask mask)))
+ (x64_pshufb a (const_to_xmm_mem_128 (shuffle_0_31_mask mask))))
;; For the case where the shuffle mask contains out-of-bounds values (values
;; greater than 31) we must mask off those resulting values in the result of
@@ -4691,7 +4728,7 @@
(rule 2 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))
(if-let true (has_avx512vl))
(if-let true (has_avx512vbmi))
- (x64_andps (x64_vpermi2b (x64_xmm_load_const $I8X16 mask) a b) zeros))
+ (x64_andps (x64_vpermi2b (x64_xmm_load_const $I8X16 mask) a b) (const_to_xmm_mem_128 zeros)))
;; However, if the shuffle mask contains no out-of-bounds values, we can use
;; `vpermi2b` without any masking.
@@ -4705,8 +4742,8 @@
;; above, we build the `constructed_mask` for each case statically.
(rule (lower (shuffle _ a b (vec_mask_from_immediate mask)))
(x64_por
- (lower_pshufb a (shuffle_0_15_mask mask))
- (lower_pshufb b (shuffle_16_31_mask mask))))
+ (lower_pshufb a (const_to_xmm_mem_128 (shuffle_0_15_mask mask)))
+ (lower_pshufb b (const_to_xmm_mem_128 (shuffle_16_31_mask mask)))))
;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -4717,7 +4754,7 @@
;; Wasm SIMD semantics for this instruction. The instruction format maps to
;; variables like: %dst = swizzle %src, %mask
(rule (lower (swizzle _ src mask))
- (let ((mask Xmm (x64_paddusb mask (emit_u128_le_const 0x70707070707070707070707070707070))))
+ (let ((mask Xmm (x64_paddusb mask (emit_xmm_mem_128_le_const 0x70707070707070707070707070707070))))
(lower_pshufb src mask)))
;; Rules for `x86_pshufb` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -4733,14 +4770,14 @@
;; of SSSE3 to use the `pshufb` instruction itself.
;;
;; Recursion: at most once to implement the memory load case.
-(decl rec lower_pshufb (Xmm RegMem) Xmm)
-(rule 1 (lower_pshufb src mask)
+(decl rec lower_pshufb (Xmm XmmMem128) Xmm)
+(rule 2 (lower_pshufb src mask)
(if-let true (has_ssse3))
(x64_pshufb src mask))
-(rule (lower_pshufb src (RegMem.Reg mask))
+(rule 1 (lower_pshufb src (is_xmm_128 mask))
(libcall_2 (LibCall.X86Pshufb) src mask))
-(rule (lower_pshufb src (RegMem.Mem addr))
- (lower_pshufb src (x64_movdqu_load addr)))
+(rule 0 (lower_pshufb src (is_mem_128 addr))
+ (lower_pshufb src (x64_movdqu_load (synthetic_amode_to_xmm_mem_128 addr))))
;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -4836,7 +4873,8 @@
;; i8x16 splats: use `vpbroadcastb` on AVX2 and otherwise `pshufb` broadcasts
;; with a mask of zero which is calculated with an xor-against-itself register.
(rule 0 (lower (has_type $I8X16 (splat _ src)))
- (let ((src Xmm (x64_movd_to_xmm src)))
+ (let ((src Gpr src)
+ (src Xmm (x64_movd_to_xmm src)))
(x64_pshufd (x64_pshuflw (x64_punpcklbw src src) 0) 0)))
(rule 1 (lower (has_type $I8X16 (splat _ src)))
(if-let true (has_ssse3))
@@ -4848,7 +4886,7 @@
(if-let true (has_sse41))
(if-let true (has_ssse3))
(x64_pshufb (x64_pinsrb (xmm_uninit_value) addr 0) (xmm_zero $I8X16)))
-(rule 4 (lower (has_type $I8X16 (splat _ (sinkable_load_exact addr))))
+(rule 4 (lower (has_type $I8X16 (splat _ (sinkable_load_8 addr))))
(if-let true (use_avx2))
(x64_vpbroadcastb addr))
@@ -4864,7 +4902,7 @@
(x64_vpbroadcastw (bitcast_gpr_to_xmm 32 src)))
(rule 2 (lower (has_type $I16X8 (splat _ (sinkable_load_exact addr))))
(x64_pshufd (x64_pshuflw (x64_pinsrw (xmm_uninit_value) addr 0) 0) 0))
-(rule 3 (lower (has_type $I16X8 (splat _ (sinkable_load_exact addr))))
+(rule 3 (lower (has_type $I16X8 (splat _ (sinkable_load_16 addr))))
(if-let true (use_avx2))
(x64_vpbroadcastw addr))
@@ -4900,7 +4938,7 @@
(rule 5 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load addr))))
(let ((tmp Xmm (x64_movss_load addr)))
(x64_shufps tmp tmp 0)))
-(rule 6 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load addr))))
+(rule 6 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load_32 addr))))
(if-let true (has_avx))
(x64_vbroadcastss addr))
@@ -4911,7 +4949,7 @@
(x64_pshufd (bitcast_gpr_to_xmm 64 src) 0b01_00_01_00))
(rule 0 (lower (has_type $F64X2 (splat _ src)))
(x64_pshufd (put_in_xmm src) 0b01_00_01_00))
-(rule 6 (lower (has_type (multi_lane 64 2) (splat _ (sinkable_load addr))))
+(rule 6 (lower (has_type (multi_lane 64 2) (splat _ (sinkable_load_64 addr))))
(if-let true (has_sse3))
(x64_movddup addr))
@@ -5018,7 +5056,7 @@
(let ((src1 Xmm qx)
(src2 Xmm qy)
- (mask XmmMem (emit_u128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))
+ (mask XmmMem128 (emit_xmm_mem_128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))
(dst Xmm (x64_pmulhrsw src1 src2))
(cmp Xmm (x64_pcmpeqw dst mask)))
(x64_pxor dst cmp)))
@@ -5043,7 +5081,7 @@
(mul_lo Xmm (x64_punpcklwd mul_lsb mul_msb))
(mul_hi Xmm (x64_punpckhwd mul_lsb mul_msb))
;; Add the 0x4000 constant to all multiplications
- (val Xmm (x64_movdqu_load (emit_u128_le_const 0x00004000_00004000_00004000_00004000)))
+ (val Xmm (x64_movdqu_load (emit_xmm_mem_128_le_const 0x00004000_00004000_00004000_00004000)))
(mul_lo Xmm (x64_paddd mul_lo val))
(mul_hi Xmm (x64_paddd mul_hi val))
;; Perform the right-shift by 15 to all multiplications
@@ -5083,16 +5121,16 @@
(dst Xmm (x64_maxpd src zeros))
;; 4294967295.0 is equivalent to 0x41EFFFFFFFE00000
- (umax_mask XmmMem (emit_u128_le_const 0x41EFFFFFFFE00000_41EFFFFFFFE00000))
+ (umax_mask XmmMem128 (emit_xmm_mem_128_le_const 0x41EFFFFFFFE00000_41EFFFFFFFE00000))
;; MINPD xmm_y, [wasm_f64x2_splat(4294967295.0)]
(dst Xmm (x64_minpd dst umax_mask))
;; ROUNDPD xmm_y, xmm_y, 0x0B
- (dst Xmm (x64_round $F64X2 dst (RoundImm.RoundZero)))
+ (dst Xmm (x64_round_f64x2 dst (RoundImm.RoundZero)))
;; ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)]
- (uint_mask XmmMem (emit_u128_le_const 0x4330000000000000_4330000000000000))
+ (uint_mask XmmMem128 (emit_xmm_mem_128_le_const 0x4330000000000000_4330000000000000))
(dst Xmm (x64_addpd dst uint_mask)))
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 60d50df4ea..5a1ada113e 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -43,12 +43,226 @@ type BoxAtomic128XchgSeqArgs = Box<Atomic128XchgSeqArgs>;
/// need to fix the types we'll use.
type AssemblerInst = asm::Inst<CraneliftRegisters>;
+#[derive(Clone)]
pub struct SinkableLoad {
inst: Inst,
addr_input: InsnInput,
offset: i32,
}
+/// Width-tagged wrappers around [`SinkableLoad`].
+///
+/// A `SinkableLoad` is a load instruction that's a candidate to be sunk (fused)
+/// into another instruction as a memory operand. Because an ISLE rule can be
+/// written where the *consumer* instruction expects a different-width load
+/// than the original CLIF load, using an untagged `SinkableLoad` makes it easy
+/// to construct a "load N bytes into an instruction that reads M bytes" bug
+/// (the exact class we're catching with sized `XmmMem{N}` / `GprMem{N}`
+/// elsewhere).
+///
+/// These newtypes pin the width at the type level: `SinkableLoadN` can only
+/// be constructed via `sinkable_load_N`, which extracts only when the source
+/// `Value`'s CLIF type is exactly N bits wide. A converter
+/// `SinkableLoadN -> {Gpr,Xmm}MemN` is then safe.
+#[derive(Clone)]
+pub struct SinkableLoad8(SinkableLoad);
+#[derive(Clone)]
+pub struct SinkableLoad16(SinkableLoad);
+#[derive(Clone)]
+pub struct SinkableLoad32(SinkableLoad);
+#[derive(Clone)]
+pub struct SinkableLoad64(SinkableLoad);
+#[derive(Clone)]
+pub struct SinkableLoad128(SinkableLoad);
+
+/// Generate the sized-operand helpers for one GPR width. Invoked inside
+/// the `impl generated_code::Context for IsleContext` block.
+///
+/// Parameters: the bit width, the sized newtype names, and explicit names
+/// for each generated method. See the invocation sites for layout.
+macro_rules! sized_gpr_mem_helpers {
+ (
+ $n:literal, $gpr_mem_n:ident, $gpr_mem_imm_n:ident,
+ $put_in_gpr_mem_n:ident, $put_in_gpr_mem_imm_n:ident,
+ $gpr_to_gpr_mem_n:ident, $gpr_to_gpr_mem_imm_n:ident,
+ $gpr_mem_imm_n_imm:ident,
+ $gpr_mem_n_to_gpr_mem:ident, $gpr_mem_imm_n_to_gpr_mem_imm:ident
+ ) => {
+ fn $put_in_gpr_mem_n(&mut self, val: Value) -> $gpr_mem_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_gpr_mem_n),
+ $n,
+ );
+ $gpr_mem_n::unwrap_new(self.put_in_reg_mem(val))
+ }
+
+ fn $put_in_gpr_mem_imm_n(&mut self, val: Value) -> $gpr_mem_imm_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_gpr_mem_imm_n),
+ $n,
+ );
+ $gpr_mem_imm_n::unwrap_new(self.put_in_reg_mem_imm(val))
+ }
+
+ #[inline]
+ fn $gpr_to_gpr_mem_n(&mut self, gpr: Gpr) -> $gpr_mem_n {
+ $gpr_mem_n::from(gpr)
+ }
+
+ #[inline]
+ fn $gpr_to_gpr_mem_imm_n(&mut self, gpr: Gpr) -> $gpr_mem_imm_n {
+ $gpr_mem_imm_n::from(gpr)
+ }
+
+ #[inline]
+ fn $gpr_mem_imm_n_imm(&mut self, simm32: u32) -> $gpr_mem_imm_n {
+ $gpr_mem_imm_n::unwrap_new(RegMemImm::Imm { simm32 })
+ }
+
+ #[inline]
+ fn $gpr_mem_n_to_gpr_mem(&mut self, x: &$gpr_mem_n) -> GprMem {
+ GprMem::unwrap_new(x.clone().into())
+ }
+
+ #[inline]
+ fn $gpr_mem_imm_n_to_gpr_mem_imm(&mut self, x: &$gpr_mem_imm_n) -> GprMemImm {
+ GprMemImm::unwrap_new(x.clone().into())
+ }
+ };
+}
+
+/// Generate the sized-operand helpers for one XMM width. Invoked inside
+/// the `impl generated_code::Context for IsleContext` block. Covers the
+/// non-aligned/aligned variants plus their `Imm` forms.
+macro_rules! sized_xmm_mem_helpers {
+ (
+ $n:literal,
+ $xmm_mem_n:ident, $xmm_mem_aligned_n:ident,
+ $xmm_mem_imm_n:ident, $xmm_mem_aligned_imm_n:ident,
+ $put_in_xmm_mem_n:ident, $put_in_xmm_mem_aligned_n:ident,
+ $put_in_xmm_mem_imm_n:ident, $put_in_xmm_mem_aligned_imm_n:ident,
+ $xmm_to_xmm_mem_n:ident, $xmm_to_xmm_mem_aligned_n:ident,
+ $xmm_to_xmm_mem_imm_n:ident, $xmm_to_xmm_mem_aligned_imm_n:ident,
+ $xmm_mem_imm_n_imm:ident,
+ $xmm_mem_n_to_xmm_mem:ident, $xmm_mem_aligned_n_to_xmm_mem_aligned:ident,
+ $xmm_mem_imm_n_to_xmm_mem_imm:ident,
+ $xmm_mem_aligned_imm_n_to_xmm_mem_aligned_imm:ident
+ ) => {
+ fn $put_in_xmm_mem_n(&mut self, val: Value) -> $xmm_mem_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_xmm_mem_n),
+ $n,
+ );
+ $xmm_mem_n::unwrap_new(self.put_in_xmm_mem(val).to_reg_mem())
+ }
+
+ fn $put_in_xmm_mem_aligned_n(&mut self, val: Value) -> $xmm_mem_aligned_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_xmm_mem_aligned_n),
+ $n,
+ );
+ let sized = $xmm_mem_n::unwrap_new(self.put_in_xmm_mem(val).to_reg_mem());
+ // Round-trip through the un-sized aligning helper so unaligned
+ // memory sources are copied to a register first.
+ let aligned = self.xmm_mem_to_xmm_mem_aligned(&XmmMem::unwrap_new(sized.into()));
+ $xmm_mem_aligned_n::unwrap_new(aligned.into())
+ }
+
+ fn $put_in_xmm_mem_imm_n(&mut self, val: Value) -> $xmm_mem_imm_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_xmm_mem_imm_n),
+ $n,
+ );
+ $xmm_mem_imm_n::unwrap_new(self.put_in_xmm_mem_imm(val).into())
+ }
+
+ fn $put_in_xmm_mem_aligned_imm_n(&mut self, val: Value) -> $xmm_mem_aligned_imm_n {
+ let ty = self.lower_ctx.dfg().value_type(val);
+ assert_eq!(
+ ty.bits(),
+ $n,
+ "{}: expected a {}-bit value, got value of type {ty}",
+ stringify!($put_in_xmm_mem_aligned_imm_n),
+ $n,
+ );
+ let imm = self.put_in_xmm_mem_imm(val);
+ let aligned = self.xmm_mem_imm_to_xmm_mem_aligned_imm(&imm);
+ $xmm_mem_aligned_imm_n::unwrap_new(aligned.into())
+ }
+
+ #[inline]
+ fn $xmm_to_xmm_mem_n(&mut self, xmm: Xmm) -> $xmm_mem_n {
+ $xmm_mem_n::from(xmm)
+ }
+
+ #[inline]
+ fn $xmm_to_xmm_mem_aligned_n(&mut self, xmm: Xmm) -> $xmm_mem_aligned_n {
+ $xmm_mem_aligned_n::from(xmm)
+ }
+
+ #[inline]
+ fn $xmm_to_xmm_mem_imm_n(&mut self, xmm: Xmm) -> $xmm_mem_imm_n {
+ $xmm_mem_imm_n::from(xmm)
+ }
+
+ #[inline]
+ fn $xmm_to_xmm_mem_aligned_imm_n(&mut self, xmm: Xmm) -> $xmm_mem_aligned_imm_n {
+ $xmm_mem_aligned_imm_n::from(xmm)
+ }
+
+ #[inline]
+ fn $xmm_mem_imm_n_imm(&mut self, simm32: u32) -> $xmm_mem_imm_n {
+ $xmm_mem_imm_n::unwrap_new(RegMemImm::Imm { simm32 })
+ }
+
+ #[inline]
+ fn $xmm_mem_n_to_xmm_mem(&mut self, x: &$xmm_mem_n) -> XmmMem {
+ XmmMem::unwrap_new(x.clone().into())
+ }
+
+ #[inline]
+ fn $xmm_mem_aligned_n_to_xmm_mem_aligned(
+ &mut self,
+ x: &$xmm_mem_aligned_n,
+ ) -> XmmMemAligned {
+ XmmMemAligned::unwrap_new(x.clone().into())
+ }
+
+ #[inline]
+ fn $xmm_mem_imm_n_to_xmm_mem_imm(&mut self, x: &$xmm_mem_imm_n) -> XmmMemImm {
+ XmmMemImm::unwrap_new(x.clone().into())
+ }
+
+ #[inline]
+ fn $xmm_mem_aligned_imm_n_to_xmm_mem_aligned_imm(
+ &mut self,
+ x: &$xmm_mem_aligned_imm_n,
+ ) -> XmmMemAlignedImm {
+ XmmMemAlignedImm::unwrap_new(x.clone().into())
+ }
+ };
+}
+
/// The main entry point for lowering with ISLE.
pub(crate) fn lower(
lower_ctx: &mut Lower<MInst>,
@@ -379,6 +593,111 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
None
}
+ // Width-tagged sinkable-load extractors. Each only fires when the `Value`
+ // being sunk has a CLIF type of exactly N bits, so the resulting
+ // `SinkableLoadN` carries that width in its type and can be folded into
+ // an N-bit memory operand without a dynamic width check.
+
+ fn sinkable_load_8(&mut self, val: Value) -> Option<SinkableLoad8> {
+ if self.lower_ctx.dfg().value_type(val).bits() != 8 {
+ return None;
+ }
+ self.sinkable_load_exact(val).map(SinkableLoad8)
+ }
+
+ fn sinkable_load_16(&mut self, val: Value) -> Option<SinkableLoad16> {
+ if self.lower_ctx.dfg().value_type(val).bits() != 16 {
+ return None;
+ }
+ self.sinkable_load_exact(val).map(SinkableLoad16)
+ }
+
+ fn sinkable_load_32(&mut self, val: Value) -> Option<SinkableLoad32> {
+ if self.lower_ctx.dfg().value_type(val).bits() != 32 {
+ return None;
+ }
+ self.sinkable_load_exact(val).map(SinkableLoad32)
+ }
+
+ fn sinkable_load_64(&mut self, val: Value) -> Option<SinkableLoad64> {
+ if self.lower_ctx.dfg().value_type(val).bits() != 64 {
+ return None;
+ }
+ self.sinkable_load_exact(val).map(SinkableLoad64)
+ }
+
+ fn sinkable_load_128(&mut self, val: Value) -> Option<SinkableLoad128> {
+ if self.lower_ctx.dfg().value_type(val).bits() != 128 {
+ return None;
+ }
+ self.sinkable_load_exact(val).map(SinkableLoad128)
+ }
+
+ // Conversions from width-tagged sinkable loads to sized memory operands.
+ // These are safe because the width was pinned at extractor tim.e
+
+ #[inline]
+ fn sink_load_8(&mut self, load: &SinkableLoad8) -> SyntheticAmode {
+ self.sink_load(&load.0)
+ }
+ #[inline]
+ fn sink_load_16(&mut self, load: &SinkableLoad16) -> SyntheticAmode {
+ self.sink_load(&load.0)
+ }
+ #[inline]
+ fn sink_load_32(&mut self, load: &SinkableLoad32) -> SyntheticAmode {
+ self.sink_load(&load.0)
+ }
+ #[inline]
+ fn sink_load_64(&mut self, load: &SinkableLoad64) -> SyntheticAmode {
+ self.sink_load(&load.0)
+ }
+ #[inline]
+ fn sink_load_128(&mut self, load: &SinkableLoad128) -> SyntheticAmode {
+ self.sink_load(&load.0)
+ }
+
+ // Direct sized-memory wrappers. Each sinks the load and then wraps the
+ // resulting address at the matching width. Safe by construction.
+
+ #[inline]
+ fn sink_load_8_to_gpr_mem_8(&mut self, load: &SinkableLoad8) -> GprMem8 {
+ GprMem8::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_16_to_gpr_mem_16(&mut self, load: &SinkableLoad16) -> GprMem16 {
+ GprMem16::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_32_to_gpr_mem_32(&mut self, load: &SinkableLoad32) -> GprMem32 {
+ GprMem32::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_64_to_gpr_mem_64(&mut self, load: &SinkableLoad64) -> GprMem64 {
+ GprMem64::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+
+ #[inline]
+ fn sink_load_8_to_xmm_mem_8(&mut self, load: &SinkableLoad8) -> XmmMem8 {
+ XmmMem8::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_16_to_xmm_mem_16(&mut self, load: &SinkableLoad16) -> XmmMem16 {
+ XmmMem16::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_32_to_xmm_mem_32(&mut self, load: &SinkableLoad32) -> XmmMem32 {
+ XmmMem32::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_64_to_xmm_mem_64(&mut self, load: &SinkableLoad64) -> XmmMem64 {
+ XmmMem64::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+ #[inline]
+ fn sink_load_128_to_xmm_mem_128(&mut self, load: &SinkableLoad128) -> XmmMem128 {
+ XmmMem128::unwrap_new(RegMem::mem(self.sink_load(&load.0)))
+ }
+
fn sink_load(&mut self, load: &SinkableLoad) -> SyntheticAmode {
self.lower_ctx.sink_inst(load.inst);
let addr = lower_to_amode(self.lower_ctx, load.addr_input, load.offset);
@@ -421,6 +740,164 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
SyntheticAmode::ConstantOffset(c)
}
+ /// Wrap a `VCodeConstant` that holds a 128-bit value as an `XmmMem128`
+ /// operand. The caller is responsible for ensuring the pool entry is
+ /// actually 128 bits wide (e.g. via `emit_u128_le_const`). There is
+ /// intentionally no auto-conversion `VCodeConstant -> XmmMem128`
+ /// because `VCodeConstant` doesn't carry width.
+ #[inline]
+ fn const_to_xmm_mem_128(&mut self, c: VCodeConstant) -> XmmMem128 {
+ XmmMem128::unwrap_new(RegMem::mem(SyntheticAmode::ConstantOffset(c)))
+ }
+
+ /// Wrap a `SyntheticAmode` as a 128-bit XMM memory operand. Pins the
+ /// width at the call site; no auto-conversion is provided.
+ #[inline]
+ fn synthetic_amode_to_xmm_mem_128(&mut self, amode: &SyntheticAmode) -> XmmMem128 {
+ XmmMem128::unwrap_new(RegMem::mem(amode.clone()))
+ }
+
+ /// Wrap a `SyntheticAmode` as a 64-bit XMM memory operand.
+ #[inline]
+ fn synthetic_amode_to_xmm_mem_64(&mut self, amode: &SyntheticAmode) -> XmmMem64 {
+ XmmMem64::unwrap_new(RegMem::mem(amode.clone()))
+ }
+
+ /// Wrap a `SyntheticAmode` as a 32-bit XMM memory operand.
+ #[inline]
+ fn synthetic_amode_to_xmm_mem_32(&mut self, amode: &SyntheticAmode) -> XmmMem32 {
+ XmmMem32::unwrap_new(RegMem::mem(amode.clone()))
+ }
+
+ /// Unchecked narrowing from an un-sized `GprMem` to a sized
+ /// `GprMem{N}`. Used only by `x64_movzx` / `x64_movsx`, whose
+ /// per-ExtMode rules statically pin the source width.
+ #[inline]
+ fn gpr_mem_as_gpr_mem_8(&mut self, gm: &GprMem) -> GprMem8 {
+ GprMem8::unwrap_new(gm.clone().into())
+ }
+ #[inline]
+ fn gpr_mem_as_gpr_mem_16(&mut self, gm: &GprMem) -> GprMem16 {
+ GprMem16::unwrap_new(gm.clone().into())
+ }
+ #[inline]
+ fn gpr_mem_as_gpr_mem_32(&mut self, gm: &GprMem) -> GprMem32 {
+ GprMem32::unwrap_new(gm.clone().into())
+ }
+ #[inline]
+ fn gpr_mem_as_gpr_mem_64(&mut self, gm: &GprMem) -> GprMem64 {
+ GprMem64::unwrap_new(gm.clone().into())
+ }
+
+ /// Unchecked narrowing from an un-sized `XmmMem` / `XmmMemAligned`
+ /// to a sized variant. Used at fixed-width call sites (e.g. vector
+ /// shift helpers `x64_psllw` etc.) where the width is implied by
+ /// the opcode rather than a Cranelift `Type`.
+ #[inline]
+ fn xmm_mem_as_xmm_mem_32(&mut self, xm: &XmmMem) -> XmmMem32 {
+ XmmMem32::unwrap_new(xm.clone().into())
+ }
+ #[inline]
+ fn xmm_mem_as_xmm_mem_64(&mut self, xm: &XmmMem) -> XmmMem64 {
+ XmmMem64::unwrap_new(xm.clone().into())
+ }
+ #[inline]
+ fn xmm_mem_as_xmm_mem_128(&mut self, xm: &XmmMem) -> XmmMem128 {
+ XmmMem128::unwrap_new(xm.clone().into())
+ }
+ #[inline]
+ fn xmm_mem_aligned_as_xmm_mem_aligned_32(&mut self, xm: &XmmMemAligned) -> XmmMemAligned32 {
+ XmmMemAligned32::unwrap_new(xm.clone().into())
+ }
+ #[inline]
+ fn xmm_mem_aligned_as_xmm_mem_aligned_64(&mut self, xm: &XmmMemAligned) -> XmmMemAligned64 {
+ XmmMemAligned64::unwrap_new(xm.clone().into())
+ }
+ #[inline]
+ fn xmm_mem_aligned_as_xmm_mem_aligned_128(&mut self, xm: &XmmMemAligned) -> XmmMemAligned128 {
+ XmmMemAligned128::unwrap_new(xm.clone().into())
+ }
+
+ fn gpr_mem_8_for_ty(&mut self, ty: Type, gm: &GprMem) -> GprMem8 {
+ assert_eq!(
+ ty.bits(),
+ 8,
+ "gpr_mem_8_for_ty: expected an 8-bit Type, got {ty}",
+ );
+ GprMem8::unwrap_new(gm.clone().into())
+ }
+ fn gpr_mem_16_for_ty(&mut self, ty: Type, gm: &GprMem) -> GprMem16 {
+ assert_eq!(
+ ty.bits(),
+ 16,
+ "gpr_mem_16_for_ty: expected a 16-bit Type, got {ty}",
+ );
+ GprMem16::unwrap_new(gm.clone().into())
+ }
+ fn gpr_mem_32_for_ty(&mut self, ty: Type, gm: &GprMem) -> GprMem32 {
+ assert_eq!(
+ ty.bits(),
+ 32,
+ "gpr_mem_32_for_ty: expected a 32-bit Type, got {ty}",
+ );
+ GprMem32::unwrap_new(gm.clone().into())
+ }
+ fn gpr_mem_64_for_ty(&mut self, ty: Type, gm: &GprMem) -> GprMem64 {
+ assert_eq!(
+ ty.bits(),
+ 64,
+ "gpr_mem_64_for_ty: expected a 64-bit Type, got {ty}",
+ );
+ GprMem64::unwrap_new(gm.clone().into())
+ }
+ fn gpr_mem_as_gpr_mem_32_or_widened_in_reg(&mut self, ty: Type, gm: &GprMem) -> GprMem32 {
+ let safe = match gm.clone().to_reg_mem() {
+ _ if ty.bits() == 32 => true,
+ RegMem::Reg { .. } => true,
+ RegMem::Mem {
+ addr: SyntheticAmode::ConstantOffset(_),
+ } => true,
+ RegMem::Mem { .. } => false,
+ };
+ assert!(
+ safe,
+ "gpr_mem_as_gpr_mem_32_or_widened_in_reg: cannot widen a {ty} memory operand"
+ );
+ GprMem32::unwrap_new(gm.clone().into())
+ }
+ fn xmm_mem_32_for_ty(&mut self, ty: Type, xm: &XmmMem) -> XmmMem32 {
+ assert_eq!(
+ ty.bits(),
+ 32,
+ "xmm_mem_32_for_ty: expected a 32-bit Type, got {ty}",
+ );
+ XmmMem32::unwrap_new(xm.clone().into())
+ }
+ fn xmm_mem_64_for_ty(&mut self, ty: Type, xm: &XmmMem) -> XmmMem64 {
+ assert_eq!(
+ ty.bits(),
+ 64,
+ "xmm_mem_64_for_ty: expected a 64-bit Type, got {ty}",
+ );
+ XmmMem64::unwrap_new(xm.clone().into())
+ }
+ fn xmm_mem_128_for_ty(&mut self, ty: Type, xm: &XmmMem) -> XmmMem128 {
+ assert_eq!(
+ ty.bits(),
+ 128,
+ "xmm_mem_128_for_ty: expected a 128-bit Type, got {ty}",
+ );
+ XmmMem128::unwrap_new(xm.clone().into())
+ }
+ fn xmm_mem_aligned_128_for_ty(&mut self, ty: Type, xm: &XmmMemAligned) -> XmmMemAligned128 {
+ assert_eq!(
+ ty.bits(),
+ 128,
+ "xmm_mem_aligned_128_for_ty: expected a 128-bit Type, got {ty}",
+ );
+ XmmMemAligned128::unwrap_new(xm.clone().into())
+ }
+
#[inline]
fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg {
r.to_writable_reg()
@@ -431,6 +908,104 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
r.to_writable_reg()
}
+ #[inline]
+ fn synthetic_amode_to_gpr_mem_8(&mut self, amode: &SyntheticAmode) -> GprMem8 {
+ GprMem8::unwrap_new(RegMem::mem(amode.clone()))
+ }
+ #[inline]
+ fn synthetic_amode_to_gpr_mem_16(&mut self, amode: &SyntheticAmode) -> GprMem16 {
+ GprMem16::unwrap_new(RegMem::mem(amode.clone()))
+ }
+ #[inline]
+ fn synthetic_amode_to_gpr_mem_32(&mut self, amode: &SyntheticAmode) -> GprMem32 {
+ GprMem32::unwrap_new(RegMem::mem(amode.clone()))
+ }
+ #[inline]
+ fn synthetic_amode_to_gpr_mem_64(&mut self, amode: &SyntheticAmode) -> GprMem64 {
+ GprMem64::unwrap_new(RegMem::mem(amode.clone()))
+ }
+ #[inline]
+ fn synthetic_amode_to_xmm_mem_8(&mut self, amode: &SyntheticAmode) -> XmmMem8 {
+ XmmMem8::unwrap_new(RegMem::mem(amode.clone()))
+ }
+ #[inline]
+ fn synthetic_amode_to_xmm_mem_16(&mut self, amode: &SyntheticAmode) -> XmmMem16 {
+ XmmMem16::unwrap_new(RegMem::mem(amode.clone()))
+ }
+
+ #[inline]
+ fn reg_to_gpr_mem_8(&mut self, r: Reg) -> GprMem8 {
+ GprMem8::unwrap_new(RegMem::reg(r))
+ }
+ #[inline]
+ fn reg_to_gpr_mem_16(&mut self, r: Reg) -> GprMem16 {
+ GprMem16::unwrap_new(RegMem::reg(r))
+ }
+ #[inline]
+ fn reg_to_gpr_mem_32(&mut self, r: Reg) -> GprMem32 {
+ GprMem32::unwrap_new(RegMem::reg(r))
+ }
+ #[inline]
+ fn reg_to_gpr_mem_64(&mut self, r: Reg) -> GprMem64 {
+ GprMem64::unwrap_new(RegMem::reg(r))
+ }
+
+ #[inline]
+ fn writable_gpr_to_gpr_mem_8(&mut self, r: WritableGpr) -> GprMem8 {
+ GprMem8::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+ #[inline]
+ fn writable_gpr_to_gpr_mem_16(&mut self, r: WritableGpr) -> GprMem16 {
+ GprMem16::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+ #[inline]
+ fn writable_gpr_to_gpr_mem_32(&mut self, r: WritableGpr) -> GprMem32 {
+ GprMem32::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+ #[inline]
+ fn writable_gpr_to_gpr_mem_64(&mut self, r: WritableGpr) -> GprMem64 {
+ GprMem64::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+ #[inline]
+ fn writable_xmm_to_xmm_mem_128(&mut self, r: WritableXmm) -> XmmMem128 {
+ XmmMem128::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+ #[inline]
+ fn writable_xmm_to_xmm_mem_aligned_128(&mut self, r: WritableXmm) -> XmmMemAligned128 {
+ XmmMemAligned128::unwrap_new(RegMem::reg(r.to_reg().to_reg()))
+ }
+
+ #[inline]
+ fn xmm_mem_32_to_xmm_mem_aligned_32(&mut self, x: &XmmMem32) -> XmmMemAligned32 {
+ match XmmMemAligned32::new(x.clone().into()) {
+ Some(a) => a,
+ None => match x.clone().into() {
+ RegMem::Mem { addr } => self.load_xmm_unaligned(addr).into(),
+ _ => unreachable!(),
+ },
+ }
+ }
+ #[inline]
+ fn xmm_mem_64_to_xmm_mem_aligned_64(&mut self, x: &XmmMem64) -> XmmMemAligned64 {
+ match XmmMemAligned64::new(x.clone().into()) {
+ Some(a) => a,
+ None => match x.clone().into() {
+ RegMem::Mem { addr } => self.load_xmm_unaligned(addr).into(),
+ _ => unreachable!(),
+ },
+ }
+ }
+ #[inline]
+ fn xmm_mem_128_to_xmm_mem_aligned_128(&mut self, x: &XmmMem128) -> XmmMemAligned128 {
+ match XmmMemAligned128::new(x.clone().into()) {
+ Some(a) => a,
+ None => match x.clone().into() {
+ RegMem::Mem { addr } => self.load_xmm_unaligned(addr).into(),
+ _ => unreachable!(),
+ },
+ }
+ }
+
fn ishl_i8x16_mask_for_const(&mut self, amt: u32) -> SyntheticAmode {
// When the shift amount is known, we can statically (i.e. at compile
// time) determine the mask to use and only emit that.
@@ -577,6 +1152,178 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
GprMemImm::from(gpr)
}
+ // ---- Sized operand conversions ----
+ //
+ // For each sized newtype `{Gpr,Xmm}Mem{Imm,Aligned,AlignedImm}{N}`, we
+ // generate a small fixed set of helpers:
+ //
+ // * `put_in_..._N(Value)` converts a CLIF `Value` to the sized
+ // operand. It **release-asserts** (not `debug_assert!`) that the
+ // value's CLIF type is N bits wide, so a width mismatch at lowering
+ // time becomes a hard failure instead of silently emitting a wider
+ // load than intended.
+ // * `{gpr,xmm}_to_..._N({Gpr,Xmm})` wraps a typed register into the
+ // sized operand unconditionally -- `Gpr`/`Xmm` carry no width; the
+ // consumer's instruction determines it.
+ // * `..._N_to_...` downcasts a sized operand back to its un-sized
+ // form. These are transition bridges into still-un-sized helpers
+ // (mostly auto-generated assembler constructors); they should
+ // go away as the refactor proceeds.
+ //
+ // A `{gpr,xmm}_mem_imm_N_imm(u32)` typed-immediate constructor is also
+ // provided so call sites can build `GprMemImm{N}` without having to
+ // pierce the sized wrapper via `RegMemImm.Imm`.
+
+ sized_gpr_mem_helpers!(
+ 8,
+ GprMem8,
+ GprMemImm8,
+ put_in_gpr_mem_8,
+ put_in_gpr_mem_imm_8,
+ gpr_to_gpr_mem_8,
+ gpr_to_gpr_mem_imm_8,
+ gpr_mem_imm_8_imm,
+ gpr_mem_8_to_gpr_mem,
+ gpr_mem_imm_8_to_gpr_mem_imm
+ );
+ sized_gpr_mem_helpers!(
+ 16,
+ GprMem16,
+ GprMemImm16,
+ put_in_gpr_mem_16,
+ put_in_gpr_mem_imm_16,
+ gpr_to_gpr_mem_16,
+ gpr_to_gpr_mem_imm_16,
+ gpr_mem_imm_16_imm,
+ gpr_mem_16_to_gpr_mem,
+ gpr_mem_imm_16_to_gpr_mem_imm
+ );
+ sized_gpr_mem_helpers!(
+ 32,
+ GprMem32,
+ GprMemImm32,
+ put_in_gpr_mem_32,
+ put_in_gpr_mem_imm_32,
+ gpr_to_gpr_mem_32,
+ gpr_to_gpr_mem_imm_32,
+ gpr_mem_imm_32_imm,
+ gpr_mem_32_to_gpr_mem,
+ gpr_mem_imm_32_to_gpr_mem_imm
+ );
+ sized_gpr_mem_helpers!(
+ 64,
+ GprMem64,
+ GprMemImm64,
+ put_in_gpr_mem_64,
+ put_in_gpr_mem_imm_64,
+ gpr_to_gpr_mem_64,
+ gpr_to_gpr_mem_imm_64,
+ gpr_mem_imm_64_imm,
+ gpr_mem_64_to_gpr_mem,
+ gpr_mem_imm_64_to_gpr_mem_imm
+ );
+
+ sized_xmm_mem_helpers!(
+ 8,
+ XmmMem8,
+ XmmMemAligned8,
+ XmmMemImm8,
+ XmmMemAlignedImm8,
+ put_in_xmm_mem_8,
+ put_in_xmm_mem_aligned_8,
+ put_in_xmm_mem_imm_8,
+ put_in_xmm_mem_aligned_imm_8,
+ xmm_to_xmm_mem_8,
+ xmm_to_xmm_mem_aligned_8,
+ xmm_to_xmm_mem_imm_8,
+ xmm_to_xmm_mem_aligned_imm_8,
+ xmm_mem_imm_8_imm,
+ xmm_mem_8_to_xmm_mem,
+ xmm_mem_aligned_8_to_xmm_mem_aligned,
+ xmm_mem_imm_8_to_xmm_mem_imm,
+ xmm_mem_aligned_imm_8_to_xmm_mem_aligned_imm
+ );
+ sized_xmm_mem_helpers!(
+ 16,
+ XmmMem16,
+ XmmMemAligned16,
+ XmmMemImm16,
+ XmmMemAlignedImm16,
+ put_in_xmm_mem_16,
+ put_in_xmm_mem_aligned_16,
+ put_in_xmm_mem_imm_16,
+ put_in_xmm_mem_aligned_imm_16,
+ xmm_to_xmm_mem_16,
+ xmm_to_xmm_mem_aligned_16,
+ xmm_to_xmm_mem_imm_16,
+ xmm_to_xmm_mem_aligned_imm_16,
+ xmm_mem_imm_16_imm,
+ xmm_mem_16_to_xmm_mem,
+ xmm_mem_aligned_16_to_xmm_mem_aligned,
+ xmm_mem_imm_16_to_xmm_mem_imm,
+ xmm_mem_aligned_imm_16_to_xmm_mem_aligned_imm
+ );
+ sized_xmm_mem_helpers!(
+ 32,
+ XmmMem32,
+ XmmMemAligned32,
+ XmmMemImm32,
+ XmmMemAlignedImm32,
+ put_in_xmm_mem_32,
+ put_in_xmm_mem_aligned_32,
+ put_in_xmm_mem_imm_32,
+ put_in_xmm_mem_aligned_imm_32,
+ xmm_to_xmm_mem_32,
+ xmm_to_xmm_mem_aligned_32,
+ xmm_to_xmm_mem_imm_32,
+ xmm_to_xmm_mem_aligned_imm_32,
+ xmm_mem_imm_32_imm,
+ xmm_mem_32_to_xmm_mem,
+ xmm_mem_aligned_32_to_xmm_mem_aligned,
+ xmm_mem_imm_32_to_xmm_mem_imm,
+ xmm_mem_aligned_imm_32_to_xmm_mem_aligned_imm
+ );
+ sized_xmm_mem_helpers!(
+ 64,
+ XmmMem64,
+ XmmMemAligned64,
+ XmmMemImm64,
+ XmmMemAlignedImm64,
+ put_in_xmm_mem_64,
+ put_in_xmm_mem_aligned_64,
+ put_in_xmm_mem_imm_64,
+ put_in_xmm_mem_aligned_imm_64,
+ xmm_to_xmm_mem_64,
+ xmm_to_xmm_mem_aligned_64,
+ xmm_to_xmm_mem_imm_64,
+ xmm_to_xmm_mem_aligned_imm_64,
+ xmm_mem_imm_64_imm,
+ xmm_mem_64_to_xmm_mem,
+ xmm_mem_aligned_64_to_xmm_mem_aligned,
+ xmm_mem_imm_64_to_xmm_mem_imm,
+ xmm_mem_aligned_imm_64_to_xmm_mem_aligned_imm
+ );
+ sized_xmm_mem_helpers!(
+ 128,
+ XmmMem128,
+ XmmMemAligned128,
+ XmmMemImm128,
+ XmmMemAlignedImm128,
+ put_in_xmm_mem_128,
+ put_in_xmm_mem_aligned_128,
+ put_in_xmm_mem_imm_128,
+ put_in_xmm_mem_aligned_imm_128,
+ xmm_to_xmm_mem_128,
+ xmm_to_xmm_mem_aligned_128,
+ xmm_to_xmm_mem_imm_128,
+ xmm_to_xmm_mem_aligned_imm_128,
+ xmm_mem_imm_128_imm,
+ xmm_mem_128_to_xmm_mem,
+ xmm_mem_aligned_128_to_xmm_mem_aligned,
+ xmm_mem_imm_128_to_xmm_mem_imm,
+ xmm_mem_aligned_imm_128_to_xmm_mem_aligned_imm
+ );
+
#[inline]
fn type_register_class(&mut self, ty: Type) -> Option<RegisterClass> {
if is_int_or_ref_ty(ty) || ty == I128 {
@@ -1111,6 +1858,47 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
}
}
+ // Sized variants of `is_xmm`/`is_mem`. Each extracts the inner
+ // `Xmm`/`SyntheticAmode` from a `XmmMem{N}`. No width check is
+ // needed -- the sized wrapper already pinned it.
+
+ fn is_xmm_32(&mut self, src: &XmmMem32) -> Option<Xmm> {
+ match src.clone().into() {
+ RegMem::Reg { reg } => Xmm::new(reg),
+ _ => None,
+ }
+ }
+ fn is_xmm_64(&mut self, src: &XmmMem64) -> Option<Xmm> {
+ match src.clone().into() {
+ RegMem::Reg { reg } => Xmm::new(reg),
+ _ => None,
+ }
+ }
+ fn is_xmm_128(&mut self, src: &XmmMem128) -> Option<Xmm> {
+ match src.clone().into() {
+ RegMem::Reg { reg } => Xmm::new(reg),
+ _ => None,
+ }
+ }
+ fn is_mem_32(&mut self, src: &XmmMem32) -> Option<SyntheticAmode> {
+ match src.clone().into() {
+ RegMem::Reg { .. } => None,
+ RegMem::Mem { addr } => Some(addr),
+ }
+ }
+ fn is_mem_64(&mut self, src: &XmmMem64) -> Option<SyntheticAmode> {
+ match src.clone().into() {
+ RegMem::Reg { .. } => None,
+ RegMem::Mem { addr } => Some(addr),
+ }
+ }
+ fn is_mem_128(&mut self, src: &XmmMem128) -> Option<SyntheticAmode> {
+ match src.clone().into() {
+ RegMem::Reg { .. } => None,
+ RegMem::Mem { addr } => Some(addr),
+ }
+ }
+
// Custom constructors for `mulx` which only calculates the high half of the
// result meaning that the same output operand is used in both destination
// registers. This is in contrast to the assembler-generated version of this
@@ -1180,17 +1968,25 @@ impl IsleContext<'_, '_, MInst, X64Backend> {
}
/// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_gpr_mem_to_assembler_read_gpr_mem(&self, read: &GprMem) -> asm::GprMem<Gpr, Gpr> {
+ // These convert helpers are called from code generated by the
+ // `cranelift-assembler-x64` crate; each accepts any sized
+ // `{Gpr,Xmm}Mem{Aligned}{N}` newtype via `Clone + Into<RegMem>`, so
+ // the width-tagged types produced by the generator flow through a
+ // single implementation.
+
+ fn convert_gpr_mem_to_assembler_read_gpr_mem<T: Clone + Into<RegMem>>(
+ &self,
+ read: &T,
+ ) -> asm::GprMem<Gpr, Gpr> {
match read.clone().into() {
RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_xmm_mem_to_assembler_read_xmm_mem_aligned(
+ fn convert_xmm_mem_to_assembler_read_xmm_mem_aligned<T: Clone + Into<RegMem>>(
&self,
- read: &XmmMemAligned,
+ read: &T,
) -> asm::XmmMem<Xmm, Gpr> {
match read.clone().into() {
RegMem::Reg { reg } => asm::XmmMem::Xmm(Xmm::new(reg).unwrap()),
@@ -1198,18 +1994,19 @@ impl IsleContext<'_, '_, MInst, X64Backend> {
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_xmm_mem_to_assembler_read_xmm_mem(&self, read: &XmmMem) -> asm::XmmMem<Xmm, Gpr> {
+ fn convert_xmm_mem_to_assembler_read_xmm_mem<T: Clone + Into<RegMem>>(
+ &self,
+ read: &T,
+ ) -> asm::XmmMem<Xmm, Gpr> {
match read.clone().into() {
RegMem::Reg { reg } => asm::XmmMem::Xmm(Xmm::new(reg).unwrap()),
RegMem::Mem { addr } => asm::XmmMem::Mem(addr.into()),
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_xmm_mem_to_assembler_write_xmm_mem(
+ fn convert_xmm_mem_to_assembler_write_xmm_mem<T: Clone + Into<RegMem>>(
&self,
- write: &XmmMem,
+ write: &T,
) -> asm::XmmMem<Writable<Xmm>, Gpr> {
match write.clone().into() {
RegMem::Reg { reg } => asm::XmmMem::Xmm(Writable::from_reg(Xmm::new(reg).unwrap())),
@@ -1217,10 +2014,9 @@ impl IsleContext<'_, '_, MInst, X64Backend> {
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_xmm_mem_to_assembler_write_xmm_mem_aligned(
+ fn convert_xmm_mem_to_assembler_write_xmm_mem_aligned<T: Clone + Into<RegMem>>(
&self,
- write: &XmmMemAligned,
+ write: &T,
) -> asm::XmmMem<Writable<Xmm>, Gpr> {
match write.clone().into() {
RegMem::Reg { reg } => asm::XmmMem::Xmm(Writable::from_reg(Xmm::new(reg).unwrap())),
@@ -1228,10 +2024,9 @@ impl IsleContext<'_, '_, MInst, X64Backend> {
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_gpr_mem_to_assembler_read_write_gpr_mem(
+ fn convert_gpr_mem_to_assembler_read_write_gpr_mem<T: Clone + Into<RegMem>>(
&mut self,
- read: &GprMem,
+ read: &T,
) -> asm::GprMem<PairedGpr, Gpr> {
match read.clone().into() {
RegMem::Reg { reg } => asm::GprMem::Gpr(
@@ -1243,10 +2038,9 @@ impl IsleContext<'_, '_, MInst, X64Backend> {
}
}
- /// Helper used by code generated by the `cranelift-assembler-x64` crate.
- fn convert_gpr_mem_to_assembler_write_gpr_mem(
+ fn convert_gpr_mem_to_assembler_write_gpr_mem<T: Clone + Into<RegMem>>(
&mut self,
- read: &GprMem,
+ read: &T,
) -> asm::GprMem<WritableGpr, Gpr> {
match read.clone().into() {
RegMem::Reg { reg } => asm::GprMem::Gpr(WritableGpr::from_reg(Gpr::new(reg).unwrap())),
diff --git a/cranelift/filetests/filetests/isa/x64/fcvt-from-sint-widen-load.clif b/cranelift/filetests/filetests/isa/x64/fcvt-from-sint-widen-load.clif
new file mode 100644
index 0000000000..f6913b9f75
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/fcvt-from-sint-widen-load.clif
@@ -0,0 +1,32 @@
+test compile precise-output
+target x86_64 sse41
+
+function %fcvt_from_sint_f64x2_widen_load(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.i32x4 v0
+ v2 = swiden_low v1
+ v3 = fcvt_from_sint.f64x2 v2
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; cvtdq2pd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; cvtdq2pd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/scalar-bitselect-load.clif b/cranelift/filetests/filetests/isa/x64/scalar-bitselect-load.clif
new file mode 100644
index 0000000000..687b05c471
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/scalar-bitselect-load.clif
@@ -0,0 +1,93 @@
+test compile precise-output
+target x86_64 sse41
+
+function %bitselect_f32_loads(i64, i64, i64) -> f32 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = load.f32 v0
+ v4 = load.f32 v1
+ v5 = load.f32 v2
+ v6 = bitselect v3, v4, v5
+ return v6
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movss (%rdi), %xmm3
+; movss (%rsi), %xmm2
+; movss (%rdx), %xmm1
+; movdqa %xmm3, %xmm0
+; andps %xmm2, %xmm0
+; uninit %xmm2
+; pcmpeqd %xmm2, %xmm2
+; xorps %xmm2, %xmm3
+; andps %xmm1, %xmm3
+; orps %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movss (%rdi), %xmm3 ; trap: heap_oob
+; movss (%rsi), %xmm2 ; trap: heap_oob
+; movss (%rdx), %xmm1 ; trap: heap_oob
+; movdqa %xmm3, %xmm0
+; andps %xmm2, %xmm0
+; pcmpeqd %xmm2, %xmm2
+; xorps %xmm2, %xmm3
+; andps %xmm1, %xmm3
+; orps %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %bitselect_f64_loads(i64, i64, i64) -> f64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = load.f64 v0
+ v4 = load.f64 v1
+ v5 = load.f64 v2
+ v6 = bitselect v3, v4, v5
+ return v6
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movsd (%rdi), %xmm3
+; movsd (%rsi), %xmm2
+; movsd (%rdx), %xmm1
+; movdqa %xmm3, %xmm0
+; andpd %xmm2, %xmm0
+; uninit %xmm2
+; pcmpeqd %xmm2, %xmm2
+; xorpd %xmm2, %xmm3
+; andpd %xmm1, %xmm3
+; orpd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movsd (%rdi), %xmm3 ; trap: heap_oob
+; movsd (%rsi), %xmm2 ; trap: heap_oob
+; movsd (%rdx), %xmm1 ; trap: heap_oob
+; movdqa %xmm3, %xmm0
+; andpd %xmm2, %xmm0
+; pcmpeqd %xmm2, %xmm2
+; xorpd %xmm2, %xmm3
+; andpd %xmm1, %xmm3
+; orpd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/widen-low-load.clif b/cranelift/filetests/filetests/isa/x64/widen-low-load.clif
new file mode 100644
index 0000000000..4d570cecf9
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/widen-low-load.clif
@@ -0,0 +1,171 @@
+test compile precise-output
+target x86_64 sse41
+
+function %swiden_low_i8x16_load(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i8x16 v0
+ v2 = swiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovsxbw %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovsxbw %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uwiden_low_i8x16_load(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i8x16 v0
+ v2 = uwiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovzxbw %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovzxbw %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %swiden_low_i16x8_load(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i16x8 v0
+ v2 = swiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovsxwd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovsxwd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uwiden_low_i16x8_load(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i16x8 v0
+ v2 = uwiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovzxwd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovzxwd %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %swiden_low_i32x4_load(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i32x4 v0
+ v2 = swiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovsxdq %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovsxdq %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uwiden_low_i32x4_load(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i32x4 v0
+ v2 = uwiden_low v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqu (%rdi), %xmm3
+; pmovzxdq %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqu (%rdi), %xmm3 ; trap: heap_oob
+; pmovzxdq %xmm3, %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/widen-narrow-load.clif b/cranelift/filetests/filetests/isa/x64/widen-narrow-load.clif
new file mode 100644
index 0000000000..4589eaf550
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/widen-narrow-load.clif
@@ -0,0 +1,240 @@
+test compile precise-output
+set opt_level=speed
+target x86_64 has_bmi1
+
+function %andn_i8_load(i64, i8) -> i8 {
+block0(v0: i64, v1: i8):
+ v2 = load.i8 v0
+ v3 = band_not v2, v1
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movzbq (%rdi), %rdi
+; andnl %edi, %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movzbq (%rdi), %rdi ; trap: heap_oob
+; andnl %edi, %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %andn_i16_load(i64, i16) -> i16 {
+block0(v0: i64, v1: i16):
+ v2 = load.i16 v0
+ v3 = band_not v2, v1
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movzwq (%rdi), %rdi
+; andnl %edi, %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movzwq (%rdi), %rdi ; trap: heap_oob
+; andnl %edi, %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %andn_i32_load(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+ v2 = load.i32 v0
+ v3 = band_not v2, v1
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; andnl (%rdi), %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; andnl (%rdi), %esi, %eax ; trap: heap_oob
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %andn_i64_load(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = load.i64 v0
+ v3 = band_not v2, v1
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; andnq (%rdi), %rsi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; andnq (%rdi), %rsi, %rax ; trap: heap_oob
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+;; ---- `select` via `cmove` ---------------------------------------------
+
+;; The consequent (`a` in `select cond a b`) flows to the `c` slot of `cmove`.
+
+function %select_i8_load(i32, i64, i8) -> i8 {
+block0(v0: i32, v1: i64, v2: i8):
+ v3 = load.i8 v1
+ v4 = select.i8 v0, v3, v2
+ return v4
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movzbq (%rsi), %rsi
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movzbq (%rsi), %rsi ; trap: heap_oob
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %select_i16_load(i32, i64, i16) -> i16 {
+block0(v0: i32, v1: i64, v2: i16):
+ v3 = load.i16 v1
+ v4 = select.i16 v0, v3, v2
+ return v4
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movzwq (%rsi), %rsi
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movzwq (%rsi), %rsi ; trap: heap_oob
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel %esi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %select_i32_load(i32, i64, i32) -> i32 {
+block0(v0: i32, v1: i64, v2: i32):
+ v3 = load.i32 v1
+ v4 = select.i32 v0, v3, v2
+ return v4
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel (%rsi), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovnel (%rsi), %eax ; trap: heap_oob
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %select_i64_load(i32, i64, i64) -> i64 {
+block0(v0: i32, v1: i64, v2: i64):
+ v3 = load.i64 v1
+ v4 = select.i64 v0, v3, v2
+ return v4
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovneq (%rsi), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; testl %edi, %edi
+; movq %rdx, %rax
+; cmovneq (%rsi), %rax ; trap: heap_oob
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+