Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: x64: fix i128 shift amounts in several lowering rules. (#14652)

Chris Fallin034285b3b50c
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 6c1e9d1d7c..bc28c4c1d2 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -3947,9 +3947,10 @@
;; can pattern-match to the native `bt` instruction. Note that to have the
;; same semantics this requires that `a` is in a register which forces `bt` to
;; use modulo semantics for the second operand `b`, thus `put_in_gpr` is
-;; manually used.
+;; manually used. The shift amount `b` may be an `i128`; only its low 64 bits
+;; matter, since the amount is taken modulo the type's width.
(rule 1 (is_nonzero_band (ty_32_or_64 ty) a (ishl _ (u64_from_iconst 1) b))
- (CondResult.CC (x64_bt ty (put_in_gpr a) b) (CC.B)))
+ (CondResult.CC (x64_bt ty (put_in_gpr a) (lo_gpr b)) (CC.B)))
;; If a value is and'd one shifted by a variable value that matches `bt` as
;; well.
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 09adefd5ae..765863ba4d 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -467,12 +467,13 @@
;; which is generated to be N bits large. Note that if the index is larger than
;; the bit-width of the type then `bzhi` doesn't have the same semantics as
;; `ishl`, so an `and` instruction is required to mask the index to match the
-;; semantics of Cranelift's `ishl`.
+;; semantics of Cranelift's `ishl`. The index may be an `i128`; only its low 64
+;; bits matter, since it is masked to the type's width anyway.
(rule 16 (lower (band (ty_32_or_64 ty) x y))
(if-let true (has_bmi2))
(if-let (ishl _ (u64_from_iconst 1) index) (val_minus_one y))
- (x64_bzhi ty x (x64_and ty index (RegMemImm.Imm (u32_wrapping_sub (ty_bits ty) 1)))))
+ (x64_bzhi ty x (x64_and ty (lo_gpr index) (RegMemImm.Imm (u32_wrapping_sub (ty_bits ty) 1)))))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/filetests/filetests/isa/x64/i128-shift-amount-bt-bzhi.clif b/cranelift/filetests/filetests/isa/x64/i128-shift-amount-bt-bzhi.clif
new file mode 100644
index 0000000000..43b3eaa7bf
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/i128-shift-amount-bt-bzhi.clif
@@ -0,0 +1,93 @@
+; The `bt` and `bzhi` fusion rules must accept an `i128` shift amount: only its
+; low 64 bits matter, since the amount is taken modulo the type's width.
+test compile precise-output
+target x86_64 has_bmi2
+
+function %bt_i128_amount(i64, i64) -> i8 {
+block0(v0: i64, v9: i64):
+ v1 = uextend.i128 v9
+ v2 = iconst.i64 1
+ v3 = ishl v2, v1
+ v4 = band v0, v3
+ brif v4, block1, block2
+block1:
+ v5 = iconst.i8 1
+ return v5
+block2:
+ v6 = iconst.i8 0
+ return v6
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; uninit %r8
+; xorq %r8, %r8
+; btq %rsi, %rdi
+; jb label2; j label1
+; block1:
+; uninit %rax
+; xorl %eax, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movl $0x1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; xorq %r8, %r8
+; btq %rsi, %rdi
+; jb 0x18
+; block2: ; offset 0x11
+; xorl %eax, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x18
+; movl $1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %bzhi_i128_amount(i64, i64) -> i64 {
+block0(v0: i64, v9: i64):
+ v1 = uextend.i128 v9
+ v2 = iconst.i64 1
+ v3 = ishl v2, v1
+ v4 = isub v3, v2
+ v5 = band v0, v4
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; uninit %r8
+; xorq %r8, %r8
+; andq $0x3f, %rsi
+; bzhiq %rsi, %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; xorq %r8, %r8
+; andq $0x3f, %rsi
+; bzhiq %rsi, %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/runtests/i128-shift-amount-bt-bzhi.clif b/cranelift/filetests/filetests/runtests/i128-shift-amount-bt-bzhi.clif
new file mode 100644
index 0000000000..df7a505007
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/i128-shift-amount-bt-bzhi.clif
@@ -0,0 +1,46 @@
+; `band x, (ishl 1, amt)` (fused into `bt` on x86-64) and
+; `band x, (ishl 1, amt) - 1` (fused into `bzhi` with BMI2) with an `i128`
+; shift amount, including amounts of 2^64 and above, whose high half must be
+; ignored because the amount is taken modulo the type's width.
+test interpret
+test run
+set enable_llvm_abi_extensions=true
+target x86_64
+target x86_64 has_bmi2
+target aarch64
+target s390x
+target riscv64
+target pulley64
+target pulley64be
+
+function %bt(i64, i128) -> i8 {
+block0(v0: i64, v1: i128):
+ v2 = iconst.i64 1
+ v3 = ishl v2, v1
+ v4 = band v0, v3
+ brif v4, block1, block2
+block1:
+ v5 = iconst.i8 1
+ return v5
+block2:
+ v6 = iconst.i8 0
+ return v6
+}
+; run: %bt(0x10, 4) == 1
+; run: %bt(0x10, 5) == 0
+; run: %bt(0x10, 0x44) == 1
+; run: %bt(0x10, 0x1_00000000_00000004) == 1
+; run: %bt(0x10, 0x1_00000000_00000005) == 0
+
+function %bzhi(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+ v2 = iconst.i64 1
+ v3 = ishl v2, v1
+ v4 = isub v3, v2
+ v5 = band v0, v4
+ return v5
+}
+; run: %bzhi(-1, 4) == 0xf
+; run: %bzhi(-1, 0x44) == 0xf
+; run: %bzhi(-1, 0x1_00000000_00000004) == 0xf
+; run: %bzhi(-1, 0) == 0