Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

aarch64: skip the extend of a scalar comparison's result (#14082)

Nick Fitzgeralde8f0f507e7b4
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index d2a2c8dc04..f3c37ff603 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -1273,6 +1273,16 @@
(if-let mem_op (is_sinkable_inst x))
(load_acquire in flags (sink_atomic_load mem_op)))
+;; Lower scalar comparisons here so we know their upper bits are clear and can
+;; skip the extension.
+;;
+;; This does not hold true for vector comparisons, which instead produce a
+;; per-lane mask of either all ones or zeroes. rather than 0 or 1.
+(rule 1 (lower (uextend (fits_in_64 _) (icmp _ cond x @ (value_type (ty_int _)) y)))
+ (lower_cond_result_bool (emit_icmp cond x y)))
+(rule 1 (lower (uextend (fits_in_64 _) (fcmp _ cond x @ (value_type (ty_scalar_float _)) y)))
+ (lower_cond_result_bool (emit_fcmp cond x y)))
+
;; Conversion to 128-bit needs a zero-extension of the lower bits and the upper
;; bits are all zero.
(attr rule uextend_i128 (tag i128))
@@ -2127,7 +2137,7 @@
(vec_size VectorSize (vector_size ty)))
(value_reg (float_cmp_zero_swap cond rn vec_size))))
-(rule 0 (lower (fcmp _ cond x @ (value_type (ty_scalar_float in_ty)) y))
+(rule 0 (lower (fcmp _ cond x @ (value_type (ty_scalar_float _)) y))
(lower_cond_result_bool (emit_fcmp cond x y)))
(rule -1 (lower (fcmp _ cond x @ (value_type in_ty) y))
@@ -2166,10 +2176,12 @@
(rm Reg (put_in_reg y)))
(vec_cmp rn rm in_ty cond)))
-(rule -2 (lower (icmp _ cond x @ (value_type (ty_int ty)) y))
+(rule -2 (lower (icmp _ cond x @ (value_type (ty_int _)) y))
(lower_cond_result_bool (emit_icmp cond x y)))
;; Helper to put a `CondResult` into a general register.
+;;
+;; Upper bits of the result register are zeroed.
(decl lower_cond_result_bool (CondResult) Reg)
(rule (lower_cond_result_bool (CondResult.Zero reg size))
(value_regs_get (with_flags (cmp_imm size reg (u8_into_imm12 0)) (cset (Cond.Eq))) 0))
diff --git a/cranelift/filetests/filetests/isa/aarch64/uextend-sextend.clif b/cranelift/filetests/filetests/isa/aarch64/uextend-sextend.clif
index 0cb99d3754..af9cc94283 100644
--- a/cranelift/filetests/filetests/isa/aarch64/uextend-sextend.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/uextend-sextend.clif
@@ -194,3 +194,183 @@ block0(v0: i32):
; sxtw x0, w0
; ret
+
+;; A comparison's result is produced by `cset`, which already zeroes the whole
+;; register, so extending it needs no instruction of its own.
+function %f_u_icmp_32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i32 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; subs wzr, w0, w1
+; cset x0, eq
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp w0, w1
+; cset x0, eq
+; ret
+
+;; Any output width, since `cset` defines all 64 bits.
+function %f_u_icmp_64(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp slt v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; subs wzr, w0, w1
+; cset x0, lt
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp w0, w1
+; cset x0, lt
+; ret
+
+function %f_u_icmp_16(i64, i64) -> i16 {
+block0(v0: i64, v1: i64):
+ v2 = icmp ugt v0, v1
+ v3 = uextend.i16 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; subs xzr, x0, x1
+; cset x0, hi
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp x0, x1
+; cset x0, hi
+; ret
+
+;; An `i128` comparison takes several instructions to compute its flags, but
+;; still ends in one `cset`.
+function %f_u_icmp_i128(i128, i128) -> i32 {
+block0(v0: i128, v1: i128):
+ v2 = icmp ult v0, v1
+ v3 = uextend.i32 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; subs xzr, x0, x2
+; sbcs xzr, x1, x3
+; cset x0, lo
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp x0, x2
+; sbcs xzr, x1, x3
+; cset x0, lo
+; ret
+
+;; `fcmp` lowers through the same helper.
+function %f_u_fcmp(f64, f64) -> i32 {
+block0(v0: f64, v1: f64):
+ v2 = fcmp lt v0, v1
+ v3 = uextend.i32 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; fcmp d0, d1
+; cset x0, mi
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; fcmp d0, d1
+; cset x0, mi
+; ret
+
+;; `fcmp one` needs two `cset`s combined with an `orr`, which is also a 32-bit
+;; write of 0 or 1.
+function %f_u_fcmp_one(f32, f32) -> i64 {
+block0(v0: f32, v1: f32):
+ v2 = fcmp one v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; fcmp s0, s1
+; cset x4, mi
+; cset x6, gt
+; orr w0, w4, w6
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; fcmp s0, s1
+; cset x4, mi
+; cset x6, gt
+; orr w0, w4, w6
+; ret
+
+;; Each extended use lowers the comparison independently.
+function %f_u_icmp_multi_use(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i64 v2
+ v4 = uextend.i64 v2
+ v5 = ishl v4, v3
+ return v5
+}
+
+; VCode:
+; block0:
+; subs wzr, w0, w1
+; cset x6, eq
+; subs wzr, w0, w1
+; cset x7, eq
+; lsl x0, x7, x6
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp w0, w1
+; cset x6, eq
+; cmp w0, w1
+; cset x7, eq
+; lsl x0, x7, x6
+; ret
+
+;; Extending to `i128` needs a zero upper half, so it keeps its own rule.
+function %f_u_icmp_i128_out(i32, i32) -> i128 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i128 v2
+ return v3
+}
+
+; VCode:
+; block0:
+; subs wzr, w0, w1
+; cset x6, eq
+; uxtb w0, w6
+; movz x1, #0
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; cmp w0, w1
+; cset x6, eq
+; uxtb w0, w6
+; mov x1, #0
+; ret
diff --git a/cranelift/filetests/filetests/runtests/extend-of-compare.clif b/cranelift/filetests/filetests/runtests/extend-of-compare.clif
new file mode 100644
index 0000000000..c7f1f3cf30
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/extend-of-compare.clif
@@ -0,0 +1,152 @@
+test interpret
+test run
+target x86_64
+target s390x
+target aarch64
+target riscv64
+target pulley64
+target pulley64be
+
+;; A comparison's result is 0 or 1, so extending it must produce exactly 0 or 1
+;; in the wider type -- a backend that skips the extend because its comparison
+;; already wrote the whole register must leave no other bits set.
+
+function %uext_icmp_eq_32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i32 v2
+ return v3
+}
+; run: %uext_icmp_eq_32(0, 0) == 1
+; run: %uext_icmp_eq_32(1, 0) == 0
+; run: %uext_icmp_eq_32(-1, -1) == 1
+; run: %uext_icmp_eq_32(-1, 0x7fffffff) == 0
+
+function %uext_icmp_slt_64(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp slt v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_icmp_slt_64(-1, 0) == 1
+; run: %uext_icmp_slt_64(0, -1) == 0
+; run: %uext_icmp_slt_64(0x80000000, 0x7fffffff) == 1
+; run: %uext_icmp_slt_64(0, 0) == 0
+
+function %uext_icmp_ult_64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = icmp ult v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_icmp_ult_64(0, -1) == 1
+; run: %uext_icmp_ult_64(-1, 0) == 0
+; run: %uext_icmp_ult_64(-1, -1) == 0
+
+;; A narrow output width, and one narrower than the comparison's operands.
+function %uext_icmp_16(i64, i64) -> i16 {
+block0(v0: i64, v1: i64):
+ v2 = icmp ugt v0, v1
+ v3 = uextend.i16 v2
+ return v3
+}
+; run: %uext_icmp_16(1, 0) == 1
+; run: %uext_icmp_16(0, 1) == 0
+
+;; Narrow comparison operands: the `i8` inputs must not leak their upper bits
+;; into the result.
+function %uext_icmp_i8_operands(i8, i8) -> i64 {
+block0(v0: i8, v1: i8):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_icmp_i8_operands(-1, -1) == 1
+; run: %uext_icmp_i8_operands(-1, 0xff) == 1
+; run: %uext_icmp_i8_operands(0x7f, 0x80) == 0
+
+;; The extended value has to be usable as an integer, not just compared against
+;; 0 or 1: an `imul` by it exposes any stray upper bit.
+function %uext_icmp_arith(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i64 v2
+ v4 = iconst.i64 0x1234_5678_9abc_def0
+ v5 = imul v3, v4
+ return v5
+}
+; run: %uext_icmp_arith(0, 0) == 0x123456789abcdef0
+; run: %uext_icmp_arith(0, 1) == 0
+
+;; Several extends of one comparison.
+function %uext_icmp_multi_use(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = uextend.i64 v2
+ v4 = uextend.i64 v2
+ v5 = iadd v3, v4
+ return v5
+}
+; run: %uext_icmp_multi_use(7, 7) == 2
+; run: %uext_icmp_multi_use(7, 8) == 0
+
+;; Sign-extending a comparison is the same as zero-extending it, which is a
+;; mid-end rewrite; check it still holds end to end.
+function %sext_icmp(i32, i32) -> i64 {
+block0(v0: i32, v1: i32):
+ v2 = icmp eq v0, v1
+ v3 = sextend.i64 v2
+ return v3
+}
+; run: %sext_icmp(5, 5) == 1
+; run: %sext_icmp(5, 6) == 0
+
+;; Floating point, where the unordered cases are what a 0-or-1 assumption is
+;; most likely to get wrong.
+function %uext_fcmp_lt(f64, f64) -> i64 {
+block0(v0: f64, v1: f64):
+ v2 = fcmp lt v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_fcmp_lt(0x0.0, 0x1.0) == 1
+; run: %uext_fcmp_lt(0x1.0, 0x0.0) == 0
+; run: %uext_fcmp_lt(-0x0.0, 0x0.0) == 0
+; run: %uext_fcmp_lt(+NaN, 0x0.0) == 0
+; run: %uext_fcmp_lt(0x0.0, +NaN) == 0
+
+;; `ne` is true for NaN.
+function %uext_fcmp_ne(f32, f32) -> i64 {
+block0(v0: f32, v1: f32):
+ v2 = fcmp ne v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_fcmp_ne(0x1.0, 0x1.0) == 0
+; run: %uext_fcmp_ne(0x1.0, 0x2.0) == 1
+; run: %uext_fcmp_ne(+NaN, +NaN) == 1
+; run: %uext_fcmp_ne(-0x0.0, 0x0.0) == 0
+
+;; `one` and `ueq` are the two conditions aarch64 builds from a pair of `cset`s
+;; combined with `orr`.
+function %uext_fcmp_one(f32, f32) -> i64 {
+block0(v0: f32, v1: f32):
+ v2 = fcmp one v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_fcmp_one(0x1.0, 0x2.0) == 1
+; run: %uext_fcmp_one(0x1.0, 0x1.0) == 0
+; run: %uext_fcmp_one(+NaN, 0x1.0) == 0
+; run: %uext_fcmp_one(+NaN, +NaN) == 0
+
+function %uext_fcmp_ueq(f64, f64) -> i64 {
+block0(v0: f64, v1: f64):
+ v2 = fcmp ueq v0, v1
+ v3 = uextend.i64 v2
+ return v3
+}
+; run: %uext_fcmp_ueq(0x1.0, 0x1.0) == 1
+; run: %uext_fcmp_ueq(0x1.0, 0x2.0) == 0
+; run: %uext_fcmp_ueq(+NaN, 0x1.0) == 1
+; run: %uext_fcmp_ueq(-0x0.0, 0x0.0) == 1