Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

[Cranelift] add type-aware imm64 constant folding operations (#12826)

Bongjun Jangc6a9443bf69d
diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs
index a5dfce0ad4..84d2438088 100644
--- a/cranelift/codegen/src/isle_prelude.rs
+++ b/cranelift/codegen/src/isle_prelude.rs
@@ -74,6 +74,17 @@ macro_rules! isle_common_prelude_methods {
Some(Imm64::new(result).mask_to_width(type_width))
}
+ #[inline]
+ fn imm64_udiv(&mut self, ty: Type, x: Imm64, y: Imm64) -> Option<Imm64> {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+ let mask = self.ty_mask(ty);
+ let x = (x.bits() as u64) & mask;
+ let y = (y.bits() as u64) & mask;
+ let result = x.checked_div(y)?;
+ Some(Imm64::new(result as i64).mask_to_width(type_width))
+ }
+
#[inline]
fn imm64_srem(&mut self, ty: Type, x: Imm64, y: Imm64) -> Option<Imm64> {
// Sign extend `x` and `y`.
@@ -89,6 +100,125 @@ macro_rules! isle_common_prelude_methods {
Some(Imm64::new(result).mask_to_width(type_width))
}
+ #[inline]
+ fn imm64_urem(&mut self, ty: Type, x: Imm64, y: Imm64) -> Option<Imm64> {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+ let mask = self.ty_mask(ty);
+ let x = (x.bits() as u64) & mask;
+ let y = (y.bits() as u64) & mask;
+ let result = x.checked_rem(y)?;
+ Some(Imm64::new(result as i64).mask_to_width(type_width))
+ }
+
+ #[inline]
+ fn imm64_add(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new(x.bits().wrapping_add(y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_sub(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new(x.bits().wrapping_sub(y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_mul(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new(x.bits().wrapping_mul(y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_and(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new((x.bits() & y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_or(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new((x.bits() | y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_xor(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new((x.bits() ^ y.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_not(&mut self, ty: Type, x: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new((!x.bits()) & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_neg(&mut self, ty: Type, x: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty) as i64;
+ Imm64::new(x.bits().wrapping_neg() & ty_mask)
+ }
+
+ #[inline]
+ fn imm64_abs(&mut self, ty: Type, x: Imm64) -> Option<Imm64> {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+
+ let x = x.sign_extend_from_width(type_width).bits();
+ let shift = 64 - type_width;
+ let min = ((self.ty_smin(ty) as i64) << shift) >> shift;
+ if x == min {
+ return None;
+ }
+
+ Some(Imm64::new(x.abs()).mask_to_width(type_width))
+ }
+
+ #[inline]
+ fn imm64_ilog2(&mut self, ty: Type, x: Imm64) -> Option<Imm64> {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+ let masked = (x.bits() as u64) & self.ty_mask(ty);
+ let result = masked.checked_ilog2()?;
+ Some(Imm64::new(result.into()))
+ }
+
+ #[inline]
+ fn imm64_umin(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty);
+ let x_u = (x.bits() as u64) & ty_mask;
+ let y_u = (y.bits() as u64) & ty_mask;
+ Imm64::new((if x_u <= y_u { x_u } else { y_u }) as i64)
+ }
+
+ #[inline]
+ fn imm64_umax(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let ty_mask = self.ty_mask(ty);
+ let x_u = (x.bits() as u64) & ty_mask;
+ let y_u = (y.bits() as u64) & ty_mask;
+ Imm64::new((if x_u >= y_u { x_u } else { y_u }) as i64)
+ }
+
+ #[inline]
+ fn imm64_smin(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+ let x_s = x.sign_extend_from_width(type_width).bits();
+ let y_s = y.sign_extend_from_width(type_width).bits();
+ let selected = if x_s <= y_s { x_s } else { y_s };
+ Imm64::new(selected).mask_to_width(type_width)
+ }
+
+ #[inline]
+ fn imm64_smax(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
+ let type_width = ty.bits();
+ assert!(type_width <= 64);
+ let x_s = x.sign_extend_from_width(type_width).bits();
+ let y_s = y.sign_extend_from_width(type_width).bits();
+ let selected = if x_s >= y_s { x_s } else { y_s };
+ Imm64::new(selected).mask_to_width(type_width)
+ }
+
#[inline]
fn imm64_shl(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
// Mask off any excess shift bits.
diff --git a/cranelift/codegen/src/opts/arithmetic.isle b/cranelift/codegen/src/opts/arithmetic.isle
index e9f4f66130..d92ffedf7b 100644
--- a/cranelift/codegen/src/opts/arithmetic.isle
+++ b/cranelift/codegen/src/opts/arithmetic.isle
@@ -226,10 +226,10 @@
;; or(x, C) + (-C) --> and(x, ~C)
(rule
(simplify (iadd ty
- (bor ty x (iconst_s ty n))
- (iconst_s ty m)))
- (if-let m (i64_checked_neg n))
- (band ty x (iconst ty (imm64_masked ty (i64_cast_unsigned (i64_not n))))))
+ (bor ty x (iconst ty n))
+ (iconst ty m)))
+ (if-let m (imm64_neg ty n))
+ (band ty x (iconst ty (imm64_not ty n))))
;; (x + y) - (x | y) --> x & y
(rule (simplify (isub ty (iadd ty x y) (bor ty x y))) (band ty x y))
@@ -500,3 +500,6 @@
(rule (simplify (ugt ty x (umax ty y x))) (iconst_u ty 0))
(rule (simplify (ult ty (umax ty x y) x)) (iconst_u ty 0))
(rule (simplify (ult ty (umax ty y x) x)) (iconst_u ty 0))
+
+;; (-X) * C = X * (-C)
+(rule (simplify (imul (fits_in_64 ty) (ineg ty x) (iconst ty y))) (imul ty x (iconst ty (imm64_neg ty y))))
diff --git a/cranelift/codegen/src/opts/cprop.isle b/cranelift/codegen/src/opts/cprop.isle
index 81a6273a08..f40032dfbe 100644
--- a/cranelift/codegen/src/opts/cprop.isle
+++ b/cranelift/codegen/src/opts/cprop.isle
@@ -13,21 +13,21 @@
(rule (simplify
(iadd (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_wrapping_add k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_add ty k1 k2))))
(rule (simplify
(isub (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_wrapping_sub k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_sub ty k1 k2))))
(rule (simplify
(imul (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_wrapping_mul k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_mul ty k1 k2))))
(rule (simplify_skeleton
(sdiv (iconst ty k1)
@@ -42,39 +42,39 @@
(iconst ty d))
(rule (simplify_skeleton
- (udiv (iconst_u ty k1)
- (iconst_u ty k2)))
- (if-let d (u64_checked_div k1 k2))
- (iconst ty (imm64_masked ty d)))
+ (udiv (iconst ty k1)
+ (iconst ty k2)))
+ (if-let d (imm64_udiv ty k1 k2))
+ (iconst ty d))
(rule (simplify_skeleton
- (urem (iconst_u ty k1)
- (iconst_u ty k2)))
- (if-let d (u64_checked_rem k1 k2))
- (iconst ty (imm64_masked ty d)))
+ (urem (iconst ty k1)
+ (iconst ty k2)))
+ (if-let d (imm64_urem ty k1 k2))
+ (iconst ty d))
(rule (simplify
(bor (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_or k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_or ty k1 k2))))
(rule (simplify
(band (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_and k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_and ty k1 k2))))
(rule (simplify
(bxor (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k1))
- (iconst ty (u64_from_imm64 k2))))
- (subsume (iconst ty (imm64_masked ty (u64_xor k1 k2)))))
+ (iconst ty k1)
+ (iconst ty k2)))
+ (subsume (iconst ty (imm64_xor ty k1 k2))))
(rule (simplify
(bnot (fits_in_64 ty)
- (iconst ty (u64_from_imm64 k))))
- (subsume (iconst ty (imm64_masked ty (u64_not k)))))
+ (iconst ty k)))
+ (subsume (iconst ty (imm64_not ty k))))
(rule (simplify (ishl (fits_in_64 ty)
(iconst ty k1)
@@ -166,25 +166,25 @@
;; - (add (sub x k1) k2) -> (add x (sub k2 k1))
;; - (add (sub k1 x) k2) -> (sub (add k1 k2) x)
(rule (simplify (isub ty
- (isub ty x (iconst ty (u64_from_imm64 k1)))
- (iconst ty (u64_from_imm64 k2))))
- (isub ty x (iconst ty (imm64_masked ty (u64_wrapping_add k1 k2)))))
+ (isub ty x (iconst ty k1))
+ (iconst ty k2)))
+ (isub ty x (iconst ty (imm64_add ty k1 k2))))
(rule (simplify (isub ty
- (isub ty (iconst ty (u64_from_imm64 k1)) x)
- (iconst ty (u64_from_imm64 k2))))
- (isub ty (iconst ty (imm64_masked ty (u64_wrapping_sub k1 k2))) x))
+ (isub ty (iconst ty k1) x)
+ (iconst ty k2)))
+ (isub ty (iconst ty (imm64_sub ty k1 k2)) x))
(rule (simplify (isub ty
- (iadd ty x (iconst ty (u64_from_imm64 k1)))
- (iconst ty (u64_from_imm64 k2))))
- (isub ty x (iconst ty (imm64_masked ty (u64_wrapping_sub k2 k1)))))
+ (iadd ty x (iconst ty k1))
+ (iconst ty k2)))
+ (isub ty x (iconst ty (imm64_sub ty k2 k1))))
(rule (simplify (iadd ty
- (isub ty x (iconst ty (u64_from_imm64 k1)))
- (iconst ty (u64_from_imm64 k2))))
- (iadd ty x (iconst ty (imm64_masked ty (u64_wrapping_sub k2 k1)))))
+ (isub ty x (iconst ty k1))
+ (iconst ty k2)))
+ (iadd ty x (iconst ty (imm64_sub ty k2 k1))))
(rule (simplify (iadd ty
- (isub ty (iconst ty (u64_from_imm64 k1)) x)
- (iconst ty (u64_from_imm64 k2))))
- (isub ty (iconst ty (imm64_masked ty (u64_wrapping_add k1 k2))) x))
+ (isub ty (iconst ty k1) x)
+ (iconst ty k2)))
+ (isub ty (iconst ty (imm64_add ty k1 k2)) x))
(rule (simplify
(imul ty (imul ty x k1 @ (iconst ty _)) k2 @ (iconst ty _)))
@@ -231,7 +231,7 @@
(rule (simplify (isub ty x (iconst_s ty k)))
(if-let true (u64_lt (i64_cast_unsigned (i64_wrapping_neg k))
(i64_cast_unsigned k)))
- (iadd ty x (iconst ty (imm64_masked ty (i64_cast_unsigned (i64_wrapping_neg k))))))
+ (iadd ty x (iconst ty (imm64_neg ty (imm64 (i64_cast_unsigned k))))))
;; A splat of a constant can become a direct `vconst` with the appropriate bit
;; pattern.
diff --git a/cranelift/codegen/src/opts/selects.isle b/cranelift/codegen/src/opts/selects.isle
index 77b2b8b943..43b72970af 100644
--- a/cranelift/codegen/src/opts/selects.isle
+++ b/cranelift/codegen/src/opts/selects.isle
@@ -94,10 +94,10 @@
;; fold add-select to select-add
(rule (simplify
- (iadd ty (select ty c (iconst_u ty x) (iconst_u ty y)) (iconst_u ty z)))
+ (iadd ty (select ty c (iconst ty x) (iconst ty y)) (iconst ty z)))
(select ty c
- (iconst ty (imm64_masked ty (u64_wrapping_add x z)))
- (iconst ty (imm64_masked ty (u64_wrapping_add y z)))))
+ (iconst ty (imm64_add ty x z))
+ (iconst ty (imm64_add ty y z))))
(rule (simplify (select ty d a (select ty d _ y))) (select ty d a y))
(rule (simplify (select ty d (select ty d x _) a)) (select ty d x a))
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index 8beecf6cd2..cef9ac2c52 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -73,9 +73,57 @@
(decl pure partial imm64_sdiv (Type Imm64 Imm64) Imm64)
(extern constructor imm64_sdiv imm64_sdiv)
+(decl pure partial imm64_udiv (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_udiv imm64_udiv)
+
(decl pure partial imm64_srem (Type Imm64 Imm64) Imm64)
(extern constructor imm64_srem imm64_srem)
+(decl pure partial imm64_urem (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_urem imm64_urem)
+
+(decl pure imm64_add (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_add imm64_add)
+
+(decl pure imm64_sub (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_sub imm64_sub)
+
+(decl pure imm64_mul (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_mul imm64_mul)
+
+(decl pure imm64_and (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_and imm64_and)
+
+(decl pure imm64_or (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_or imm64_or)
+
+(decl pure imm64_xor (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_xor imm64_xor)
+
+(decl pure imm64_not (Type Imm64) Imm64)
+(extern constructor imm64_not imm64_not)
+
+(decl pure imm64_neg (Type Imm64) Imm64)
+(extern constructor imm64_neg imm64_neg)
+
+(decl pure partial imm64_abs (Type Imm64) Imm64)
+(extern constructor imm64_abs imm64_abs)
+
+(decl pure partial imm64_ilog2 (Type Imm64) Imm64)
+(extern constructor imm64_ilog2 imm64_ilog2)
+
+(decl pure imm64_umin (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_umin imm64_umin)
+
+(decl pure imm64_umax (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_umax imm64_umax)
+
+(decl pure imm64_smin (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_smin imm64_smin)
+
+(decl pure imm64_smax (Type Imm64 Imm64) Imm64)
+(extern constructor imm64_smax imm64_smax)
+
(decl pure imm64_shl (Type Imm64 Imm64) Imm64)
(extern constructor imm64_shl imm64_shl)
diff --git a/cranelift/filetests/filetests/egraph/issue-10409.clif b/cranelift/filetests/filetests/egraph/issue-10409.clif
index 49bd7e6f21..8945ae91d8 100644
--- a/cranelift/filetests/filetests/egraph/issue-10409.clif
+++ b/cranelift/filetests/filetests/egraph/issue-10409.clif
@@ -23,6 +23,6 @@ function u0:1(i64) -> i64 system_v {
return v4
}
-; check: v5 = ireduce.i32 v1
-; next: v6 = uextend.i64 v5
-; next: return v6
+; check: v7 = ireduce.i32 v1
+; next: v8 = uextend.i64 v7
+; next: return v8
diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif
index c0319a9b82..5a08ef1a9d 100644
--- a/cranelift/filetests/filetests/egraph/shifts.clif
+++ b/cranelift/filetests/filetests/egraph/shifts.clif
@@ -86,8 +86,8 @@ block0(v0: i32):
v3 = ishl v2, v1
v4 = ushr v3, v1
return v4
- ; check: v6 = uextend.i64 v0
- ; check: return v6
+ ; check: v7 = uextend.i64 v0
+ ; check: return v7
}
function %sextend_shift_32_64_signed(i32) -> i64 {
@@ -129,8 +129,8 @@ block0(v0: i8):
v3 = ishl v2, v1
v4 = ushr v3, v1
return v4
- ; check: v6 = uextend.i64 v0
- ; check: return v6
+ ; check: v7 = uextend.i64 v0
+ ; check: return v7
}
function %sextend_shift_8_64_signed(i8) -> i64 {
@@ -271,9 +271,9 @@ block0(v0: i32):
v1 = ishl_imm v0, 24
v2 = ushr_imm v1, 24
return v2
- ; check: v5 = ireduce.i8 v0
- ; check: v6 = uextend.i32 v5
- ; check: return v6
+ ; check: v7 = ireduce.i8 v0
+ ; check: v8 = uextend.i32 v7
+ ; check: return v8
}
function %i32_shl_ushr_16_to_ireduce(i32) -> i32 {
@@ -281,9 +281,9 @@ block0(v0: i32):
v1 = ishl_imm v0, 16
v2 = ushr_imm v1, 16
return v2
- ; check: v5 = ireduce.i16 v0
- ; check: v6 = uextend.i32 v5
- ; check: return v6
+ ; check: v7 = ireduce.i16 v0
+ ; check: v8 = uextend.i32 v7
+ ; check: return v8
}
function %i64_shl_ushr_8_to_ireduce(i64) -> i64 {
@@ -291,9 +291,9 @@ block0(v0: i64):
v1 = ishl_imm v0, 56
v2 = ushr_imm v1, 56
return v2
- ; check: v5 = ireduce.i8 v0
- ; check: v6 = uextend.i64 v5
- ; check: return v6
+ ; check: v7 = ireduce.i8 v0
+ ; check: v8 = uextend.i64 v7
+ ; check: return v8
}
function %i64_shl_ushr_16_to_ireduce(i64) -> i64 {
@@ -301,9 +301,9 @@ block0(v0: i64):
v1 = ishl_imm v0, 48
v2 = ushr_imm v1, 48
return v2
- ; check: v5 = ireduce.i16 v0
- ; check: v6 = uextend.i64 v5
- ; check: return v6
+ ; check: v7 = ireduce.i16 v0
+ ; check: v8 = uextend.i64 v7
+ ; check: return v8
}
function %i64_shl_ushr_32_to_ireduce(i64) -> i64 {
@@ -311,9 +311,9 @@ block0(v0: i64):
v1 = ishl_imm v0, 32
v2 = ushr_imm v1, 32
return v2
- ; check: v5 = ireduce.i32 v0
- ; check: v6 = uextend.i64 v5
- ; check: return v6
+ ; check: v7 = ireduce.i32 v0
+ ; check: v8 = uextend.i64 v7
+ ; check: return v8
}
function %ishl_amt_type_ireduce(i8, i16) -> i8 {