Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

S390x: implement arith overflow (#12523)

Jimmy Brissond06fd9e0fa93
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index b504959772..479acf1131 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -1115,8 +1115,10 @@
(SubLogical64)
(SubLogical64Ext32)
(Mul32)
+ (Mul32CC)
(Mul32Ext16)
(Mul64)
+ (Mul64CC)
(Mul64Ext16)
(Mul64Ext32)
(And32)
@@ -1205,11 +1207,13 @@
(Add32x4)
(Add64x2)
(Add128)
+ (Add128Cout)
(Sub8x16)
(Sub16x8)
(Sub32x4)
(Sub64x2)
(Sub128)
+ (Sub128Cout)
;; Multiplication
(Mul8x16)
(Mul16x8)
@@ -3872,6 +3876,31 @@
(intcc_as_cond (IntCC.SignedLessThan)))))
(select_bool_reg $I64 l_cond l_bound ub)))
+;; Helpers for generating integer arithmetic instructions with overflow outputs ;;;
+
+;; For fit-in-16 bit integers, we shift them into the most significant positions of their
+;; 32-bit registers, use the codition codes for the overflow, and shift back into the
+;; expected least-significant position to generate the result
+(decl type_shift_up (Type) u8)
+(rule (type_shift_up $I8) 24)
+(rule (type_shift_up $I16) 16)
+
+(decl overflow_and_result_from_shifted (Type ALUOp Reg Reg Cond) InstOutput)
+(rule (overflow_and_result_from_shifted (fits_in_16 ty) op x y cond)
+ (let ((x_shifted Reg (lshl_imm $I32 x (type_shift_up ty)))
+ (y_shifted Reg (lshl_imm $I32 y (type_shift_up ty)))
+ (producer ProducesFlags (alu_rrr_with_flags_paired ty op x_shifted y_shifted))
+ (overflow Reg (lower_bool $I8 (bool (produces_flags_ignore producer) cond)))
+ (out Reg (lshr_imm $I32 (produces_flags_get_reg producer) (type_shift_up ty))))
+ (output_pair out overflow)))
+
+;; Generate the paired overflow result from the generated condition codes
+(decl overflow_and_result_from_producer (ProducesFlags Cond) InstOutput)
+(rule (overflow_and_result_from_producer producer cond)
+ (output_pair
+ (produces_flags_get_reg producer)
+ (lower_bool $I8 (bool (produces_flags_ignore producer) cond))))
+
;; Helpers for generating `add` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3892,6 +3921,10 @@
(decl add_reg (Type Reg Reg) Reg)
(rule (add_reg ty x y) (alu_rrr ty (aluop_add ty) x y))
+(decl add_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (add_reg_with_flags_paired ty x y)
+ (alu_rrr_with_flags_paired ty (aluop_add ty) x y))
+
(decl add_reg_sext32 (Type Reg Reg) Reg)
(rule (add_reg_sext32 ty x y) (alu_rr ty (aluop_add_sext32 ty) x y))
@@ -3965,6 +3998,11 @@
(rule (add_logical_mem_zext32_with_flags_paired ty x y)
(alu_rx_with_flags_paired ty (aluop_add_logical_zext32 ty) x y))
+(decl vecop_add_logical_cout (Type) VecBinaryOp)
+(rule (vecop_add_logical_cout $I128) (VecBinaryOp.Add128Cout))
+
+(decl vec_add_logical_cout (Type Reg Reg) Reg)
+(rule (vec_add_logical_cout ty x y) (vec_rrr ty (vecop_add_logical_cout ty) x y))
;; Helpers for generating `sub` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3985,6 +4023,10 @@
(decl sub_reg (Type Reg Reg) Reg)
(rule (sub_reg ty x y) (alu_rrr ty (aluop_sub ty) x y))
+(decl sub_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (sub_reg_with_flags_paired ty x y)
+ (alu_rrr_with_flags_paired ty (aluop_sub ty) x y))
+
(decl sub_reg_sext32 (Type Reg Reg) Reg)
(rule (sub_reg_sext32 ty x y) (alu_rr ty (aluop_sub_sext32 ty) x y))
@@ -4020,6 +4062,10 @@
(decl sub_logical_reg (Type Reg Reg) Reg)
(rule (sub_logical_reg ty x y) (alu_rrr ty (aluop_sub_logical ty) x y))
+(decl sub_logical_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (sub_logical_reg_with_flags_paired ty x y)
+ (alu_rrr_with_flags_paired ty (aluop_sub_logical ty) x y))
+
(decl sub_logical_reg_zext32 (Type Reg Reg) Reg)
(rule (sub_logical_reg_zext32 ty x y) (alu_rr ty (aluop_sub_logical_zext32 ty) x y))
@@ -4032,6 +4078,11 @@
(decl sub_logical_mem_zext32 (Type Reg MemArg) Reg)
(rule (sub_logical_mem_zext32 ty x y) (alu_rx ty (aluop_sub_logical ty) x y))
+(decl vecop_sub_logical_cout (Type) VecBinaryOp)
+(rule (vecop_sub_logical_cout $I128) (VecBinaryOp.Sub128Cout))
+
+(decl vec_sub_logical_cout (Type Reg Reg) Reg)
+(rule (vec_sub_logical_cout ty x y) (vec_rrr ty (vecop_sub_logical_cout ty) x y))
;; Helpers for generating `mul` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -4041,6 +4092,10 @@
(rule (aluop_mul $I32) (ALUOp.Mul32))
(rule (aluop_mul $I64) (ALUOp.Mul64))
+(decl aluop_mul_cc (Type) ALUOp)
+(rule (aluop_mul_cc $I32) (ALUOp.Mul32CC))
+(rule (aluop_mul_cc $I64) (ALUOp.Mul64CC))
+
(decl aluop_mul_sext16 (Type) ALUOp)
(rule (aluop_mul_sext16 $I16) (ALUOp.Mul32Ext16))
(rule (aluop_mul_sext16 $I32) (ALUOp.Mul32Ext16))
@@ -4052,6 +4107,10 @@
(decl mul_reg (Type Reg Reg) Reg)
(rule (mul_reg ty x y) (alu_rrr ty (aluop_mul ty) x y))
+(decl mul_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (mul_reg_with_flags_paired ty x y)
+ (alu_rrr_with_flags_paired ty (aluop_mul_cc ty) x y))
+
(decl mul_reg_sext32 (Type Reg Reg) Reg)
(rule (mul_reg_sext32 ty x y) (alu_rr ty (aluop_mul_sext32 ty) x y))
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index 9132c12a69..5e2e42c186 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -1516,7 +1516,9 @@ impl Inst {
ALUOp::SubLogical32 => (0xb9fb, true), // SLRK
ALUOp::SubLogical64 => (0xb9eb, true), // SLGRK
ALUOp::Mul32 => (0xb9fd, true), // MSRKC
+ ALUOp::Mul32CC => (0xb9fd, false), // MSRKC
ALUOp::Mul64 => (0xb9ed, true), // MSGRKC
+ ALUOp::Mul64CC => (0xb9ed, false), // MSGRKC
ALUOp::And32 => (0xb9f4, true), // NRK
ALUOp::And64 => (0xb9e4, true), // NGRK
ALUOp::Orr32 => (0xb9f6, true), // ORK
@@ -2745,11 +2747,13 @@ impl Inst {
VecBinaryOp::Add32x4 => (0xe7f3, 2), // VAF
VecBinaryOp::Add64x2 => (0xe7f3, 3), // VAG
VecBinaryOp::Add128 => (0xe7f3, 4), // VAQ
+ VecBinaryOp::Add128Cout => (0xe7f1, 4), // VACCQ
VecBinaryOp::Sub8x16 => (0xe7f7, 0), // VSB
VecBinaryOp::Sub16x8 => (0xe7f7, 1), // VSH
VecBinaryOp::Sub32x4 => (0xe7f7, 2), // VSF
VecBinaryOp::Sub64x2 => (0xe7f7, 3), // VSG
VecBinaryOp::Sub128 => (0xe7f7, 4), // VSQ
+ VecBinaryOp::Sub128Cout => (0xe7f5, 4), // VSCBI
VecBinaryOp::Mul8x16 => (0xe7a2, 0), // VMLB
VecBinaryOp::Mul16x8 => (0xe7a2, 1), // VMLHW
VecBinaryOp::Mul32x4 => (0xe7a2, 2), // VMLF
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index bf3a0a6dcf..52ce2d2603 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -1314,7 +1314,9 @@ impl Inst {
ALUOp::SubLogical32 => ("slrk", true),
ALUOp::SubLogical64 => ("slgrk", true),
ALUOp::Mul32 => ("msrkc", true),
+ ALUOp::Mul32CC => ("msrkc", false),
ALUOp::Mul64 => ("msgrkc", true),
+ ALUOp::Mul64CC => ("msgrkc", false),
ALUOp::And32 => ("nrk", true),
ALUOp::And64 => ("ngrk", true),
ALUOp::Orr32 => ("ork", true),
@@ -2529,11 +2531,13 @@ impl Inst {
VecBinaryOp::Add32x4 => "vaf",
VecBinaryOp::Add64x2 => "vag",
VecBinaryOp::Add128 => "vaq",
+ VecBinaryOp::Add128Cout => "vaccq",
VecBinaryOp::Sub8x16 => "vsb",
VecBinaryOp::Sub16x8 => "vsh",
VecBinaryOp::Sub32x4 => "vsf",
VecBinaryOp::Sub64x2 => "vsg",
VecBinaryOp::Sub128 => "vsq",
+ VecBinaryOp::Sub128Cout => "vscbiq",
VecBinaryOp::Mul8x16 => "vmlb",
VecBinaryOp::Mul16x8 => "vmlhw",
VecBinaryOp::Mul32x4 => "vmlf",
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 500636cc47..7ad613c3f2 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -4326,12 +4326,116 @@
;;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule 0 (lower (has_type (ty_32_or_64 ty) (uadd_overflow _ x y)))
- (let ((sum Reg (add_reg ty x y))
- (overflow Reg
- (lower_bool $I8
- (bool (icmpu_reg ty sum x) (intcc_as_cond (IntCC.UnsignedLessThan))))))
- (output_pair sum overflow)))
+(rule 1 (lower (uadd_overflow (fits_in_16 ty) x y))
+ (overflow_and_result_from_shifted ty (aluop_add_logical $I32) x y (mask_as_cond 3)))
+
+(rule 0 (lower (uadd_overflow (ty_32_or_64 ty) x y))
+ (overflow_and_result_from_producer (add_logical_reg_with_flags_paired ty x y) (mask_as_cond 3)))
+
+(rule 2 (lower (uadd_overflow $I128 x y))
+ (output_pair
+ (vec_add $I128 x y)
+ (vec_extract_lane $I64X2 (vec_add_logical_cout $I128 x y) 1 (zero_reg))))
+
+;;;; Rules for `usub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Note: s390x stores and computes the borrow bit as a 0 when an overflow is present
+;; so all of the conditions and computed borrows are inverted
+
+(rule 1 (lower (usub_overflow (fits_in_16 ty) x y))
+ (overflow_and_result_from_shifted ty (aluop_sub_logical $I32) x y
+ (invert_cond (mask_as_cond 3))))
+
+(rule 0 (lower (usub_overflow (ty_32_or_64 ty) x y))
+ (overflow_and_result_from_producer
+ (sub_logical_reg_with_flags_paired ty x y)
+ (invert_cond (mask_as_cond 3))))
+
+(rule 2 (lower (usub_overflow $I128 x y))
+ (output_pair
+ (vec_sub $I128 x y)
+ (xor_uimm32shifted $I8
+ (vec_extract_lane $I64X2 (vec_sub_logical_cout $I128 x y) 1 (zero_reg))
+ (uimm32shifted 1 0))))
+
+;;;; Rules for `sadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule 1 (lower (sadd_overflow (fits_in_16 ty) x y))
+ (overflow_and_result_from_shifted ty (aluop_add $I32) x y (mask_as_cond 1)))
+
+(rule 0 (lower (sadd_overflow (ty_32_or_64 ty) x y))
+ (overflow_and_result_from_producer (add_reg_with_flags_paired ty x y) (mask_as_cond 1)))
+
+(rule 2 (lower (sadd_overflow $I128 x y))
+ (let ((res Reg (vec_add $I128 x y))
+ (res_hi Reg (vec_extract_lane $I64X2 res 0 (zero_reg)))
+ (x_hi Reg (vec_extract_lane $I64X2 x 0 (zero_reg)))
+ (y_hi Reg (vec_extract_lane $I64X2 y 0 (zero_reg)))
+ (of_in_sign Reg
+ (and_reg $I64
+ (xor_reg $I64 x_hi res_hi)
+ (xor_reg $I64 y_hi res_hi))))
+ (output_pair res (lshr_imm $I64 of_in_sign 63))))
+
+;;;; Rules for `ssub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule 1 (lower (ssub_overflow (fits_in_16 ty) x y))
+ (overflow_and_result_from_shifted ty (aluop_sub $I32) x y (mask_as_cond 1)))
+
+;; Use flags generated by the add instruction to handle overflow
+(rule 0 (lower (ssub_overflow (ty_32_or_64 ty) x y))
+ (overflow_and_result_from_producer (sub_reg_with_flags_paired ty x y) (mask_as_cond 1)))
+
+(rule 2 (lower (ssub_overflow $I128 x y))
+ (let ((res Reg (vec_sub $I128 x y))
+ (res_hi Reg (vec_extract_lane $I64X2 res 0 (zero_reg)))
+ (x_hi Reg (vec_extract_lane $I64X2 x 0 (zero_reg)))
+ (y_hi Reg (vec_extract_lane $I64X2 y 0 (zero_reg)))
+ (of_in_sign Reg
+ (and_reg $I64
+ (xor_reg $I64 x_hi res_hi)
+ (xor_reg $I64 x_hi y_hi))))
+ (output_pair res (lshr_imm $I64 of_in_sign 63))))
+
+;;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule 1 (lower (umul_overflow (fits_in_32 ty) x y))
+ (let ((result Reg (mul_reg $I64 (zext64_reg ty y) (zext64_reg ty x)))
+ (of Reg (lower_bool $I8 (invert_bool (bool
+ (icmps_simm16 $I32 (lshr_imm $I64 result (ty_bits ty)) 0)
+ (intcc_as_cond (IntCC.Equal)))))))
+ (output_pair result of)))
+
+(rule 0 (lower (umul_overflow ty @ $I64 x y))
+ (let ((mul_out RegPair (umul_wide y x))
+ (result Reg (regpair_lo mul_out))
+ (of Reg (lower_bool $I8 (invert_bool (bool
+ (icmps_simm16 ty (regpair_hi mul_out) 0)
+ (intcc_as_cond (IntCC.Equal)))))))
+ (output_pair result of)))
+
+;;;; Rules for `smul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; For fit-in-16 bit integers, we shift just the rhs into the most
+;; significant positions of its 32-bit register, use the codition
+;; codes for the overflow, and shifting back into the expected
+;; least-significant position to generate the result.
+(rule 1 (lower (smul_overflow (fits_in_16 ty) x y))
+ (let ((y_ext Reg (sext32_reg ty y))
+ (x_shifted Reg (lshl_imm $I32 x (type_shift_up ty)))
+ (producer ProducesFlags
+ (mul_reg_with_flags_paired $I32 x_shifted y_ext))
+ (overflow Reg (lower_bool $I8 (bool
+ (produces_flags_ignore producer)
+ (mask_as_cond 1))))
+ (out Reg (lshr_imm $I32
+ (produces_flags_get_reg producer)
+ (type_shift_up ty))))
+ (output_pair out overflow)))
+
+;; Use flags generated by the add instruction to handle overflow
+(rule 0 (lower (smul_overflow (ty_32_or_64 ty) x y))
+ (overflow_and_result_from_producer (mul_reg_with_flags_paired ty x y) (mask_as_cond 1)))
;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/filetests/filetests/isa/s390x/sadd_overflow.clif b/cranelift/filetests/filetests/isa/s390x/sadd_overflow.clif
new file mode 100644
index 0000000000..cbd6e83cb2
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/sadd_overflow.clif
@@ -0,0 +1,145 @@
+test compile precise-output
+;; needed for i128
+set enable_multi_ret_implicit_sret
+target s390x
+
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = sadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 24
+; lgr %r2, %r4
+; sllk %r5, %r2, 24
+; ark %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 24
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x18
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x18
+; ark %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x18
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = sadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 16
+; lgr %r2, %r4
+; sllk %r5, %r2, 16
+; ark %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 16
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x10
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x10
+; ark %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x10
+; br %r14
+
+function %f2(i32, i32) -> i32, i8 {
+block0(v0: i32, v1: i32):
+ v2, v3 = sadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; ar %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; ar %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
+function %f4(i64, i64) -> i64, i8 {
+block0(v0: i64, v1: i64):
+ v2, v3 = sadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; agr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; agr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
+function %f5(i128, i128) -> i128, i8 {
+block0(v0: i128, v1: i128):
+ v2, v3 = sadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vaq %v6, %v1, %v3
+; lgdr %r3, %f6
+; lgdr %r2, %f1
+; lgdr %r4, %f3
+; xgr %r2, %r3
+; xgrk %r3, %r4, %r3
+; ngr %r2, %r3
+; srlg %r2, %r2, 63
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vaq %v6, %v1, %v3
+; lgdr %r3, %f6
+; lgdr %r2, %f1
+; lgdr %r4, %f3
+; xgr %r2, %r3
+; xgrk %r3, %r4, %r3
+; ngr %r2, %r3
+; srlg %r2, %r2, 0x3f
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/smul_overflow.clif b/cranelift/filetests/filetests/isa/s390x/smul_overflow.clif
new file mode 100644
index 0000000000..b04192f17a
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/smul_overflow.clif
@@ -0,0 +1,95 @@
+test compile precise-output
+target s390x
+
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = smul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lbr %r3, %r3
+; sllk %r5, %r2, 24
+; msrkc %r2, %r5, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 24
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lbr %r3, %r3
+; sllk %r5, %r2, 0x18
+; msrkc %r2, %r5, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x18
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = smul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lhr %r3, %r3
+; sllk %r5, %r2, 16
+; msrkc %r2, %r5, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 16
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lhr %r3, %r3
+; sllk %r5, %r2, 0x10
+; msrkc %r2, %r5, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x10
+; br %r14
+
+function %f2(i32, i32) -> i32, i8 {
+block0(v0: i32, v1: i32):
+ v2, v3 = smul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; msrkc %r2, %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; msrkc %r2, %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
+function %f4(i64, i64) -> i64, i8 {
+block0(v0: i64, v1: i64):
+ v2, v3 = smul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; msgrkc %r2, %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; msgrkc %r2, %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/ssub_overflow.clif b/cranelift/filetests/filetests/isa/s390x/ssub_overflow.clif
new file mode 100644
index 0000000000..fa4af1734c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/ssub_overflow.clif
@@ -0,0 +1,149 @@
+test compile precise-output
+set enable_multi_ret_implicit_sret
+target s390x
+
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = ssub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 24
+; lgr %r2, %r4
+; sllk %r5, %r2, 24
+; srk %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 24
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x18
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x18
+; srk %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x18
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = ssub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 16
+; lgr %r2, %r4
+; sllk %r5, %r2, 16
+; srk %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 16
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x10
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x10
+; srk %r2, %r3, %r5
+; lhi %r3, 0
+; lochio %r3, 1
+; srlk %r2, %r2, 0x10
+; br %r14
+
+function %f2(i32, i32) -> i32, i8 {
+block0(v0: i32, v1: i32):
+ v2, v3 = ssub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; sr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; sr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
+function %f4(i64, i64) -> i64, i8 {
+block0(v0: i64, v1: i64):
+ v2, v3 = ssub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; sgr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; sgr %r2, %r3
+; lhi %r3, 0
+; lochio %r3, 1
+; br %r14
+
+function %f5(i128, i128) -> i128, i8 {
+block0(v0: i128, v1: i128):
+ v2, v3 = ssub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; stmg %r14, %r15, 112(%r15)
+; block0:
+; lgr %r14, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vsq %v6, %v1, %v3
+; lgdr %r4, %f6
+; lgdr %r3, %f1
+; lgdr %r5, %f3
+; xgrk %r2, %r3, %r4
+; xgr %r3, %r5
+; ngr %r2, %r3
+; srlg %r2, %r2, 63
+; lgr %r3, %r14
+; vst %v6, 0(%r3)
+; lmg %r14, %r15, 112(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r14, %r15, 0x70(%r15)
+; block1: ; offset 0x6
+; lgr %r14, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vsq %v6, %v1, %v3
+; lgdr %r4, %f6
+; lgdr %r3, %f1
+; lgdr %r5, %f3
+; xgrk %r2, %r3, %r4
+; xgr %r3, %r5
+; ngr %r2, %r3
+; srlg %r2, %r2, 0x3f
+; lgr %r3, %r14
+; vst %v6, 0(%r3)
+; lmg %r14, %r15, 0x70(%r15)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/uadd_overflow.clif b/cranelift/filetests/filetests/isa/s390x/uadd_overflow.clif
index 6668bfb011..ed5a6c3182 100644
--- a/cranelift/filetests/filetests/isa/s390x/uadd_overflow.clif
+++ b/cranelift/filetests/filetests/isa/s390x/uadd_overflow.clif
@@ -1,6 +1,68 @@
test compile precise-output
+;; needed for i128
+set enable_multi_ret_implicit_sret
target s390x
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = uadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 24
+; lgr %r2, %r4
+; sllk %r5, %r2, 24
+; alrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochinle %r3, 1
+; srlk %r2, %r2, 24
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x18
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x18
+; alrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochinle %r3, 1
+; srlk %r2, %r2, 0x18
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = uadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 16
+; lgr %r2, %r4
+; sllk %r5, %r2, 16
+; alrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochinle %r3, 1
+; srlk %r2, %r2, 16
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x10
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x10
+; alrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochinle %r3, 1
+; srlk %r2, %r2, 0x10
+; br %r14
+
function %f2(i32, i32) -> i32, i8 {
block0(v0: i32, v1: i32):
v2, v3 = uadd_overflow v0, v1
@@ -9,20 +71,16 @@ block0(v0: i32, v1: i32):
; VCode:
; block0:
-; ark %r4, %r2, %r3
-; clr %r4, %r2
+; alr %r2, %r3
; lhi %r3, 0
-; lochil %r3, 1
-; lgr %r2, %r4
+; lochinle %r3, 1
; br %r14
;
; Disassembled:
; block0: ; offset 0x0
-; ark %r4, %r2, %r3
-; clr %r4, %r2
+; alr %r2, %r3
; lhi %r3, 0
-; lochil %r3, 1
-; lgr %r2, %r4
+; lochinle %r3, 1
; br %r14
function %f4(i64, i64) -> i64, i8 {
@@ -33,19 +91,45 @@ block0(v0: i64, v1: i64):
; VCode:
; block0:
-; agrk %r4, %r2, %r3
-; clgr %r4, %r2
+; algr %r2, %r3
; lhi %r3, 0
-; lochil %r3, 1
-; lgr %r2, %r4
+; lochinle %r3, 1
; br %r14
;
; Disassembled:
; block0: ; offset 0x0
-; agrk %r4, %r2, %r3
-; clgr %r4, %r2
+; algr %r2, %r3
; lhi %r3, 0
-; lochil %r3, 1
-; lgr %r2, %r4
+; lochinle %r3, 1
+; br %r14
+
+function %f5(i128, i128) -> i128, i8 {
+block0(v0: i128, v1: i128):
+ v2, v3 = uadd_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vaq %v6, %v1, %v3
+; vaccq %v16, %v1, %v3
+; vlgvg %r2, %v16, 1
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vaq %v6, %v1, %v3
+; vaccq %v16, %v1, %v3
+; vlgvg %r2, %v16, 1
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
; br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/umul_overflow.clif b/cranelift/filetests/filetests/isa/s390x/umul_overflow.clif
new file mode 100644
index 0000000000..888fd8b700
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/umul_overflow.clif
@@ -0,0 +1,113 @@
+test compile precise-output
+target s390x
+
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = umul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; llgcr %r3, %r3
+; llgcr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 8
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; llgcr %r3, %r3
+; llgcr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 8
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = umul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; llghr %r3, %r3
+; llghr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 16
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; llghr %r3, %r3
+; llghr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 0x10
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+
+function %f2(i32, i32) -> i32, i8 {
+block0(v0: i32, v1: i32):
+ v2, v3 = umul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; llgfr %r3, %r3
+; llgfr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 32
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; llgfr %r3, %r3
+; llgfr %r5, %r2
+; msgrkc %r2, %r3, %r5
+; srlg %r3, %r2, 0x20
+; chi %r3, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; br %r14
+
+function %f4(i64, i64) -> i64, i8 {
+block0(v0: i64, v1: i64):
+ v2, v3 = umul_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; mlgr %r2, %r2
+; lgr %r4, %r3
+; cghi %r2, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; lgr %r2, %r4
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; mlgr %r2, %r2
+; lgr %r4, %r3
+; cghi %r2, 0
+; lhi %r3, 0
+; lochine %r3, 1
+; lgr %r2, %r4
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/usub_overflow.clif b/cranelift/filetests/filetests/isa/s390x/usub_overflow.clif
new file mode 100644
index 0000000000..26228bd022
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/usub_overflow.clif
@@ -0,0 +1,136 @@
+test compile precise-output
+set enable_multi_ret_implicit_sret
+target s390x
+
+function %f2(i8, i8) -> i8, i8 {
+block0(v0: i8, v1: i8):
+ v2, v3 = usub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 24
+; lgr %r2, %r4
+; sllk %r5, %r2, 24
+; slrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochile %r3, 1
+; srlk %r2, %r2, 24
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x18
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x18
+; slrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochile %r3, 1
+; srlk %r2, %r2, 0x18
+; br %r14
+
+function %f2(i16, i16) -> i16, i8 {
+block0(v0: i16, v1: i16):
+ v2, v3 = usub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r4, %r3
+; sllk %r3, %r2, 16
+; lgr %r2, %r4
+; sllk %r5, %r2, 16
+; slrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochile %r3, 1
+; srlk %r2, %r2, 16
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r4, %r3
+; sllk %r3, %r2, 0x10
+; lgr %r2, %r4
+; sllk %r5, %r2, 0x10
+; slrk %r2, %r3, %r5
+; lhi %r3, 0
+; lochile %r3, 1
+; srlk %r2, %r2, 0x10
+; br %r14
+
+function %f2(i32, i32) -> i32, i8 {
+block0(v0: i32, v1: i32):
+ v2, v3 = usub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; slr %r2, %r3
+; lhi %r3, 0
+; lochile %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; slr %r2, %r3
+; lhi %r3, 0
+; lochile %r3, 1
+; br %r14
+
+function %f4(i64, i64) -> i64, i8 {
+block0(v0: i64, v1: i64):
+ v2, v3 = usub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; slgr %r2, %r3
+; lhi %r3, 0
+; lochile %r3, 1
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; slgr %r2, %r3
+; lhi %r3, 0
+; lochile %r3, 1
+; br %r14
+
+function %f5(i128, i128) -> i128, i8 {
+block0(v0: i128, v1: i128):
+ v2, v3 = usub_overflow v0, v1
+ return v2, v3
+}
+
+; VCode:
+; block0:
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vsq %v6, %v1, %v3
+; vscbiq %v16, %v1, %v3
+; vlgvg %r2, %v16, 1
+; xilf %r2, 1
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; lgr %r5, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; vsq %v6, %v1, %v3
+; vscbiq %v16, %v1, %v3
+; vlgvg %r2, %v16, 1
+; xilf %r2, 1
+; lgr %r3, %r5
+; vst %v6, 0(%r3)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/runtests/sadd_overflow.clif b/cranelift/filetests/filetests/runtests/sadd_overflow.clif
index 69c35f30d3..ca538e6b2e 100644
--- a/cranelift/filetests/filetests/runtests/sadd_overflow.clif
+++ b/cranelift/filetests/filetests/runtests/sadd_overflow.clif
@@ -3,6 +3,7 @@ test run
set enable_llvm_abi_extensions=true
target aarch64
set enable_multi_ret_implicit_sret
+target s390x
target x86_64
function %saddof_i128(i128, i128) -> i128, i8 {
diff --git a/cranelift/filetests/filetests/runtests/smul_overflow.clif b/cranelift/filetests/filetests/runtests/smul_overflow.clif
index b70b8ca65a..8544bca714 100644
--- a/cranelift/filetests/filetests/runtests/smul_overflow.clif
+++ b/cranelift/filetests/filetests/runtests/smul_overflow.clif
@@ -2,6 +2,7 @@ test interpret
test run
target x86_64
target x86_64 has_bmi2
+target s390x
target aarch64
function %smulof_i64(i64, i64) -> i64, i8 {
diff --git a/cranelift/filetests/filetests/runtests/ssub_overflow.clif b/cranelift/filetests/filetests/runtests/ssub_overflow.clif
index ad6c86841d..27b075140f 100644
--- a/cranelift/filetests/filetests/runtests/ssub_overflow.clif
+++ b/cranelift/filetests/filetests/runtests/ssub_overflow.clif
@@ -3,6 +3,7 @@ test run
set enable_llvm_abi_extensions=true
target aarch64
set enable_multi_ret_implicit_sret
+target s390x
target x86_64
function %ssubof_i128(i128, i128) -> i128, i8 {
diff --git a/cranelift/filetests/filetests/runtests/uadd_overflow_128.clif b/cranelift/filetests/filetests/runtests/uadd_overflow_128.clif
index aff84a7188..5e3a92ef7d 100644
--- a/cranelift/filetests/filetests/runtests/uadd_overflow_128.clif
+++ b/cranelift/filetests/filetests/runtests/uadd_overflow_128.clif
@@ -5,6 +5,7 @@ target aarch64
set enable_multi_ret_implicit_sret
target x86_64
target riscv64
+target s390x
function %uaddof_i128(i128, i128) -> i128, i8 {
block0(v0: i128,v1: i128):
diff --git a/cranelift/filetests/filetests/runtests/uadd_overflow_narrow.clif b/cranelift/filetests/filetests/runtests/uadd_overflow_narrow.clif
index df8c87d1e9..681f244867 100644
--- a/cranelift/filetests/filetests/runtests/uadd_overflow_narrow.clif
+++ b/cranelift/filetests/filetests/runtests/uadd_overflow_narrow.clif
@@ -3,6 +3,7 @@ test run
set enable_llvm_abi_extensions=true
target aarch64
set enable_multi_ret_implicit_sret
+target s390x
target x86_64
function %uaddof_i8(i8, i8) -> i8, i8 {
diff --git a/cranelift/filetests/filetests/runtests/umul_overflow.clif b/cranelift/filetests/filetests/runtests/umul_overflow.clif
index 6f765ab488..0ebbfd76f7 100644
--- a/cranelift/filetests/filetests/runtests/umul_overflow.clif
+++ b/cranelift/filetests/filetests/runtests/umul_overflow.clif
@@ -2,6 +2,7 @@ test interpret
test run
target x86_64
target x86_64 has_bmi2
+target s390x
target aarch64
function %umulof_i64(i64, i64) -> i64, i8 {
diff --git a/cranelift/filetests/filetests/runtests/usub_overflow.clif b/cranelift/filetests/filetests/runtests/usub_overflow.clif
index 7ad4e82399..0a1828bf56 100644
--- a/cranelift/filetests/filetests/runtests/usub_overflow.clif
+++ b/cranelift/filetests/filetests/runtests/usub_overflow.clif
@@ -3,6 +3,7 @@ test run
set enable_llvm_abi_extensions=true
target aarch64
set enable_multi_ret_implicit_sret
+target s390x
target x86_64
function %usubof_i128(i128, i128) -> i128, i8 {