A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch(x64): Avoid undefined behavior on ctz/clz (#13829)
Saúl Cabrera8c4618ac6663
diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_const.watindex 1fd75c373c..dbbfb59610 100644--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat@@ -14,19 +14,18 @@;; movq 0x18(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x54+;; ja 0x51;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $1, %eax;; bsrl %eax, %eax-;; movl $0, %r11d-;; setne %r11b+;; movl $0xffffffff, %r11d+;; cmovel %r11d, %eax;; negl %eax-;; addl $0x20, %eax-;; subl %r11d, %eax+;; addl $0x1f, %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; 54: ud2+;; 51: ud2diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_local.watindex d371440707..89cc051523 100644--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat@@ -19,7 +19,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x66+;; ja 0x63;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -29,12 +29,11 @@;; movl %eax, 0xc(%rsp);; movl 0xc(%rsp), %eax;; bsrl %eax, %eax-;; movl $0, %r11d-;; setne %r11b+;; movl $0xffffffff, %r11d+;; cmovel %r11d, %eax;; negl %eax-;; addl $0x20, %eax-;; subl %r11d, %eax+;; addl $0x1f, %eax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 66: ud2+;; 63: ud2diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_param.watindex 29c3ee3567..36405e758e 100644--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat@@ -14,7 +14,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x58+;; ja 0x55;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -22,12 +22,11 @@;; movl %edx, 0xc(%rsp);; movl 0xc(%rsp), %eax;; bsrl %eax, %eax-;; movl $0, %r11d-;; setne %r11b+;; movl $0xffffffff, %r11d+;; cmovel %r11d, %eax;; negl %eax-;; addl $0x20, %eax-;; subl %r11d, %eax+;; addl $0x1f, %eax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 58: ud2+;; 55: ud2diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_const.watindex 0ad37cddc0..0cf0250d86 100644--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat@@ -14,18 +14,16 @@;; movq 0x18(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x51+;; ja 0x4a;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $1, %eax;; bsfl %eax, %eax-;; movl $0, %r11d-;; sete %r11b-;; shll $5, %r11d-;; addl %r11d, %eax+;; movl $0x20, %r11d+;; cmovel %r11d, %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; 51: ud2+;; 4a: ud2diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_local.watindex 42265ffa21..a827def48c 100644--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat@@ -19,7 +19,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x63+;; ja 0x5c;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -29,11 +29,9 @@;; movl %eax, 0xc(%rsp);; movl 0xc(%rsp), %eax;; bsfl %eax, %eax-;; movl $0, %r11d-;; sete %r11b-;; shll $5, %r11d-;; addl %r11d, %eax+;; movl $0x20, %r11d+;; cmovel %r11d, %eax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 63: ud2+;; 5c: ud2diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_param.watindex 7d3c5e5bde..edf6f15d0c 100644--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat@@ -14,7 +14,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x55+;; ja 0x4e;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -22,11 +22,9 @@;; movl %edx, 0xc(%rsp);; movl 0xc(%rsp), %eax;; bsfl %eax, %eax-;; movl $0, %r11d-;; sete %r11b-;; shll $5, %r11d-;; addl %r11d, %eax+;; movl $0x20, %r11d+;; cmovel %r11d, %eax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 55: ud2+;; 4e: ud2diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_const.watindex edea5ab8df..d8b9bdd2c0 100644--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat@@ -14,19 +14,18 @@;; movq 0x18(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x57+;; ja 0x55;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $1, %eax;; bsrq %rax, %rax-;; movl $0, %r11d-;; setne %r11b+;; movq $18446744073709551615, %r11+;; cmoveq %r11, %rax;; negq %rax-;; addq $0x40, %rax-;; subq %r11, %rax+;; addq $0x3f, %rax;; addq $0x10, %rsp;; popq %rbp;; retq-;; 57: ud2+;; 55: ud2diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_local.watindex 6dfd97a817..e7fe656f65 100644--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat@@ -19,7 +19,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x6b+;; ja 0x69;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -29,12 +29,11 @@;; movq %rax, 8(%rsp);; movq 8(%rsp), %rax;; bsrq %rax, %rax-;; movl $0, %r11d-;; setne %r11b+;; movq $18446744073709551615, %r11+;; cmoveq %r11, %rax;; negq %rax-;; addq $0x40, %rax-;; subq %r11, %rax+;; addq $0x3f, %rax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 6b: ud2+;; 69: ud2diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_param.watindex 3f1cd9d6ff..99010af90f 100644--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat@@ -14,7 +14,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x5d+;; ja 0x5b;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -22,12 +22,11 @@;; movq %rdx, 8(%rsp);; movq 8(%rsp), %rax;; bsrq %rax, %rax-;; movl $0, %r11d-;; setne %r11b+;; movq $18446744073709551615, %r11+;; cmoveq %r11, %rax;; negq %rax-;; addq $0x40, %rax-;; subq %r11, %rax+;; addq $0x3f, %rax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 5d: ud2+;; 5b: ud2diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_const.watindex da1ef98aab..64eb745d04 100644--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat@@ -14,18 +14,16 @@;; movq 0x18(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x52+;; ja 0x4b;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $1, %eax;; bsfq %rax, %rax-;; movl $0, %r11d-;; sete %r11b-;; shlq $6, %r11-;; addq %r11, %rax+;; movl $0x40, %r11d+;; cmoveq %r11, %rax;; addq $0x10, %rsp;; popq %rbp;; retq-;; 52: ud2+;; 4b: ud2diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_local.watindex da476e1d39..fdced990cc 100644--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat@@ -19,7 +19,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x66+;; ja 0x5f;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -29,11 +29,9 @@;; movq %rax, 8(%rsp);; movq 8(%rsp), %rax;; bsfq %rax, %rax-;; movl $0, %r11d-;; sete %r11b-;; shlq $6, %r11-;; addq %r11, %rax+;; movl $0x40, %r11d+;; cmoveq %r11, %rax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 66: ud2+;; 5f: ud2diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_param.watindex 631bbb36a8..c49fd4b49d 100644--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat@@ -14,7 +14,7 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x58+;; ja 0x51;; 1c: movq %rdi, %r14;; subq $0x20, %rsp;; movq %rdi, 0x18(%rsp)@@ -22,11 +22,9 @@;; movq %rdx, 8(%rsp);; movq 8(%rsp), %rax;; bsfq %rax, %rax-;; movl $0, %r11d-;; sete %r11b-;; shlq $6, %r11-;; addq %r11, %rax+;; movl $0x40, %r11d+;; cmoveq %r11, %rax;; addq $0x20, %rsp;; popq %rbp;; retq-;; 58: ud2+;; 51: ud2diff --git a/tests/disas/winch/x64/loop/as_unary_operand.wat b/tests/disas/winch/x64/loop/as_unary_operand.watindex 847017e590..67cc40a538 100644--- a/tests/disas/winch/x64/loop/as_unary_operand.wat+++ b/tests/disas/winch/x64/loop/as_unary_operand.wat@@ -30,7 +30,7 @@;; movq 0x18(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0xa1+;; ja 0x9a;; 5c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp)@@ -41,11 +41,9 @@;; movq 8(%rsp), %r14;; movl $0xd, %eax;; bsfl %eax, %eax-;; movl $0, %r11d-;; sete %r11b-;; shll $5, %r11d-;; addl %r11d, %eax+;; movl $0x20, %r11d+;; cmovel %r11d, %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; a1: ud2+;; 9a: ud2diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex 9fea5d7876..2c2fd67d7c 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -1043,14 +1043,21 @@ impl Masm for MacroAssembler {self.asm.lzcnt(src, dst, size);} else {self.with_scratch::<IntScratch, _>(|masm, scratch| {- // Use the following approach:- // dst = size.num_bits() - bsr(src) - is_not_zero- // = size.num.bits() + -bsr(src) - is_not_zero.+ // For a non-zero input, BSR returns the index of the+ // most-significant set bit, so clz is:+ // dst = (num_bits - 1) - bsr(src)+ // which is emitted as a negate followed by an add:+ // dst = -bsr(src) + (num_bits - 1)+ //+ // BSR leaves the destination undefined when the source is 0+ //+ // A conditional move substitutes -1 for the undefined index+ // in the zero case.masm.asm.bsr(src, dst, size);- masm.asm.setcc(IntCmpKind::Ne, scratch.writable());+ masm.asm.mov_ir((-1i64) as u64, scratch.writable(), size);+ masm.asm.cmov(scratch.inner(), dst, IntCmpKind::Eq, size);masm.asm.neg(dst.to_reg(), dst, size);- masm.asm.add_ir(size.num_bits() as i32, dst, size);- masm.asm.sub_rr(scratch.inner(), dst, size);+ masm.asm.add_ir((size.num_bits() - 1) as i32, dst, size);});}@@ -1062,17 +1069,18 @@ impl Masm for MacroAssembler {self.asm.tzcnt(src, dst, size);} else {self.with_scratch::<IntScratch, _>(|masm, scratch| {- // Use the following approach:- // dst = bsf(src) + (is_zero * size.num_bits())- // = bsf(src) + (is_zero << size.log2()).- // BSF outputs the correct value for every value except 0.- // When the value is 0, BSF outputs 0, correct output for ctz is- // the number of bits.+ // BSF returns the index of the least-significant set bit, which+ // is the correct output of ctz for any non-zero input.+ //+ // However, it leaves the destination undefined when the source+ // is 0.+ //+ // Perform a conditional move to ensure that the destination+ // register is always correctly defined.masm.asm.bsf(src, dst, size);- masm.asm.setcc(IntCmpKind::Eq, scratch.writable());masm.asm- .shift_ir(size.log2(), scratch.writable(), ShiftKind::Shl, size);- masm.asm.add_rr(scratch.inner(), dst, size);+ .mov_ir(size.num_bits() as u64, scratch.writable(), size);+ masm.asm.cmov(scratch.inner(), dst, IntCmpKind::Eq, size);});}diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rsindex 992a3b4449..9483ccb0f8 100644--- a/winch/codegen/src/masm.rs+++ b/winch/codegen/src/masm.rs@@ -1080,17 +1080,6 @@ impl OperandSize {}}- /// The binary logarithm of the number of bits in the operand.- pub fn log2(&self) -> u8 {- match self {- OperandSize::S8 => 3,- OperandSize::S16 => 4,- OperandSize::S32 => 5,- OperandSize::S64 => 6,- OperandSize::S128 => 7,- }- }-/// Create an [`OperandSize`] from the given number of bytes.pub fn from_bytes(bytes: u8) -> Self {use OperandSize::*;