Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch(x64): Avoid undefined behavior on ctz/clz (#13829)

Saúl Cabrera8c4618ac6663
diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat
index 1fd75c373c..dbbfb59610 100644
--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat
+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_const.wat
@@ -14,19 +14,18 @@
;; movq 0x18(%r11), %r11
;; addq $0x10, %r11
;; cmpq %rsp, %r11
-;; ja 0x54
+;; ja 0x51
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movl $1, %eax
;; bsrl %eax, %eax
-;; movl $0, %r11d
-;; setne %r11b
+;; movl $0xffffffff, %r11d
+;; cmovel %r11d, %eax
;; negl %eax
-;; addl $0x20, %eax
-;; subl %r11d, %eax
+;; addl $0x1f, %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 54: ud2
+;; 51: ud2
diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat
index d371440707..89cc051523 100644
--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat
+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_local.wat
@@ -19,7 +19,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x66
+;; ja 0x63
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -29,12 +29,11 @@
;; movl %eax, 0xc(%rsp)
;; movl 0xc(%rsp), %eax
;; bsrl %eax, %eax
-;; movl $0, %r11d
-;; setne %r11b
+;; movl $0xffffffff, %r11d
+;; cmovel %r11d, %eax
;; negl %eax
-;; addl $0x20, %eax
-;; subl %r11d, %eax
+;; addl $0x1f, %eax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 66: ud2
+;; 63: ud2
diff --git a/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat b/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat
index 29c3ee3567..36405e758e 100644
--- a/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat
+++ b/tests/disas/winch/x64/i32_clz/no_lzcnt_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x58
+;; ja 0x55
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -22,12 +22,11 @@
;; movl %edx, 0xc(%rsp)
;; movl 0xc(%rsp), %eax
;; bsrl %eax, %eax
-;; movl $0, %r11d
-;; setne %r11b
+;; movl $0xffffffff, %r11d
+;; cmovel %r11d, %eax
;; negl %eax
-;; addl $0x20, %eax
-;; subl %r11d, %eax
+;; addl $0x1f, %eax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 58: ud2
+;; 55: ud2
diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat
index 0ad37cddc0..0cf0250d86 100644
--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat
+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_const.wat
@@ -14,18 +14,16 @@
;; movq 0x18(%r11), %r11
;; addq $0x10, %r11
;; cmpq %rsp, %r11
-;; ja 0x51
+;; ja 0x4a
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movl $1, %eax
;; bsfl %eax, %eax
-;; movl $0, %r11d
-;; sete %r11b
-;; shll $5, %r11d
-;; addl %r11d, %eax
+;; movl $0x20, %r11d
+;; cmovel %r11d, %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 51: ud2
+;; 4a: ud2
diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat
index 42265ffa21..a827def48c 100644
--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat
+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_local.wat
@@ -19,7 +19,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x63
+;; ja 0x5c
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -29,11 +29,9 @@
;; movl %eax, 0xc(%rsp)
;; movl 0xc(%rsp), %eax
;; bsfl %eax, %eax
-;; movl $0, %r11d
-;; sete %r11b
-;; shll $5, %r11d
-;; addl %r11d, %eax
+;; movl $0x20, %r11d
+;; cmovel %r11d, %eax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 63: ud2
+;; 5c: ud2
diff --git a/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat b/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat
index 7d3c5e5bde..edf6f15d0c 100644
--- a/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat
+++ b/tests/disas/winch/x64/i32_ctz/no_bmi1_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x55
+;; ja 0x4e
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -22,11 +22,9 @@
;; movl %edx, 0xc(%rsp)
;; movl 0xc(%rsp), %eax
;; bsfl %eax, %eax
-;; movl $0, %r11d
-;; sete %r11b
-;; shll $5, %r11d
-;; addl %r11d, %eax
+;; movl $0x20, %r11d
+;; cmovel %r11d, %eax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 55: ud2
+;; 4e: ud2
diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat
index edea5ab8df..d8b9bdd2c0 100644
--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat
+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_const.wat
@@ -14,19 +14,18 @@
;; movq 0x18(%r11), %r11
;; addq $0x10, %r11
;; cmpq %rsp, %r11
-;; ja 0x57
+;; ja 0x55
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movl $1, %eax
;; bsrq %rax, %rax
-;; movl $0, %r11d
-;; setne %r11b
+;; movq $18446744073709551615, %r11
+;; cmoveq %r11, %rax
;; negq %rax
-;; addq $0x40, %rax
-;; subq %r11, %rax
+;; addq $0x3f, %rax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 57: ud2
+;; 55: ud2
diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat
index 6dfd97a817..e7fe656f65 100644
--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat
+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_local.wat
@@ -19,7 +19,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x6b
+;; ja 0x69
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -29,12 +29,11 @@
;; movq %rax, 8(%rsp)
;; movq 8(%rsp), %rax
;; bsrq %rax, %rax
-;; movl $0, %r11d
-;; setne %r11b
+;; movq $18446744073709551615, %r11
+;; cmoveq %r11, %rax
;; negq %rax
-;; addq $0x40, %rax
-;; subq %r11, %rax
+;; addq $0x3f, %rax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 6b: ud2
+;; 69: ud2
diff --git a/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat b/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat
index 3f1cd9d6ff..99010af90f 100644
--- a/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat
+++ b/tests/disas/winch/x64/i64_clz/no_lzcnt_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x5d
+;; ja 0x5b
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -22,12 +22,11 @@
;; movq %rdx, 8(%rsp)
;; movq 8(%rsp), %rax
;; bsrq %rax, %rax
-;; movl $0, %r11d
-;; setne %r11b
+;; movq $18446744073709551615, %r11
+;; cmoveq %r11, %rax
;; negq %rax
-;; addq $0x40, %rax
-;; subq %r11, %rax
+;; addq $0x3f, %rax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 5d: ud2
+;; 5b: ud2
diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat
index da1ef98aab..64eb745d04 100644
--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat
+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_const.wat
@@ -14,18 +14,16 @@
;; movq 0x18(%r11), %r11
;; addq $0x10, %r11
;; cmpq %rsp, %r11
-;; ja 0x52
+;; ja 0x4b
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movl $1, %eax
;; bsfq %rax, %rax
-;; movl $0, %r11d
-;; sete %r11b
-;; shlq $6, %r11
-;; addq %r11, %rax
+;; movl $0x40, %r11d
+;; cmoveq %r11, %rax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 52: ud2
+;; 4b: ud2
diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat
index da476e1d39..fdced990cc 100644
--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat
+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_local.wat
@@ -19,7 +19,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x66
+;; ja 0x5f
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -29,11 +29,9 @@
;; movq %rax, 8(%rsp)
;; movq 8(%rsp), %rax
;; bsfq %rax, %rax
-;; movl $0, %r11d
-;; sete %r11b
-;; shlq $6, %r11
-;; addq %r11, %rax
+;; movl $0x40, %r11d
+;; cmoveq %r11, %rax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 66: ud2
+;; 5f: ud2
diff --git a/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat b/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat
index 631bbb36a8..c49fd4b49d 100644
--- a/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat
+++ b/tests/disas/winch/x64/i64_ctz/no_bmi1_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x58
+;; ja 0x51
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -22,11 +22,9 @@
;; movq %rdx, 8(%rsp)
;; movq 8(%rsp), %rax
;; bsfq %rax, %rax
-;; movl $0, %r11d
-;; sete %r11b
-;; shlq $6, %r11
-;; addq %r11, %rax
+;; movl $0x40, %r11d
+;; cmoveq %r11, %rax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 58: ud2
+;; 51: ud2
diff --git a/tests/disas/winch/x64/loop/as_unary_operand.wat b/tests/disas/winch/x64/loop/as_unary_operand.wat
index 847017e590..67cc40a538 100644
--- a/tests/disas/winch/x64/loop/as_unary_operand.wat
+++ b/tests/disas/winch/x64/loop/as_unary_operand.wat
@@ -30,7 +30,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x10, %r11
;; cmpq %rsp, %r11
-;; ja 0xa1
+;; ja 0x9a
;; 5c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -41,11 +41,9 @@
;; movq 8(%rsp), %r14
;; movl $0xd, %eax
;; bsfl %eax, %eax
-;; movl $0, %r11d
-;; sete %r11b
-;; shll $5, %r11d
-;; addl %r11d, %eax
+;; movl $0x20, %r11d
+;; cmovel %r11d, %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; a1: ud2
+;; 9a: ud2
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 9fea5d7876..2c2fd67d7c 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -1043,14 +1043,21 @@ impl Masm for MacroAssembler {
self.asm.lzcnt(src, dst, size);
} else {
self.with_scratch::<IntScratch, _>(|masm, scratch| {
- // Use the following approach:
- // dst = size.num_bits() - bsr(src) - is_not_zero
- // = size.num.bits() + -bsr(src) - is_not_zero.
+ // For a non-zero input, BSR returns the index of the
+ // most-significant set bit, so clz is:
+ // dst = (num_bits - 1) - bsr(src)
+ // which is emitted as a negate followed by an add:
+ // dst = -bsr(src) + (num_bits - 1)
+ //
+ // BSR leaves the destination undefined when the source is 0
+ //
+ // A conditional move substitutes -1 for the undefined index
+ // in the zero case.
masm.asm.bsr(src, dst, size);
- masm.asm.setcc(IntCmpKind::Ne, scratch.writable());
+ masm.asm.mov_ir((-1i64) as u64, scratch.writable(), size);
+ masm.asm.cmov(scratch.inner(), dst, IntCmpKind::Eq, size);
masm.asm.neg(dst.to_reg(), dst, size);
- masm.asm.add_ir(size.num_bits() as i32, dst, size);
- masm.asm.sub_rr(scratch.inner(), dst, size);
+ masm.asm.add_ir((size.num_bits() - 1) as i32, dst, size);
});
}
@@ -1062,17 +1069,18 @@ impl Masm for MacroAssembler {
self.asm.tzcnt(src, dst, size);
} else {
self.with_scratch::<IntScratch, _>(|masm, scratch| {
- // Use the following approach:
- // dst = bsf(src) + (is_zero * size.num_bits())
- // = bsf(src) + (is_zero << size.log2()).
- // BSF outputs the correct value for every value except 0.
- // When the value is 0, BSF outputs 0, correct output for ctz is
- // the number of bits.
+ // BSF returns the index of the least-significant set bit, which
+ // is the correct output of ctz for any non-zero input.
+ //
+ // However, it leaves the destination undefined when the source
+ // is 0.
+ //
+ // Perform a conditional move to ensure that the destination
+ // register is always correctly defined.
masm.asm.bsf(src, dst, size);
- masm.asm.setcc(IntCmpKind::Eq, scratch.writable());
masm.asm
- .shift_ir(size.log2(), scratch.writable(), ShiftKind::Shl, size);
- masm.asm.add_rr(scratch.inner(), dst, size);
+ .mov_ir(size.num_bits() as u64, scratch.writable(), size);
+ masm.asm.cmov(scratch.inner(), dst, IntCmpKind::Eq, size);
});
}
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 992a3b4449..9483ccb0f8 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1080,17 +1080,6 @@ impl OperandSize {
}
}
- /// The binary logarithm of the number of bits in the operand.
- pub fn log2(&self) -> u8 {
- match self {
- OperandSize::S8 => 3,
- OperandSize::S16 => 4,
- OperandSize::S32 => 5,
- OperandSize::S64 => 6,
- OperandSize::S128 => 7,
- }
- }
-
/// Create an [`OperandSize`] from the given number of bytes.
pub fn from_bytes(bytes: u8) -> Self {
use OperandSize::*;