A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Fix bounds-check elision with 4 GiB guard pages. (#11973)
Dan Gohmanc8dce6fbdeb7
diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rsindex c0dc9ad664..805e37d1e6 100644--- a/crates/cranelift/src/bounds_checks.rs+++ b/crates/cranelift/src/bounds_checks.rs@@ -402,6 +402,7 @@ fn bounds_check_field_access(if can_use_virtual_memory&& heap.memory.minimum_byte_size().unwrap_or(u64::MAX) <= memory_reservation&& !heap.memory.memory_may_move(env.tunables())+ && memory_reservation >= offset_and_size{let adjusted_bound = memory_reservation.checked_sub(offset_and_size).unwrap();let adjusted_bound_value = builderdiff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rsindex d6f148e68e..72dcc9fe3c 100644--- a/crates/environ/src/types.rs+++ b/crates/environ/src/types.rs@@ -2152,7 +2152,7 @@ impl Memory {pub fn can_elide_bounds_check(&self, tunables: &Tunables, host_page_size_log2: u8) -> bool {self.can_use_virtual_memory(tunables, host_page_size_log2)&& self.idx_type == IndexType::I32- && tunables.memory_reservation >= (1 << 32)+ && tunables.memory_reservation + tunables.memory_guard_size >= (1 << 32)}/// Returns the static size of this heap in bytes at runtime, if available.diff --git a/tests/disas/bounds-check.wat b/tests/disas/bounds-check.watnew file mode 100644index 0000000000..0ac31ec59f--- /dev/null+++ b/tests/disas/bounds-check.wat@@ -0,0 +1,56 @@+;;! test = "optimize"+;;! target = "x86_64"+;;! flags = ["-Omemory-reservation=0x8000000", "-Omemory-guard-size=0x100000000", "-Omemory-may-move=n"]++(module+ (memory 16)+ (func $store (param i32)+ ;; No offset. But because we have a 4 GiB guard, this needs no bounds check.+ local.get 0+ i32.const 0+ i32.store8 0++ ;; The greatest possible offset that can ever be in bounds. Again, no+ ;; bounds check.+ local.get 0+ i32.const 0+ i32.store8 0 offset=134217727++ ;; The greatest encodable offset. This will never be in bounds, given+ ;; our memory reservation size, so optimization isn't a concern.+ local.get 0+ i32.const 0+ i32.store8 0 offset=4294967295+ )+ (export "store" (func $store))+)+;; function u0:0(i64 vmctx, i64, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+64+;; gv5 = load.i64 notrap aligned readonly can_move checked gv3+56+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @002a v3 = iconst.i32 0+;; @002c v5 = load.i64 notrap aligned readonly can_move checked v0+56+;; @002c v4 = uextend.i64 v2+;; @002c v6 = iadd v5, v4+;; @002c istore8 little heap v3, v6 ; v3 = 0+;; @0033 v11 = iconst.i64 0x07ff_ffff+;; @0033 v12 = iadd v6, v11 ; v11 = 0x07ff_ffff+;; @0033 istore8 little heap v3, v12 ; v3 = 0+;; @003d v15 = load.i64 notrap aligned v0+64+;; @003d v16 = icmp ugt v4, v15+;; @003d v21 = iconst.i64 0+;; @003d v19 = iconst.i64 0xffff_ffff+;; @003d v20 = iadd v6, v19 ; v19 = 0xffff_ffff+;; @003d v22 = select_spectre_guard v16, v21, v20 ; v21 = 0+;; @003d istore8 little heap v3, v22 ; v3 = 0+;; @0044 jump block1+;;+;; block1:+;; @0044 return+;; }diff --git a/tests/disas/winch/x64/load/v128_load32x2_s_oob_avx.wat b/tests/disas/winch/x64/load/v128_load32x2_s_oob_avx.watindex 2cee93cf9d..5b55d34585 100644--- a/tests/disas/winch/x64/load/v128_load32x2_s_oob_avx.wat+++ b/tests/disas/winch/x64/load/v128_load32x2_s_oob_avx.wat@@ -14,27 +14,22 @@;; movq 0x10(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x72+;; ja 0x64;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $0xfff9, %eax-;; movl $0x10000, %ecx-;; movl %eax, %edx-;; addq $8, %rdx-;; jb 0x74-;; 48: cmpq %rcx, %rdx-;; ja 0x76-;; 51: movq 0x38(%r14), %rbx-;; addq %rax, %rbx-;; movl $0, %esi-;; cmpq %rcx, %rdx-;; cmovaq %rsi, %rbx-;; vpmovsxdq (%rbx), %xmm0+;; cmpq $0xfff8, %rax+;; ja 0x66+;; 40: movq 0x38(%r14), %rcx+;; addq %rax, %rcx+;; movl $0, %edx+;; cmpq $0xfff8, %rax+;; cmovaq %rdx, %rcx+;; vpmovsxdq (%rcx), %xmm0;; addq $0x10, %rsp;; popq %rbp;; retq-;; 72: ud2-;; 74: ud2-;; 76: ud2+;; 64: ud2+;; 66: ud2diff --git a/tests/disas/winch/x64/load/v128_load32x2_u_oob_avx.wat b/tests/disas/winch/x64/load/v128_load32x2_u_oob_avx.watindex 2e5ffedd96..645a318e3f 100644--- a/tests/disas/winch/x64/load/v128_load32x2_u_oob_avx.wat+++ b/tests/disas/winch/x64/load/v128_load32x2_u_oob_avx.wat@@ -14,27 +14,22 @@;; movq 0x10(%r11), %r11;; addq $0x10, %r11;; cmpq %rsp, %r11-;; ja 0x72+;; ja 0x64;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movl $0xfff9, %eax-;; movl $0x10000, %ecx-;; movl %eax, %edx-;; addq $8, %rdx-;; jb 0x74-;; 48: cmpq %rcx, %rdx-;; ja 0x76-;; 51: movq 0x38(%r14), %rbx-;; addq %rax, %rbx-;; movl $0, %esi-;; cmpq %rcx, %rdx-;; cmovaq %rsi, %rbx-;; vpmovzxdq (%rbx), %xmm0+;; cmpq $0xfff8, %rax+;; ja 0x66+;; 40: movq 0x38(%r14), %rcx+;; addq %rax, %rcx+;; movl $0, %edx+;; cmpq $0xfff8, %rax+;; cmovaq %rdx, %rcx+;; vpmovzxdq (%rcx), %xmm0;; addq $0x10, %rsp;; popq %rbp;; retq-;; 72: ud2-;; 74: ud2-;; 76: ud2+;; 64: ud2+;; 66: ud2diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex 1907e90b3a..1fcbd18eba 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -695,6 +695,8 @@ where.can_elide_bounds_check(self.tunables, self.env.page_size_log2);let addr = if offset_with_access_size > heap.memory.maximum_byte_size().unwrap_or(u64::MAX)+ || (!self.tunables.memory_may_move+ && offset_with_access_size > self.tunables.memory_reservation){// Detect at compile time if the access is out of bounds.// Doing so will put the compiler in an unreachable code state,@@ -706,7 +708,80 @@ whereself.masm.trap(TrapCode::HEAP_OUT_OF_BOUNDS)?;self.context.reachable = false;None- } else if !can_elide_bounds_check {++ // Account for the case in which we can completely elide the bounds+ // checks.+ //+ // This case, makes use of the fact that if a memory access uses+ // a 32-bit index, then we be certain that+ //+ // index <= u32::MAX+ //+ // Therefore if any 32-bit index access occurs in the region+ // represented by+ //+ // bound + guard_size - (offset + access_size)+ //+ // We are certain that it's in bounds or that the underlying virtual+ // memory subsystem will report an illegal access at runtime.+ //+ // Note:+ //+ // * bound - (offset + access_size) cannot wrap, because it's checked+ // in the condition above.+ // * bound + heap.offset_guard_size is guaranteed to not overflow if+ // the heap configuration is correct, given that it's address must+ // fit in 64-bits.+ // * If the heap type is 32-bits, the offset is at most u32::MAX, so+ // no adjustment is needed as part of+ // [bounds::ensure_index_and_offset].+ } else if can_elide_bounds_check+ && u64::from(u32::MAX)+ <= self.tunables.memory_reservation + self.tunables.memory_guard_size+ - offset_with_access_size+ {+ assert!(can_elide_bounds_check);+ assert!(heap.index_type() == WasmValType::I32);+ let addr = self.context.any_gpr(self.masm)?;+ bounds::load_heap_addr_unchecked(self.masm, &heap, index, offset, addr, ptr_size)?;+ Some(addr)++ // Account for the case of a static memory size. The access is out+ // of bounds if:+ //+ // index > bound - (offset + access_size)+ //+ // bound - (offset + access_size) cannot wrap, because we already+ // checked that (offset + access_size) > bound, above.+ } else if let Some(static_size) = heap.memory.static_heap_size() {+ let bounds = Bounds::from_u64(static_size);+ let addr = bounds::load_heap_addr_checked(+ self.masm,+ &mut self.context,+ ptr_size,+ &heap,+ enable_spectre_mitigation,+ bounds,+ index,+ offset,+ |masm, bounds, index| {+ let adjusted_bounds = bounds.as_u64() - offset_with_access_size;+ let index_reg = index.as_typed_reg().reg;+ masm.cmp(+ index_reg,+ RegImm::i64(adjusted_bounds as i64),+ // Similar to the dynamic heap case, even though the+ // offset and access size are bound through the heap+ // type, when added they can overflow, resulting in+ // an erroneous comparison, therefore we rely on the+ // target pointer size.+ ptr_size,+ )?;+ Ok(IntCmpKind::GtU)+ },+ )?;+ Some(addr)+ } else {// Account for the general case for bounds-checked memories. The// access is out of bounds if:// * index + offset + access_size overflows@@ -784,80 +859,6 @@ whereself.context.free_reg(bounds.as_typed_reg().reg);self.context.free_reg(index_offset_and_access_size);Some(addr)-- // Account for the case in which we can completely elide the bounds- // checks.- //- // This case, makes use of the fact that if a memory access uses- // a 32-bit index, then we be certain that- //- // index <= u32::MAX- //- // Therefore if any 32-bit index access occurs in the region- // represented by- //- // bound + guard_size - (offset + access_size)- //- // We are certain that it's in bounds or that the underlying virtual- // memory subsystem will report an illegal access at runtime.- //- // Note:- //- // * bound - (offset + access_size) cannot wrap, because it's checked- // in the condition above.- // * bound + heap.offset_guard_size is guaranteed to not overflow if- // the heap configuration is correct, given that it's address must- // fit in 64-bits.- // * If the heap type is 32-bits, the offset is at most u32::MAX, so- // no adjustment is needed as part of- // [bounds::ensure_index_and_offset].- } else if u64::from(u32::MAX)- <= self.tunables.memory_reservation + self.tunables.memory_guard_size- - offset_with_access_size- {- assert!(can_elide_bounds_check);- assert!(heap.index_type() == WasmValType::I32);- let addr = self.context.any_gpr(self.masm)?;- bounds::load_heap_addr_unchecked(self.masm, &heap, index, offset, addr, ptr_size)?;- Some(addr)-- // Account for the all remaining cases, aka. The access is out- // of bounds if:- //- // index > bound - (offset + access_size)- //- // bound - (offset + access_size) cannot wrap, because we already- // checked that (offset + access_size) > bound, above.- } else {- assert!(can_elide_bounds_check);- assert!(heap.index_type() == WasmValType::I32);- let bounds = Bounds::from_u64(self.tunables.memory_reservation);- let addr = bounds::load_heap_addr_checked(- self.masm,- &mut self.context,- ptr_size,- &heap,- enable_spectre_mitigation,- bounds,- index,- offset,- |masm, bounds, index| {- let adjusted_bounds = bounds.as_u64() - offset_with_access_size;- let index_reg = index.as_typed_reg().reg;- masm.cmp(- index_reg,- RegImm::i64(adjusted_bounds as i64),- // Similar to the dynamic heap case, even though the- // offset and access size are bound through the heap- // type, when added they can overflow, resulting in- // an erroneous comparison, therefore we rely on the- // target pointer size.- ptr_size,- )?;- Ok(IntCmpKind::GtU)- },- )?;- Some(addr)};self.context.free_reg(index.as_typed_reg().reg);