Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: fix the interaction of `preserve_all` and stack probes+limits. (#14648)

Chris Fallinebf17985f69c
diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs
index e42d9a558c..410dfe3f6b 100644
--- a/cranelift/codegen/src/isa/pulley_shared/abi.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs
@@ -752,12 +752,12 @@ impl FrameLayout {
}
}
// Allocate space for this manually-managed clobber.
- offset -= 8;
let ty = match r_reg.class() {
RegClass::Int => I64,
RegClass::Float => F64,
RegClass::Vector => I8X16,
};
+ offset -= ty.bytes();
let offset = i32::try_from(offset).unwrap();
Some((offset, ty, Reg::from(reg.to_reg())))
})
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index 0fe5398a67..b9dae93527 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -699,7 +699,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
fn gen_inline_probestack(
insts: &mut SmallInstVec<Self::I>,
- _call_conv: isa::CallConv,
+ call_conv: isa::CallConv,
frame_size: u32,
guard_size: u32,
) {
@@ -714,6 +714,16 @@ impl ABIMachineSpec for Riscv64MachineDeps {
return;
}
+ // Probes run before any registers are saved, and a `preserve_all`
+ // function must not clobber any register. The probe loop needs both
+ // non-allocatable spill temporaries in addition to `tmp`, so always
+ // use the unrolled probes, which only need `tmp`, and use the second
+ // spill temporary for it (the first is used to restore SP).
+ if call_conv == isa::CallConv::PreserveAll {
+ Self::gen_probestack_unroll(insts, writable_spilltmp_reg2(), guard_size, probe_count);
+ return;
+ }
+
// Must be a caller-saved register that is not an argument.
let tmp = Writable::from_reg(x_reg(28)); // t3
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 3dde295c91..39d3f6cf67 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -642,7 +642,11 @@ impl ABIMachineSpec for X64ABIMachineSpec {
let probe_count = frame_size / guard_size;
if probe_count == 0 {
// No probe necessary
- } else if probe_count <= PROBE_MAX_UNROLL {
+ } else if probe_count <= PROBE_MAX_UNROLL || call_conv == isa::CallConv::PreserveAll {
+ // The probe loop needs a temporary register, but probes run
+ // before any registers are saved, and a `preserve_all` function
+ // must not clobber any register. The unrolled probes don't use
+ // any registers, so always use them for `preserve_all`.
Self::gen_probestack_unroll(insts, guard_size, probe_count)
} else {
Self::gen_probestack_loop(insts, call_conv, frame_size, guard_size)
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index 51485478e8..99e9a10747 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -78,7 +78,7 @@ use crate::ir::{
use crate::ir::{ExceptionTableItem, Signature};
use crate::isa::{CallConv, TargetIsa};
use crate::print_errors::pretty_verifier_error;
-use crate::settings::FlagsOrIsa;
+use crate::settings::{Flags, FlagsOrIsa, ProbestackStrategy};
use crate::timing;
use alloc::collections::BTreeSet;
use alloc::string::{String, ToString};
@@ -304,6 +304,7 @@ struct Verifier<'a> {
func: &'a Function,
expected_cfg: ControlFlowGraph,
expected_domtree: DominatorTree,
+ flags: &'a Flags,
isa: Option<&'a dyn TargetIsa>,
}
@@ -315,6 +316,7 @@ impl<'a> Verifier<'a> {
func,
expected_cfg,
expected_domtree,
+ flags: fisa.flags,
isa: fisa.isa,
}
}
@@ -2113,6 +2115,32 @@ impl<'a> Verifier<'a> {
Ok(())
}
+ /// A `preserve_all` function must not clobber any register, but the
+ /// prologue's stack-limit check and outline stack probe run before any
+ /// registers are saved and need scratch registers (and, for the outline
+ /// probe, a call to a function with its own clobbers). Reject both.
+ fn verify_preserve_all_prologue(&self, errors: &mut VerifierErrors) -> VerifierStepResult {
+ if self.func.signature.call_conv != CallConv::PreserveAll {
+ return Ok(());
+ }
+ if self.func.stack_limit.is_some() {
+ errors.fatal((
+ AnyEntity::Function,
+ "functions with the `preserve_all` ABI cannot have a `stack_limit`",
+ ))?;
+ }
+ if self.flags.enable_probestack()
+ && self.flags.probestack_strategy() == ProbestackStrategy::Outline
+ {
+ errors.fatal((
+ AnyEntity::Function,
+ "functions with the `preserve_all` ABI cannot use the outline stack probe \
+ (`probestack_strategy=outline`)",
+ ))?;
+ }
+ Ok(())
+ }
+
pub fn run(&self, errors: &mut VerifierErrors) -> VerifierStepResult {
self.verify_global_values(errors)?;
self.verify_alias_regions(errors)?;
@@ -2120,6 +2148,7 @@ impl<'a> Verifier<'a> {
self.check_entry_not_cold(errors)?;
self.typecheck_function_signature(errors)?;
self.verify_signatures(errors)?;
+ self.verify_preserve_all_prologue(errors)?;
for block in self.func.layout.blocks() {
if self.func.layout.first_inst(block).is_none() {
diff --git a/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif b/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif
index 67b8297bdd..3628d66ccc 100644
--- a/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif
+++ b/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif
@@ -87,38 +87,38 @@ block0(v0: i64):
; fstore64 sp+528, f29 // flags = notrap aligned
; fstore64 sp+520, f30 // flags = notrap aligned
; fstore64 sp+512, f31 // flags = notrap aligned
-; vstore128 sp+504, v0 // flags = notrap aligned little
-; vstore128 sp+496, v1 // flags = notrap aligned little
-; vstore128 sp+488, v2 // flags = notrap aligned little
-; vstore128 sp+480, v3 // flags = notrap aligned little
-; vstore128 sp+472, v4 // flags = notrap aligned little
-; vstore128 sp+464, v5 // flags = notrap aligned little
-; vstore128 sp+456, v6 // flags = notrap aligned little
-; vstore128 sp+448, v7 // flags = notrap aligned little
-; vstore128 sp+440, v8 // flags = notrap aligned little
-; vstore128 sp+432, v9 // flags = notrap aligned little
-; vstore128 sp+424, v10 // flags = notrap aligned little
-; vstore128 sp+416, v11 // flags = notrap aligned little
-; vstore128 sp+408, v12 // flags = notrap aligned little
-; vstore128 sp+400, v13 // flags = notrap aligned little
-; vstore128 sp+392, v14 // flags = notrap aligned little
-; vstore128 sp+384, v15 // flags = notrap aligned little
-; vstore128 sp+376, v16 // flags = notrap aligned little
-; vstore128 sp+368, v17 // flags = notrap aligned little
-; vstore128 sp+360, v18 // flags = notrap aligned little
-; vstore128 sp+352, v19 // flags = notrap aligned little
-; vstore128 sp+344, v20 // flags = notrap aligned little
-; vstore128 sp+336, v21 // flags = notrap aligned little
-; vstore128 sp+328, v22 // flags = notrap aligned little
-; vstore128 sp+320, v23 // flags = notrap aligned little
-; vstore128 sp+312, v24 // flags = notrap aligned little
-; vstore128 sp+304, v25 // flags = notrap aligned little
-; vstore128 sp+296, v26 // flags = notrap aligned little
-; vstore128 sp+288, v27 // flags = notrap aligned little
-; vstore128 sp+280, v28 // flags = notrap aligned little
-; vstore128 sp+272, v29 // flags = notrap aligned little
-; vstore128 sp+264, v30 // flags = notrap aligned little
-; vstore128 sp+256, v31 // flags = notrap aligned little
+; vstore128 sp+496, v0 // flags = notrap aligned little
+; vstore128 sp+480, v1 // flags = notrap aligned little
+; vstore128 sp+464, v2 // flags = notrap aligned little
+; vstore128 sp+448, v3 // flags = notrap aligned little
+; vstore128 sp+432, v4 // flags = notrap aligned little
+; vstore128 sp+416, v5 // flags = notrap aligned little
+; vstore128 sp+400, v6 // flags = notrap aligned little
+; vstore128 sp+384, v7 // flags = notrap aligned little
+; vstore128 sp+368, v8 // flags = notrap aligned little
+; vstore128 sp+352, v9 // flags = notrap aligned little
+; vstore128 sp+336, v10 // flags = notrap aligned little
+; vstore128 sp+320, v11 // flags = notrap aligned little
+; vstore128 sp+304, v12 // flags = notrap aligned little
+; vstore128 sp+288, v13 // flags = notrap aligned little
+; vstore128 sp+272, v14 // flags = notrap aligned little
+; vstore128 sp+256, v15 // flags = notrap aligned little
+; vstore128 sp+240, v16 // flags = notrap aligned little
+; vstore128 sp+224, v17 // flags = notrap aligned little
+; vstore128 sp+208, v18 // flags = notrap aligned little
+; vstore128 sp+192, v19 // flags = notrap aligned little
+; vstore128 sp+176, v20 // flags = notrap aligned little
+; vstore128 sp+160, v21 // flags = notrap aligned little
+; vstore128 sp+144, v22 // flags = notrap aligned little
+; vstore128 sp+128, v23 // flags = notrap aligned little
+; vstore128 sp+112, v24 // flags = notrap aligned little
+; vstore128 sp+96, v25 // flags = notrap aligned little
+; vstore128 sp+80, v26 // flags = notrap aligned little
+; vstore128 sp+64, v27 // flags = notrap aligned little
+; vstore128 sp+48, v28 // flags = notrap aligned little
+; vstore128 sp+32, v29 // flags = notrap aligned little
+; vstore128 sp+16, v30 // flags = notrap aligned little
+; vstore128 sp+0, v31 // flags = notrap aligned little
; block0:
; indirect_call_host CallInfo { dest: TestCase(%libcall), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [], clobbers: PRegSet { bits: [65535, 4294967295, 4294967295, 0] }, callee_conv: SystemV, caller_conv: PreserveAll, callee_pop_size: 0, try_call_info: None, patchable: false }
; x0 = xload64 sp+888 // flags = notrap aligned
@@ -169,38 +169,38 @@ block0(v0: i64):
; f29 = fload64 sp+528 // flags = notrap aligned
; f30 = fload64 sp+520 // flags = notrap aligned
; f31 = fload64 sp+512 // flags = notrap aligned
-; v0 = vload128 sp+504 // flags = notrap aligned little
-; v1 = vload128 sp+496 // flags = notrap aligned little
-; v2 = vload128 sp+488 // flags = notrap aligned little
-; v3 = vload128 sp+480 // flags = notrap aligned little
-; v4 = vload128 sp+472 // flags = notrap aligned little
-; v5 = vload128 sp+464 // flags = notrap aligned little
-; v6 = vload128 sp+456 // flags = notrap aligned little
-; v7 = vload128 sp+448 // flags = notrap aligned little
-; v8 = vload128 sp+440 // flags = notrap aligned little
-; v9 = vload128 sp+432 // flags = notrap aligned little
-; v10 = vload128 sp+424 // flags = notrap aligned little
-; v11 = vload128 sp+416 // flags = notrap aligned little
-; v12 = vload128 sp+408 // flags = notrap aligned little
-; v13 = vload128 sp+400 // flags = notrap aligned little
-; v14 = vload128 sp+392 // flags = notrap aligned little
-; v15 = vload128 sp+384 // flags = notrap aligned little
-; v16 = vload128 sp+376 // flags = notrap aligned little
-; v17 = vload128 sp+368 // flags = notrap aligned little
-; v18 = vload128 sp+360 // flags = notrap aligned little
-; v19 = vload128 sp+352 // flags = notrap aligned little
-; v20 = vload128 sp+344 // flags = notrap aligned little
-; v21 = vload128 sp+336 // flags = notrap aligned little
-; v22 = vload128 sp+328 // flags = notrap aligned little
-; v23 = vload128 sp+320 // flags = notrap aligned little
-; v24 = vload128 sp+312 // flags = notrap aligned little
-; v25 = vload128 sp+304 // flags = notrap aligned little
-; v26 = vload128 sp+296 // flags = notrap aligned little
-; v27 = vload128 sp+288 // flags = notrap aligned little
-; v28 = vload128 sp+280 // flags = notrap aligned little
-; v29 = vload128 sp+272 // flags = notrap aligned little
-; v30 = vload128 sp+264 // flags = notrap aligned little
-; v31 = vload128 sp+256 // flags = notrap aligned little
+; v0 = vload128 sp+496 // flags = notrap aligned little
+; v1 = vload128 sp+480 // flags = notrap aligned little
+; v2 = vload128 sp+464 // flags = notrap aligned little
+; v3 = vload128 sp+448 // flags = notrap aligned little
+; v4 = vload128 sp+432 // flags = notrap aligned little
+; v5 = vload128 sp+416 // flags = notrap aligned little
+; v6 = vload128 sp+400 // flags = notrap aligned little
+; v7 = vload128 sp+384 // flags = notrap aligned little
+; v8 = vload128 sp+368 // flags = notrap aligned little
+; v9 = vload128 sp+352 // flags = notrap aligned little
+; v10 = vload128 sp+336 // flags = notrap aligned little
+; v11 = vload128 sp+320 // flags = notrap aligned little
+; v12 = vload128 sp+304 // flags = notrap aligned little
+; v13 = vload128 sp+288 // flags = notrap aligned little
+; v14 = vload128 sp+272 // flags = notrap aligned little
+; v15 = vload128 sp+256 // flags = notrap aligned little
+; v16 = vload128 sp+240 // flags = notrap aligned little
+; v17 = vload128 sp+224 // flags = notrap aligned little
+; v18 = vload128 sp+208 // flags = notrap aligned little
+; v19 = vload128 sp+192 // flags = notrap aligned little
+; v20 = vload128 sp+176 // flags = notrap aligned little
+; v21 = vload128 sp+160 // flags = notrap aligned little
+; v22 = vload128 sp+144 // flags = notrap aligned little
+; v23 = vload128 sp+128 // flags = notrap aligned little
+; v24 = vload128 sp+112 // flags = notrap aligned little
+; v25 = vload128 sp+96 // flags = notrap aligned little
+; v26 = vload128 sp+80 // flags = notrap aligned little
+; v27 = vload128 sp+64 // flags = notrap aligned little
+; v28 = vload128 sp+48 // flags = notrap aligned little
+; v29 = vload128 sp+32 // flags = notrap aligned little
+; v30 = vload128 sp+16 // flags = notrap aligned little
+; v31 = vload128 sp+0 // flags = notrap aligned little
; pop_frame_restore 896, {}
; ret
;
@@ -254,38 +254,38 @@ block0(v0: i64):
; fstore64le_o32 sp, 528, f29
; fstore64le_o32 sp, 520, f30
; fstore64le_o32 sp, 512, f31
-; vstore128le_o32 sp, 504, v0
-; vstore128le_o32 sp, 496, v1
-; vstore128le_o32 sp, 488, v2
-; vstore128le_o32 sp, 480, v3
-; vstore128le_o32 sp, 472, v4
-; vstore128le_o32 sp, 464, v5
-; vstore128le_o32 sp, 456, v6
-; vstore128le_o32 sp, 448, v7
-; vstore128le_o32 sp, 440, v8
-; vstore128le_o32 sp, 432, v9
-; vstore128le_o32 sp, 424, v10
-; vstore128le_o32 sp, 416, v11
-; vstore128le_o32 sp, 408, v12
-; vstore128le_o32 sp, 400, v13
-; vstore128le_o32 sp, 392, v14
-; vstore128le_o32 sp, 384, v15
-; vstore128le_o32 sp, 376, v16
-; vstore128le_o32 sp, 368, v17
-; vstore128le_o32 sp, 360, v18
-; vstore128le_o32 sp, 352, v19
-; vstore128le_o32 sp, 344, v20
-; vstore128le_o32 sp, 336, v21
-; vstore128le_o32 sp, 328, v22
-; vstore128le_o32 sp, 320, v23
-; vstore128le_o32 sp, 312, v24
-; vstore128le_o32 sp, 304, v25
-; vstore128le_o32 sp, 296, v26
-; vstore128le_o32 sp, 288, v27
-; vstore128le_o32 sp, 280, v28
-; vstore128le_o32 sp, 272, v29
-; vstore128le_o32 sp, 264, v30
-; vstore128le_o32 sp, 256, v31
+; vstore128le_o32 sp, 496, v0
+; vstore128le_o32 sp, 480, v1
+; vstore128le_o32 sp, 464, v2
+; vstore128le_o32 sp, 448, v3
+; vstore128le_o32 sp, 432, v4
+; vstore128le_o32 sp, 416, v5
+; vstore128le_o32 sp, 400, v6
+; vstore128le_o32 sp, 384, v7
+; vstore128le_o32 sp, 368, v8
+; vstore128le_o32 sp, 352, v9
+; vstore128le_o32 sp, 336, v10
+; vstore128le_o32 sp, 320, v11
+; vstore128le_o32 sp, 304, v12
+; vstore128le_o32 sp, 288, v13
+; vstore128le_o32 sp, 272, v14
+; vstore128le_o32 sp, 256, v15
+; vstore128le_o32 sp, 240, v16
+; vstore128le_o32 sp, 224, v17
+; vstore128le_o32 sp, 208, v18
+; vstore128le_o32 sp, 192, v19
+; vstore128le_o32 sp, 176, v20
+; vstore128le_o32 sp, 160, v21
+; vstore128le_o32 sp, 144, v22
+; vstore128le_o32 sp, 128, v23
+; vstore128le_o32 sp, 112, v24
+; vstore128le_o32 sp, 96, v25
+; vstore128le_o32 sp, 80, v26
+; vstore128le_o32 sp, 64, v27
+; vstore128le_o32 sp, 48, v28
+; vstore128le_o32 sp, 32, v29
+; vstore128le_o32 sp, 16, v30
+; vstore128le_o32 sp, 0, v31
; call_indirect_host 0
; xload64le_o32 x0, sp, 888
; xload64le_o32 x1, sp, 880
@@ -335,38 +335,38 @@ block0(v0: i64):
; fload64le_o32 f29, sp, 528
; fload64le_o32 f30, sp, 520
; fload64le_o32 f31, sp, 512
-; vload128le_o32 v0, sp, 504
-; vload128le_o32 v1, sp, 496
-; vload128le_o32 v2, sp, 488
-; vload128le_o32 v3, sp, 480
-; vload128le_o32 v4, sp, 472
-; vload128le_o32 v5, sp, 464
-; vload128le_o32 v6, sp, 456
-; vload128le_o32 v7, sp, 448
-; vload128le_o32 v8, sp, 440
-; vload128le_o32 v9, sp, 432
-; vload128le_o32 v10, sp, 424
-; vload128le_o32 v11, sp, 416
-; vload128le_o32 v12, sp, 408
-; vload128le_o32 v13, sp, 400
-; vload128le_o32 v14, sp, 392
-; vload128le_o32 v15, sp, 384
-; vload128le_o32 v16, sp, 376
-; vload128le_o32 v17, sp, 368
-; vload128le_o32 v18, sp, 360
-; vload128le_o32 v19, sp, 352
-; vload128le_o32 v20, sp, 344
-; vload128le_o32 v21, sp, 336
-; vload128le_o32 v22, sp, 328
-; vload128le_o32 v23, sp, 320
-; vload128le_o32 v24, sp, 312
-; vload128le_o32 v25, sp, 304
-; vload128le_o32 v26, sp, 296
-; vload128le_o32 v27, sp, 288
-; vload128le_o32 v28, sp, 280
-; vload128le_o32 v29, sp, 272
-; vload128le_o32 v30, sp, 264
-; vload128le_o32 v31, sp, 256
+; vload128le_o32 v0, sp, 496
+; vload128le_o32 v1, sp, 480
+; vload128le_o32 v2, sp, 464
+; vload128le_o32 v3, sp, 448
+; vload128le_o32 v4, sp, 432
+; vload128le_o32 v5, sp, 416
+; vload128le_o32 v6, sp, 400
+; vload128le_o32 v7, sp, 384
+; vload128le_o32 v8, sp, 368
+; vload128le_o32 v9, sp, 352
+; vload128le_o32 v10, sp, 336
+; vload128le_o32 v11, sp, 320
+; vload128le_o32 v12, sp, 304
+; vload128le_o32 v13, sp, 288
+; vload128le_o32 v14, sp, 272
+; vload128le_o32 v15, sp, 256
+; vload128le_o32 v16, sp, 240
+; vload128le_o32 v17, sp, 224
+; vload128le_o32 v18, sp, 208
+; vload128le_o32 v19, sp, 192
+; vload128le_o32 v20, sp, 176
+; vload128le_o32 v21, sp, 160
+; vload128le_o32 v22, sp, 144
+; vload128le_o32 v23, sp, 128
+; vload128le_o32 v24, sp, 112
+; vload128le_o32 v25, sp, 96
+; vload128le_o32 v26, sp, 80
+; vload128le_o32 v27, sp, 64
+; vload128le_o32 v28, sp, 48
+; vload128le_o32 v29, sp, 32
+; vload128le_o32 v30, sp, 16
+; vload128le_o32 v31, sp, 0
; pop_frame_restore 896,
; ret
@@ -477,38 +477,38 @@ block0(v0: i64, v1: i64):
; fstore64 sp+536, f29 // flags = notrap aligned
; fstore64 sp+528, f30 // flags = notrap aligned
; fstore64 sp+520, f31 // flags = notrap aligned
-; vstore128 sp+512, v0 // flags = notrap aligned little
-; vstore128 sp+504, v1 // flags = notrap aligned little
-; vstore128 sp+496, v2 // flags = notrap aligned little
-; vstore128 sp+488, v3 // flags = notrap aligned little
-; vstore128 sp+480, v4 // flags = notrap aligned little
-; vstore128 sp+472, v5 // flags = notrap aligned little
-; vstore128 sp+464, v6 // flags = notrap aligned little
-; vstore128 sp+456, v7 // flags = notrap aligned little
-; vstore128 sp+448, v8 // flags = notrap aligned little
-; vstore128 sp+440, v9 // flags = notrap aligned little
-; vstore128 sp+432, v10 // flags = notrap aligned little
-; vstore128 sp+424, v11 // flags = notrap aligned little
-; vstore128 sp+416, v12 // flags = notrap aligned little
-; vstore128 sp+408, v13 // flags = notrap aligned little
-; vstore128 sp+400, v14 // flags = notrap aligned little
-; vstore128 sp+392, v15 // flags = notrap aligned little
-; vstore128 sp+384, v16 // flags = notrap aligned little
-; vstore128 sp+376, v17 // flags = notrap aligned little
-; vstore128 sp+368, v18 // flags = notrap aligned little
-; vstore128 sp+360, v19 // flags = notrap aligned little
-; vstore128 sp+352, v20 // flags = notrap aligned little
-; vstore128 sp+344, v21 // flags = notrap aligned little
-; vstore128 sp+336, v22 // flags = notrap aligned little
-; vstore128 sp+328, v23 // flags = notrap aligned little
-; vstore128 sp+320, v24 // flags = notrap aligned little
-; vstore128 sp+312, v25 // flags = notrap aligned little
-; vstore128 sp+304, v26 // flags = notrap aligned little
-; vstore128 sp+296, v27 // flags = notrap aligned little
-; vstore128 sp+288, v28 // flags = notrap aligned little
-; vstore128 sp+280, v29 // flags = notrap aligned little
-; vstore128 sp+272, v30 // flags = notrap aligned little
-; vstore128 sp+264, v31 // flags = notrap aligned little
+; vstore128 sp+504, v0 // flags = notrap aligned little
+; vstore128 sp+488, v1 // flags = notrap aligned little
+; vstore128 sp+472, v2 // flags = notrap aligned little
+; vstore128 sp+456, v3 // flags = notrap aligned little
+; vstore128 sp+440, v4 // flags = notrap aligned little
+; vstore128 sp+424, v5 // flags = notrap aligned little
+; vstore128 sp+408, v6 // flags = notrap aligned little
+; vstore128 sp+392, v7 // flags = notrap aligned little
+; vstore128 sp+376, v8 // flags = notrap aligned little
+; vstore128 sp+360, v9 // flags = notrap aligned little
+; vstore128 sp+344, v10 // flags = notrap aligned little
+; vstore128 sp+328, v11 // flags = notrap aligned little
+; vstore128 sp+312, v12 // flags = notrap aligned little
+; vstore128 sp+296, v13 // flags = notrap aligned little
+; vstore128 sp+280, v14 // flags = notrap aligned little
+; vstore128 sp+264, v15 // flags = notrap aligned little
+; vstore128 sp+248, v16 // flags = notrap aligned little
+; vstore128 sp+232, v17 // flags = notrap aligned little
+; vstore128 sp+216, v18 // flags = notrap aligned little
+; vstore128 sp+200, v19 // flags = notrap aligned little
+; vstore128 sp+184, v20 // flags = notrap aligned little
+; vstore128 sp+168, v21 // flags = notrap aligned little
+; vstore128 sp+152, v22 // flags = notrap aligned little
+; vstore128 sp+136, v23 // flags = notrap aligned little
+; vstore128 sp+120, v24 // flags = notrap aligned little
+; vstore128 sp+104, v25 // flags = notrap aligned little
+; vstore128 sp+88, v26 // flags = notrap aligned little
+; vstore128 sp+72, v27 // flags = notrap aligned little
+; vstore128 sp+56, v28 // flags = notrap aligned little
+; vstore128 sp+40, v29 // flags = notrap aligned little
+; vstore128 sp+24, v30 // flags = notrap aligned little
+; vstore128 sp+8, v31 // flags = notrap aligned little
; block0:
; xmov x16, x1
; indirect_call_host CallInfo { dest: TestCase(%host), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [], clobbers: PRegSet { bits: [65535, 4294967295, 4294967295, 0] }, callee_conv: SystemV, caller_conv: PreserveAll, callee_pop_size: 0, try_call_info: None, patchable: false }
@@ -562,38 +562,38 @@ block0(v0: i64, v1: i64):
; f29 = fload64 sp+536 // flags = notrap aligned
; f30 = fload64 sp+528 // flags = notrap aligned
; f31 = fload64 sp+520 // flags = notrap aligned
-; v0 = vload128 sp+512 // flags = notrap aligned little
-; v1 = vload128 sp+504 // flags = notrap aligned little
-; v2 = vload128 sp+496 // flags = notrap aligned little
-; v3 = vload128 sp+488 // flags = notrap aligned little
-; v4 = vload128 sp+480 // flags = notrap aligned little
-; v5 = vload128 sp+472 // flags = notrap aligned little
-; v6 = vload128 sp+464 // flags = notrap aligned little
-; v7 = vload128 sp+456 // flags = notrap aligned little
-; v8 = vload128 sp+448 // flags = notrap aligned little
-; v9 = vload128 sp+440 // flags = notrap aligned little
-; v10 = vload128 sp+432 // flags = notrap aligned little
-; v11 = vload128 sp+424 // flags = notrap aligned little
-; v12 = vload128 sp+416 // flags = notrap aligned little
-; v13 = vload128 sp+408 // flags = notrap aligned little
-; v14 = vload128 sp+400 // flags = notrap aligned little
-; v15 = vload128 sp+392 // flags = notrap aligned little
-; v16 = vload128 sp+384 // flags = notrap aligned little
-; v17 = vload128 sp+376 // flags = notrap aligned little
-; v18 = vload128 sp+368 // flags = notrap aligned little
-; v19 = vload128 sp+360 // flags = notrap aligned little
-; v20 = vload128 sp+352 // flags = notrap aligned little
-; v21 = vload128 sp+344 // flags = notrap aligned little
-; v22 = vload128 sp+336 // flags = notrap aligned little
-; v23 = vload128 sp+328 // flags = notrap aligned little
-; v24 = vload128 sp+320 // flags = notrap aligned little
-; v25 = vload128 sp+312 // flags = notrap aligned little
-; v26 = vload128 sp+304 // flags = notrap aligned little
-; v27 = vload128 sp+296 // flags = notrap aligned little
-; v28 = vload128 sp+288 // flags = notrap aligned little
-; v29 = vload128 sp+280 // flags = notrap aligned little
-; v30 = vload128 sp+272 // flags = notrap aligned little
-; v31 = vload128 sp+264 // flags = notrap aligned little
+; v0 = vload128 sp+504 // flags = notrap aligned little
+; v1 = vload128 sp+488 // flags = notrap aligned little
+; v2 = vload128 sp+472 // flags = notrap aligned little
+; v3 = vload128 sp+456 // flags = notrap aligned little
+; v4 = vload128 sp+440 // flags = notrap aligned little
+; v5 = vload128 sp+424 // flags = notrap aligned little
+; v6 = vload128 sp+408 // flags = notrap aligned little
+; v7 = vload128 sp+392 // flags = notrap aligned little
+; v8 = vload128 sp+376 // flags = notrap aligned little
+; v9 = vload128 sp+360 // flags = notrap aligned little
+; v10 = vload128 sp+344 // flags = notrap aligned little
+; v11 = vload128 sp+328 // flags = notrap aligned little
+; v12 = vload128 sp+312 // flags = notrap aligned little
+; v13 = vload128 sp+296 // flags = notrap aligned little
+; v14 = vload128 sp+280 // flags = notrap aligned little
+; v15 = vload128 sp+264 // flags = notrap aligned little
+; v16 = vload128 sp+248 // flags = notrap aligned little
+; v17 = vload128 sp+232 // flags = notrap aligned little
+; v18 = vload128 sp+216 // flags = notrap aligned little
+; v19 = vload128 sp+200 // flags = notrap aligned little
+; v20 = vload128 sp+184 // flags = notrap aligned little
+; v21 = vload128 sp+168 // flags = notrap aligned little
+; v22 = vload128 sp+152 // flags = notrap aligned little
+; v23 = vload128 sp+136 // flags = notrap aligned little
+; v24 = vload128 sp+120 // flags = notrap aligned little
+; v25 = vload128 sp+104 // flags = notrap aligned little
+; v26 = vload128 sp+88 // flags = notrap aligned little
+; v27 = vload128 sp+72 // flags = notrap aligned little
+; v28 = vload128 sp+56 // flags = notrap aligned little
+; v29 = vload128 sp+40 // flags = notrap aligned little
+; v30 = vload128 sp+24 // flags = notrap aligned little
+; v31 = vload128 sp+8 // flags = notrap aligned little
; pop_frame_restore 912, {x16}
; ret
;
@@ -647,38 +647,38 @@ block0(v0: i64, v1: i64):
; fstore64le_o32 sp, 536, f29
; fstore64le_o32 sp, 528, f30
; fstore64le_o32 sp, 520, f31
-; vstore128le_o32 sp, 512, v0
-; vstore128le_o32 sp, 504, v1
-; vstore128le_o32 sp, 496, v2
-; vstore128le_o32 sp, 488, v3
-; vstore128le_o32 sp, 480, v4
-; vstore128le_o32 sp, 472, v5
-; vstore128le_o32 sp, 464, v6
-; vstore128le_o32 sp, 456, v7
-; vstore128le_o32 sp, 448, v8
-; vstore128le_o32 sp, 440, v9
-; vstore128le_o32 sp, 432, v10
-; vstore128le_o32 sp, 424, v11
-; vstore128le_o32 sp, 416, v12
-; vstore128le_o32 sp, 408, v13
-; vstore128le_o32 sp, 400, v14
-; vstore128le_o32 sp, 392, v15
-; vstore128le_o32 sp, 384, v16
-; vstore128le_o32 sp, 376, v17
-; vstore128le_o32 sp, 368, v18
-; vstore128le_o32 sp, 360, v19
-; vstore128le_o32 sp, 352, v20
-; vstore128le_o32 sp, 344, v21
-; vstore128le_o32 sp, 336, v22
-; vstore128le_o32 sp, 328, v23
-; vstore128le_o32 sp, 320, v24
-; vstore128le_o32 sp, 312, v25
-; vstore128le_o32 sp, 304, v26
-; vstore128le_o32 sp, 296, v27
-; vstore128le_o32 sp, 288, v28
-; vstore128le_o32 sp, 280, v29
-; vstore128le_o32 sp, 272, v30
-; vstore128le_o32 sp, 264, v31
+; vstore128le_o32 sp, 504, v0
+; vstore128le_o32 sp, 488, v1
+; vstore128le_o32 sp, 472, v2
+; vstore128le_o32 sp, 456, v3
+; vstore128le_o32 sp, 440, v4
+; vstore128le_o32 sp, 424, v5
+; vstore128le_o32 sp, 408, v6
+; vstore128le_o32 sp, 392, v7
+; vstore128le_o32 sp, 376, v8
+; vstore128le_o32 sp, 360, v9
+; vstore128le_o32 sp, 344, v10
+; vstore128le_o32 sp, 328, v11
+; vstore128le_o32 sp, 312, v12
+; vstore128le_o32 sp, 296, v13
+; vstore128le_o32 sp, 280, v14
+; vstore128le_o32 sp, 264, v15
+; vstore128le_o32 sp, 248, v16
+; vstore128le_o32 sp, 232, v17
+; vstore128le_o32 sp, 216, v18
+; vstore128le_o32 sp, 200, v19
+; vstore128le_o32 sp, 184, v20
+; vstore128le_o32 sp, 168, v21
+; vstore128le_o32 sp, 152, v22
+; vstore128le_o32 sp, 136, v23
+; vstore128le_o32 sp, 120, v24
+; vstore128le_o32 sp, 104, v25
+; vstore128le_o32 sp, 88, v26
+; vstore128le_o32 sp, 72, v27
+; vstore128le_o32 sp, 56, v28
+; vstore128le_o32 sp, 40, v29
+; vstore128le_o32 sp, 24, v30
+; vstore128le_o32 sp, 8, v31
; xmov x16, x1
; call_indirect_host 0
; xmov x0, x16
@@ -731,38 +731,38 @@ block0(v0: i64, v1: i64):
; fload64le_o32 f29, sp, 536
; fload64le_o32 f30, sp, 528
; fload64le_o32 f31, sp, 520
-; vload128le_o32 v0, sp, 512
-; vload128le_o32 v1, sp, 504
-; vload128le_o32 v2, sp, 496
-; vload128le_o32 v3, sp, 488
-; vload128le_o32 v4, sp, 480
-; vload128le_o32 v5, sp, 472
-; vload128le_o32 v6, sp, 464
-; vload128le_o32 v7, sp, 456
-; vload128le_o32 v8, sp, 448
-; vload128le_o32 v9, sp, 440
-; vload128le_o32 v10, sp, 432
-; vload128le_o32 v11, sp, 424
-; vload128le_o32 v12, sp, 416
-; vload128le_o32 v13, sp, 408
-; vload128le_o32 v14, sp, 400
-; vload128le_o32 v15, sp, 392
-; vload128le_o32 v16, sp, 384
-; vload128le_o32 v17, sp, 376
-; vload128le_o32 v18, sp, 368
-; vload128le_o32 v19, sp, 360
-; vload128le_o32 v20, sp, 352
-; vload128le_o32 v21, sp, 344
-; vload128le_o32 v22, sp, 336
-; vload128le_o32 v23, sp, 328
-; vload128le_o32 v24, sp, 320
-; vload128le_o32 v25, sp, 312
-; vload128le_o32 v26, sp, 304
-; vload128le_o32 v27, sp, 296
-; vload128le_o32 v28, sp, 288
-; vload128le_o32 v29, sp, 280
-; vload128le_o32 v30, sp, 272
-; vload128le_o32 v31, sp, 264
+; vload128le_o32 v0, sp, 504
+; vload128le_o32 v1, sp, 488
+; vload128le_o32 v2, sp, 472
+; vload128le_o32 v3, sp, 456
+; vload128le_o32 v4, sp, 440
+; vload128le_o32 v5, sp, 424
+; vload128le_o32 v6, sp, 408
+; vload128le_o32 v7, sp, 392
+; vload128le_o32 v8, sp, 376
+; vload128le_o32 v9, sp, 360
+; vload128le_o32 v10, sp, 344
+; vload128le_o32 v11, sp, 328
+; vload128le_o32 v12, sp, 312
+; vload128le_o32 v13, sp, 296
+; vload128le_o32 v14, sp, 280
+; vload128le_o32 v15, sp, 264
+; vload128le_o32 v16, sp, 248
+; vload128le_o32 v17, sp, 232
+; vload128le_o32 v18, sp, 216
+; vload128le_o32 v19, sp, 200
+; vload128le_o32 v20, sp, 184
+; vload128le_o32 v21, sp, 168
+; vload128le_o32 v22, sp, 152
+; vload128le_o32 v23, sp, 136
+; vload128le_o32 v24, sp, 120
+; vload128le_o32 v25, sp, 104
+; vload128le_o32 v26, sp, 88
+; vload128le_o32 v27, sp, 72
+; vload128le_o32 v28, sp, 56
+; vload128le_o32 v29, sp, 40
+; vload128le_o32 v30, sp, 24
+; vload128le_o32 v31, sp, 8
; pop_frame_restore 912, x16
; ret
diff --git a/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif b/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif
index 363a6c3538..0d6f760b25 100644
--- a/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif
+++ b/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif
@@ -87,38 +87,38 @@ block0(v0: i64):
; fstore64 sp+528, f29 // flags = notrap aligned
; fstore64 sp+520, f30 // flags = notrap aligned
; fstore64 sp+512, f31 // flags = notrap aligned
-; vstore128 sp+504, v0 // flags = notrap aligned little
-; vstore128 sp+496, v1 // flags = notrap aligned little
-; vstore128 sp+488, v2 // flags = notrap aligned little
-; vstore128 sp+480, v3 // flags = notrap aligned little
-; vstore128 sp+472, v4 // flags = notrap aligned little
-; vstore128 sp+464, v5 // flags = notrap aligned little
-; vstore128 sp+456, v6 // flags = notrap aligned little
-; vstore128 sp+448, v7 // flags = notrap aligned little
-; vstore128 sp+440, v8 // flags = notrap aligned little
-; vstore128 sp+432, v9 // flags = notrap aligned little
-; vstore128 sp+424, v10 // flags = notrap aligned little
-; vstore128 sp+416, v11 // flags = notrap aligned little
-; vstore128 sp+408, v12 // flags = notrap aligned little
-; vstore128 sp+400, v13 // flags = notrap aligned little
-; vstore128 sp+392, v14 // flags = notrap aligned little
-; vstore128 sp+384, v15 // flags = notrap aligned little
-; vstore128 sp+376, v16 // flags = notrap aligned little
-; vstore128 sp+368, v17 // flags = notrap aligned little
-; vstore128 sp+360, v18 // flags = notrap aligned little
-; vstore128 sp+352, v19 // flags = notrap aligned little
-; vstore128 sp+344, v20 // flags = notrap aligned little
-; vstore128 sp+336, v21 // flags = notrap aligned little
-; vstore128 sp+328, v22 // flags = notrap aligned little
-; vstore128 sp+320, v23 // flags = notrap aligned little
-; vstore128 sp+312, v24 // flags = notrap aligned little
-; vstore128 sp+304, v25 // flags = notrap aligned little
-; vstore128 sp+296, v26 // flags = notrap aligned little
-; vstore128 sp+288, v27 // flags = notrap aligned little
-; vstore128 sp+280, v28 // flags = notrap aligned little
-; vstore128 sp+272, v29 // flags = notrap aligned little
-; vstore128 sp+264, v30 // flags = notrap aligned little
-; vstore128 sp+256, v31 // flags = notrap aligned little
+; vstore128 sp+496, v0 // flags = notrap aligned little
+; vstore128 sp+480, v1 // flags = notrap aligned little
+; vstore128 sp+464, v2 // flags = notrap aligned little
+; vstore128 sp+448, v3 // flags = notrap aligned little
+; vstore128 sp+432, v4 // flags = notrap aligned little
+; vstore128 sp+416, v5 // flags = notrap aligned little
+; vstore128 sp+400, v6 // flags = notrap aligned little
+; vstore128 sp+384, v7 // flags = notrap aligned little
+; vstore128 sp+368, v8 // flags = notrap aligned little
+; vstore128 sp+352, v9 // flags = notrap aligned little
+; vstore128 sp+336, v10 // flags = notrap aligned little
+; vstore128 sp+320, v11 // flags = notrap aligned little
+; vstore128 sp+304, v12 // flags = notrap aligned little
+; vstore128 sp+288, v13 // flags = notrap aligned little
+; vstore128 sp+272, v14 // flags = notrap aligned little
+; vstore128 sp+256, v15 // flags = notrap aligned little
+; vstore128 sp+240, v16 // flags = notrap aligned little
+; vstore128 sp+224, v17 // flags = notrap aligned little
+; vstore128 sp+208, v18 // flags = notrap aligned little
+; vstore128 sp+192, v19 // flags = notrap aligned little
+; vstore128 sp+176, v20 // flags = notrap aligned little
+; vstore128 sp+160, v21 // flags = notrap aligned little
+; vstore128 sp+144, v22 // flags = notrap aligned little
+; vstore128 sp+128, v23 // flags = notrap aligned little
+; vstore128 sp+112, v24 // flags = notrap aligned little
+; vstore128 sp+96, v25 // flags = notrap aligned little
+; vstore128 sp+80, v26 // flags = notrap aligned little
+; vstore128 sp+64, v27 // flags = notrap aligned little
+; vstore128 sp+48, v28 // flags = notrap aligned little
+; vstore128 sp+32, v29 // flags = notrap aligned little
+; vstore128 sp+16, v30 // flags = notrap aligned little
+; vstore128 sp+0, v31 // flags = notrap aligned little
; block0:
; indirect_call_host CallInfo { dest: TestCase(%libcall), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [], clobbers: PRegSet { bits: [65535, 4294967295, 4294967295, 0] }, callee_conv: SystemV, caller_conv: PreserveAll, callee_pop_size: 0, try_call_info: None, patchable: false }
; x0 = xload64 sp+888 // flags = notrap aligned
@@ -169,38 +169,38 @@ block0(v0: i64):
; f29 = fload64 sp+528 // flags = notrap aligned
; f30 = fload64 sp+520 // flags = notrap aligned
; f31 = fload64 sp+512 // flags = notrap aligned
-; v0 = vload128 sp+504 // flags = notrap aligned little
-; v1 = vload128 sp+496 // flags = notrap aligned little
-; v2 = vload128 sp+488 // flags = notrap aligned little
-; v3 = vload128 sp+480 // flags = notrap aligned little
-; v4 = vload128 sp+472 // flags = notrap aligned little
-; v5 = vload128 sp+464 // flags = notrap aligned little
-; v6 = vload128 sp+456 // flags = notrap aligned little
-; v7 = vload128 sp+448 // flags = notrap aligned little
-; v8 = vload128 sp+440 // flags = notrap aligned little
-; v9 = vload128 sp+432 // flags = notrap aligned little
-; v10 = vload128 sp+424 // flags = notrap aligned little
-; v11 = vload128 sp+416 // flags = notrap aligned little
-; v12 = vload128 sp+408 // flags = notrap aligned little
-; v13 = vload128 sp+400 // flags = notrap aligned little
-; v14 = vload128 sp+392 // flags = notrap aligned little
-; v15 = vload128 sp+384 // flags = notrap aligned little
-; v16 = vload128 sp+376 // flags = notrap aligned little
-; v17 = vload128 sp+368 // flags = notrap aligned little
-; v18 = vload128 sp+360 // flags = notrap aligned little
-; v19 = vload128 sp+352 // flags = notrap aligned little
-; v20 = vload128 sp+344 // flags = notrap aligned little
-; v21 = vload128 sp+336 // flags = notrap aligned little
-; v22 = vload128 sp+328 // flags = notrap aligned little
-; v23 = vload128 sp+320 // flags = notrap aligned little
-; v24 = vload128 sp+312 // flags = notrap aligned little
-; v25 = vload128 sp+304 // flags = notrap aligned little
-; v26 = vload128 sp+296 // flags = notrap aligned little
-; v27 = vload128 sp+288 // flags = notrap aligned little
-; v28 = vload128 sp+280 // flags = notrap aligned little
-; v29 = vload128 sp+272 // flags = notrap aligned little
-; v30 = vload128 sp+264 // flags = notrap aligned little
-; v31 = vload128 sp+256 // flags = notrap aligned little
+; v0 = vload128 sp+496 // flags = notrap aligned little
+; v1 = vload128 sp+480 // flags = notrap aligned little
+; v2 = vload128 sp+464 // flags = notrap aligned little
+; v3 = vload128 sp+448 // flags = notrap aligned little
+; v4 = vload128 sp+432 // flags = notrap aligned little
+; v5 = vload128 sp+416 // flags = notrap aligned little
+; v6 = vload128 sp+400 // flags = notrap aligned little
+; v7 = vload128 sp+384 // flags = notrap aligned little
+; v8 = vload128 sp+368 // flags = notrap aligned little
+; v9 = vload128 sp+352 // flags = notrap aligned little
+; v10 = vload128 sp+336 // flags = notrap aligned little
+; v11 = vload128 sp+320 // flags = notrap aligned little
+; v12 = vload128 sp+304 // flags = notrap aligned little
+; v13 = vload128 sp+288 // flags = notrap aligned little
+; v14 = vload128 sp+272 // flags = notrap aligned little
+; v15 = vload128 sp+256 // flags = notrap aligned little
+; v16 = vload128 sp+240 // flags = notrap aligned little
+; v17 = vload128 sp+224 // flags = notrap aligned little
+; v18 = vload128 sp+208 // flags = notrap aligned little
+; v19 = vload128 sp+192 // flags = notrap aligned little
+; v20 = vload128 sp+176 // flags = notrap aligned little
+; v21 = vload128 sp+160 // flags = notrap aligned little
+; v22 = vload128 sp+144 // flags = notrap aligned little
+; v23 = vload128 sp+128 // flags = notrap aligned little
+; v24 = vload128 sp+112 // flags = notrap aligned little
+; v25 = vload128 sp+96 // flags = notrap aligned little
+; v26 = vload128 sp+80 // flags = notrap aligned little
+; v27 = vload128 sp+64 // flags = notrap aligned little
+; v28 = vload128 sp+48 // flags = notrap aligned little
+; v29 = vload128 sp+32 // flags = notrap aligned little
+; v30 = vload128 sp+16 // flags = notrap aligned little
+; v31 = vload128 sp+0 // flags = notrap aligned little
; pop_frame_restore 896, {}
; ret
;
@@ -254,38 +254,38 @@ block0(v0: i64):
; fstore64le_o32 sp, 528, f29
; fstore64le_o32 sp, 520, f30
; fstore64le_o32 sp, 512, f31
-; vstore128le_o32 sp, 504, v0
-; vstore128le_o32 sp, 496, v1
-; vstore128le_o32 sp, 488, v2
-; vstore128le_o32 sp, 480, v3
-; vstore128le_o32 sp, 472, v4
-; vstore128le_o32 sp, 464, v5
-; vstore128le_o32 sp, 456, v6
-; vstore128le_o32 sp, 448, v7
-; vstore128le_o32 sp, 440, v8
-; vstore128le_o32 sp, 432, v9
-; vstore128le_o32 sp, 424, v10
-; vstore128le_o32 sp, 416, v11
-; vstore128le_o32 sp, 408, v12
-; vstore128le_o32 sp, 400, v13
-; vstore128le_o32 sp, 392, v14
-; vstore128le_o32 sp, 384, v15
-; vstore128le_o32 sp, 376, v16
-; vstore128le_o32 sp, 368, v17
-; vstore128le_o32 sp, 360, v18
-; vstore128le_o32 sp, 352, v19
-; vstore128le_o32 sp, 344, v20
-; vstore128le_o32 sp, 336, v21
-; vstore128le_o32 sp, 328, v22
-; vstore128le_o32 sp, 320, v23
-; vstore128le_o32 sp, 312, v24
-; vstore128le_o32 sp, 304, v25
-; vstore128le_o32 sp, 296, v26
-; vstore128le_o32 sp, 288, v27
-; vstore128le_o32 sp, 280, v28
-; vstore128le_o32 sp, 272, v29
-; vstore128le_o32 sp, 264, v30
-; vstore128le_o32 sp, 256, v31
+; vstore128le_o32 sp, 496, v0
+; vstore128le_o32 sp, 480, v1
+; vstore128le_o32 sp, 464, v2
+; vstore128le_o32 sp, 448, v3
+; vstore128le_o32 sp, 432, v4
+; vstore128le_o32 sp, 416, v5
+; vstore128le_o32 sp, 400, v6
+; vstore128le_o32 sp, 384, v7
+; vstore128le_o32 sp, 368, v8
+; vstore128le_o32 sp, 352, v9
+; vstore128le_o32 sp, 336, v10
+; vstore128le_o32 sp, 320, v11
+; vstore128le_o32 sp, 304, v12
+; vstore128le_o32 sp, 288, v13
+; vstore128le_o32 sp, 272, v14
+; vstore128le_o32 sp, 256, v15
+; vstore128le_o32 sp, 240, v16
+; vstore128le_o32 sp, 224, v17
+; vstore128le_o32 sp, 208, v18
+; vstore128le_o32 sp, 192, v19
+; vstore128le_o32 sp, 176, v20
+; vstore128le_o32 sp, 160, v21
+; vstore128le_o32 sp, 144, v22
+; vstore128le_o32 sp, 128, v23
+; vstore128le_o32 sp, 112, v24
+; vstore128le_o32 sp, 96, v25
+; vstore128le_o32 sp, 80, v26
+; vstore128le_o32 sp, 64, v27
+; vstore128le_o32 sp, 48, v28
+; vstore128le_o32 sp, 32, v29
+; vstore128le_o32 sp, 16, v30
+; vstore128le_o32 sp, 0, v31
; call_indirect_host 0
; xload64le_o32 x0, sp, 888
; xload64le_o32 x1, sp, 880
@@ -335,38 +335,38 @@ block0(v0: i64):
; fload64le_o32 f29, sp, 528
; fload64le_o32 f30, sp, 520
; fload64le_o32 f31, sp, 512
-; vload128le_o32 v0, sp, 504
-; vload128le_o32 v1, sp, 496
-; vload128le_o32 v2, sp, 488
-; vload128le_o32 v3, sp, 480
-; vload128le_o32 v4, sp, 472
-; vload128le_o32 v5, sp, 464
-; vload128le_o32 v6, sp, 456
-; vload128le_o32 v7, sp, 448
-; vload128le_o32 v8, sp, 440
-; vload128le_o32 v9, sp, 432
-; vload128le_o32 v10, sp, 424
-; vload128le_o32 v11, sp, 416
-; vload128le_o32 v12, sp, 408
-; vload128le_o32 v13, sp, 400
-; vload128le_o32 v14, sp, 392
-; vload128le_o32 v15, sp, 384
-; vload128le_o32 v16, sp, 376
-; vload128le_o32 v17, sp, 368
-; vload128le_o32 v18, sp, 360
-; vload128le_o32 v19, sp, 352
-; vload128le_o32 v20, sp, 344
-; vload128le_o32 v21, sp, 336
-; vload128le_o32 v22, sp, 328
-; vload128le_o32 v23, sp, 320
-; vload128le_o32 v24, sp, 312
-; vload128le_o32 v25, sp, 304
-; vload128le_o32 v26, sp, 296
-; vload128le_o32 v27, sp, 288
-; vload128le_o32 v28, sp, 280
-; vload128le_o32 v29, sp, 272
-; vload128le_o32 v30, sp, 264
-; vload128le_o32 v31, sp, 256
+; vload128le_o32 v0, sp, 496
+; vload128le_o32 v1, sp, 480
+; vload128le_o32 v2, sp, 464
+; vload128le_o32 v3, sp, 448
+; vload128le_o32 v4, sp, 432
+; vload128le_o32 v5, sp, 416
+; vload128le_o32 v6, sp, 400
+; vload128le_o32 v7, sp, 384
+; vload128le_o32 v8, sp, 368
+; vload128le_o32 v9, sp, 352
+; vload128le_o32 v10, sp, 336
+; vload128le_o32 v11, sp, 320
+; vload128le_o32 v12, sp, 304
+; vload128le_o32 v13, sp, 288
+; vload128le_o32 v14, sp, 272
+; vload128le_o32 v15, sp, 256
+; vload128le_o32 v16, sp, 240
+; vload128le_o32 v17, sp, 224
+; vload128le_o32 v18, sp, 208
+; vload128le_o32 v19, sp, 192
+; vload128le_o32 v20, sp, 176
+; vload128le_o32 v21, sp, 160
+; vload128le_o32 v22, sp, 144
+; vload128le_o32 v23, sp, 128
+; vload128le_o32 v24, sp, 112
+; vload128le_o32 v25, sp, 96
+; vload128le_o32 v26, sp, 80
+; vload128le_o32 v27, sp, 64
+; vload128le_o32 v28, sp, 48
+; vload128le_o32 v29, sp, 32
+; vload128le_o32 v30, sp, 16
+; vload128le_o32 v31, sp, 0
; pop_frame_restore 896,
; ret
@@ -448,38 +448,38 @@ block0(v0: i64, v1: i64):
; fstore64 sp+536, f29 // flags = notrap aligned
; fstore64 sp+528, f30 // flags = notrap aligned
; fstore64 sp+520, f31 // flags = notrap aligned
-; vstore128 sp+512, v0 // flags = notrap aligned little
-; vstore128 sp+504, v1 // flags = notrap aligned little
-; vstore128 sp+496, v2 // flags = notrap aligned little
-; vstore128 sp+488, v3 // flags = notrap aligned little
-; vstore128 sp+480, v4 // flags = notrap aligned little
-; vstore128 sp+472, v5 // flags = notrap aligned little
-; vstore128 sp+464, v6 // flags = notrap aligned little
-; vstore128 sp+456, v7 // flags = notrap aligned little
-; vstore128 sp+448, v8 // flags = notrap aligned little
-; vstore128 sp+440, v9 // flags = notrap aligned little
-; vstore128 sp+432, v10 // flags = notrap aligned little
-; vstore128 sp+424, v11 // flags = notrap aligned little
-; vstore128 sp+416, v12 // flags = notrap aligned little
-; vstore128 sp+408, v13 // flags = notrap aligned little
-; vstore128 sp+400, v14 // flags = notrap aligned little
-; vstore128 sp+392, v15 // flags = notrap aligned little
-; vstore128 sp+384, v16 // flags = notrap aligned little
-; vstore128 sp+376, v17 // flags = notrap aligned little
-; vstore128 sp+368, v18 // flags = notrap aligned little
-; vstore128 sp+360, v19 // flags = notrap aligned little
-; vstore128 sp+352, v20 // flags = notrap aligned little
-; vstore128 sp+344, v21 // flags = notrap aligned little
-; vstore128 sp+336, v22 // flags = notrap aligned little
-; vstore128 sp+328, v23 // flags = notrap aligned little
-; vstore128 sp+320, v24 // flags = notrap aligned little
-; vstore128 sp+312, v25 // flags = notrap aligned little
-; vstore128 sp+304, v26 // flags = notrap aligned little
-; vstore128 sp+296, v27 // flags = notrap aligned little
-; vstore128 sp+288, v28 // flags = notrap aligned little
-; vstore128 sp+280, v29 // flags = notrap aligned little
-; vstore128 sp+272, v30 // flags = notrap aligned little
-; vstore128 sp+264, v31 // flags = notrap aligned little
+; vstore128 sp+504, v0 // flags = notrap aligned little
+; vstore128 sp+488, v1 // flags = notrap aligned little
+; vstore128 sp+472, v2 // flags = notrap aligned little
+; vstore128 sp+456, v3 // flags = notrap aligned little
+; vstore128 sp+440, v4 // flags = notrap aligned little
+; vstore128 sp+424, v5 // flags = notrap aligned little
+; vstore128 sp+408, v6 // flags = notrap aligned little
+; vstore128 sp+392, v7 // flags = notrap aligned little
+; vstore128 sp+376, v8 // flags = notrap aligned little
+; vstore128 sp+360, v9 // flags = notrap aligned little
+; vstore128 sp+344, v10 // flags = notrap aligned little
+; vstore128 sp+328, v11 // flags = notrap aligned little
+; vstore128 sp+312, v12 // flags = notrap aligned little
+; vstore128 sp+296, v13 // flags = notrap aligned little
+; vstore128 sp+280, v14 // flags = notrap aligned little
+; vstore128 sp+264, v15 // flags = notrap aligned little
+; vstore128 sp+248, v16 // flags = notrap aligned little
+; vstore128 sp+232, v17 // flags = notrap aligned little
+; vstore128 sp+216, v18 // flags = notrap aligned little
+; vstore128 sp+200, v19 // flags = notrap aligned little
+; vstore128 sp+184, v20 // flags = notrap aligned little
+; vstore128 sp+168, v21 // flags = notrap aligned little
+; vstore128 sp+152, v22 // flags = notrap aligned little
+; vstore128 sp+136, v23 // flags = notrap aligned little
+; vstore128 sp+120, v24 // flags = notrap aligned little
+; vstore128 sp+104, v25 // flags = notrap aligned little
+; vstore128 sp+88, v26 // flags = notrap aligned little
+; vstore128 sp+72, v27 // flags = notrap aligned little
+; vstore128 sp+56, v28 // flags = notrap aligned little
+; vstore128 sp+40, v29 // flags = notrap aligned little
+; vstore128 sp+24, v30 // flags = notrap aligned little
+; vstore128 sp+8, v31 // flags = notrap aligned little
; block0:
; xmov x16, x1
; indirect_call_host CallInfo { dest: TestCase(%host), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [], clobbers: PRegSet { bits: [65535, 4294967295, 4294967295, 0] }, callee_conv: SystemV, caller_conv: PreserveAll, callee_pop_size: 0, try_call_info: None, patchable: false }
@@ -533,38 +533,38 @@ block0(v0: i64, v1: i64):
; f29 = fload64 sp+536 // flags = notrap aligned
; f30 = fload64 sp+528 // flags = notrap aligned
; f31 = fload64 sp+520 // flags = notrap aligned
-; v0 = vload128 sp+512 // flags = notrap aligned little
-; v1 = vload128 sp+504 // flags = notrap aligned little
-; v2 = vload128 sp+496 // flags = notrap aligned little
-; v3 = vload128 sp+488 // flags = notrap aligned little
-; v4 = vload128 sp+480 // flags = notrap aligned little
-; v5 = vload128 sp+472 // flags = notrap aligned little
-; v6 = vload128 sp+464 // flags = notrap aligned little
-; v7 = vload128 sp+456 // flags = notrap aligned little
-; v8 = vload128 sp+448 // flags = notrap aligned little
-; v9 = vload128 sp+440 // flags = notrap aligned little
-; v10 = vload128 sp+432 // flags = notrap aligned little
-; v11 = vload128 sp+424 // flags = notrap aligned little
-; v12 = vload128 sp+416 // flags = notrap aligned little
-; v13 = vload128 sp+408 // flags = notrap aligned little
-; v14 = vload128 sp+400 // flags = notrap aligned little
-; v15 = vload128 sp+392 // flags = notrap aligned little
-; v16 = vload128 sp+384 // flags = notrap aligned little
-; v17 = vload128 sp+376 // flags = notrap aligned little
-; v18 = vload128 sp+368 // flags = notrap aligned little
-; v19 = vload128 sp+360 // flags = notrap aligned little
-; v20 = vload128 sp+352 // flags = notrap aligned little
-; v21 = vload128 sp+344 // flags = notrap aligned little
-; v22 = vload128 sp+336 // flags = notrap aligned little
-; v23 = vload128 sp+328 // flags = notrap aligned little
-; v24 = vload128 sp+320 // flags = notrap aligned little
-; v25 = vload128 sp+312 // flags = notrap aligned little
-; v26 = vload128 sp+304 // flags = notrap aligned little
-; v27 = vload128 sp+296 // flags = notrap aligned little
-; v28 = vload128 sp+288 // flags = notrap aligned little
-; v29 = vload128 sp+280 // flags = notrap aligned little
-; v30 = vload128 sp+272 // flags = notrap aligned little
-; v31 = vload128 sp+264 // flags = notrap aligned little
+; v0 = vload128 sp+504 // flags = notrap aligned little
+; v1 = vload128 sp+488 // flags = notrap aligned little
+; v2 = vload128 sp+472 // flags = notrap aligned little
+; v3 = vload128 sp+456 // flags = notrap aligned little
+; v4 = vload128 sp+440 // flags = notrap aligned little
+; v5 = vload128 sp+424 // flags = notrap aligned little
+; v6 = vload128 sp+408 // flags = notrap aligned little
+; v7 = vload128 sp+392 // flags = notrap aligned little
+; v8 = vload128 sp+376 // flags = notrap aligned little
+; v9 = vload128 sp+360 // flags = notrap aligned little
+; v10 = vload128 sp+344 // flags = notrap aligned little
+; v11 = vload128 sp+328 // flags = notrap aligned little
+; v12 = vload128 sp+312 // flags = notrap aligned little
+; v13 = vload128 sp+296 // flags = notrap aligned little
+; v14 = vload128 sp+280 // flags = notrap aligned little
+; v15 = vload128 sp+264 // flags = notrap aligned little
+; v16 = vload128 sp+248 // flags = notrap aligned little
+; v17 = vload128 sp+232 // flags = notrap aligned little
+; v18 = vload128 sp+216 // flags = notrap aligned little
+; v19 = vload128 sp+200 // flags = notrap aligned little
+; v20 = vload128 sp+184 // flags = notrap aligned little
+; v21 = vload128 sp+168 // flags = notrap aligned little
+; v22 = vload128 sp+152 // flags = notrap aligned little
+; v23 = vload128 sp+136 // flags = notrap aligned little
+; v24 = vload128 sp+120 // flags = notrap aligned little
+; v25 = vload128 sp+104 // flags = notrap aligned little
+; v26 = vload128 sp+88 // flags = notrap aligned little
+; v27 = vload128 sp+72 // flags = notrap aligned little
+; v28 = vload128 sp+56 // flags = notrap aligned little
+; v29 = vload128 sp+40 // flags = notrap aligned little
+; v30 = vload128 sp+24 // flags = notrap aligned little
+; v31 = vload128 sp+8 // flags = notrap aligned little
; pop_frame_restore 912, {x16}
; ret
;
@@ -618,38 +618,38 @@ block0(v0: i64, v1: i64):
; fstore64le_o32 sp, 536, f29
; fstore64le_o32 sp, 528, f30
; fstore64le_o32 sp, 520, f31
-; vstore128le_o32 sp, 512, v0
-; vstore128le_o32 sp, 504, v1
-; vstore128le_o32 sp, 496, v2
-; vstore128le_o32 sp, 488, v3
-; vstore128le_o32 sp, 480, v4
-; vstore128le_o32 sp, 472, v5
-; vstore128le_o32 sp, 464, v6
-; vstore128le_o32 sp, 456, v7
-; vstore128le_o32 sp, 448, v8
-; vstore128le_o32 sp, 440, v9
-; vstore128le_o32 sp, 432, v10
-; vstore128le_o32 sp, 424, v11
-; vstore128le_o32 sp, 416, v12
-; vstore128le_o32 sp, 408, v13
-; vstore128le_o32 sp, 400, v14
-; vstore128le_o32 sp, 392, v15
-; vstore128le_o32 sp, 384, v16
-; vstore128le_o32 sp, 376, v17
-; vstore128le_o32 sp, 368, v18
-; vstore128le_o32 sp, 360, v19
-; vstore128le_o32 sp, 352, v20
-; vstore128le_o32 sp, 344, v21
-; vstore128le_o32 sp, 336, v22
-; vstore128le_o32 sp, 328, v23
-; vstore128le_o32 sp, 320, v24
-; vstore128le_o32 sp, 312, v25
-; vstore128le_o32 sp, 304, v26
-; vstore128le_o32 sp, 296, v27
-; vstore128le_o32 sp, 288, v28
-; vstore128le_o32 sp, 280, v29
-; vstore128le_o32 sp, 272, v30
-; vstore128le_o32 sp, 264, v31
+; vstore128le_o32 sp, 504, v0
+; vstore128le_o32 sp, 488, v1
+; vstore128le_o32 sp, 472, v2
+; vstore128le_o32 sp, 456, v3
+; vstore128le_o32 sp, 440, v4
+; vstore128le_o32 sp, 424, v5
+; vstore128le_o32 sp, 408, v6
+; vstore128le_o32 sp, 392, v7
+; vstore128le_o32 sp, 376, v8
+; vstore128le_o32 sp, 360, v9
+; vstore128le_o32 sp, 344, v10
+; vstore128le_o32 sp, 328, v11
+; vstore128le_o32 sp, 312, v12
+; vstore128le_o32 sp, 296, v13
+; vstore128le_o32 sp, 280, v14
+; vstore128le_o32 sp, 264, v15
+; vstore128le_o32 sp, 248, v16
+; vstore128le_o32 sp, 232, v17
+; vstore128le_o32 sp, 216, v18
+; vstore128le_o32 sp, 200, v19
+; vstore128le_o32 sp, 184, v20
+; vstore128le_o32 sp, 168, v21
+; vstore128le_o32 sp, 152, v22
+; vstore128le_o32 sp, 136, v23
+; vstore128le_o32 sp, 120, v24
+; vstore128le_o32 sp, 104, v25
+; vstore128le_o32 sp, 88, v26
+; vstore128le_o32 sp, 72, v27
+; vstore128le_o32 sp, 56, v28
+; vstore128le_o32 sp, 40, v29
+; vstore128le_o32 sp, 24, v30
+; vstore128le_o32 sp, 8, v31
; xmov x16, x1
; call_indirect_host 0
; xmov x0, x16
@@ -702,38 +702,38 @@ block0(v0: i64, v1: i64):
; fload64le_o32 f29, sp, 536
; fload64le_o32 f30, sp, 528
; fload64le_o32 f31, sp, 520
-; vload128le_o32 v0, sp, 512
-; vload128le_o32 v1, sp, 504
-; vload128le_o32 v2, sp, 496
-; vload128le_o32 v3, sp, 488
-; vload128le_o32 v4, sp, 480
-; vload128le_o32 v5, sp, 472
-; vload128le_o32 v6, sp, 464
-; vload128le_o32 v7, sp, 456
-; vload128le_o32 v8, sp, 448
-; vload128le_o32 v9, sp, 440
-; vload128le_o32 v10, sp, 432
-; vload128le_o32 v11, sp, 424
-; vload128le_o32 v12, sp, 416
-; vload128le_o32 v13, sp, 408
-; vload128le_o32 v14, sp, 400
-; vload128le_o32 v15, sp, 392
-; vload128le_o32 v16, sp, 384
-; vload128le_o32 v17, sp, 376
-; vload128le_o32 v18, sp, 368
-; vload128le_o32 v19, sp, 360
-; vload128le_o32 v20, sp, 352
-; vload128le_o32 v21, sp, 344
-; vload128le_o32 v22, sp, 336
-; vload128le_o32 v23, sp, 328
-; vload128le_o32 v24, sp, 320
-; vload128le_o32 v25, sp, 312
-; vload128le_o32 v26, sp, 304
-; vload128le_o32 v27, sp, 296
-; vload128le_o32 v28, sp, 288
-; vload128le_o32 v29, sp, 280
-; vload128le_o32 v30, sp, 272
-; vload128le_o32 v31, sp, 264
+; vload128le_o32 v0, sp, 504
+; vload128le_o32 v1, sp, 488
+; vload128le_o32 v2, sp, 472
+; vload128le_o32 v3, sp, 456
+; vload128le_o32 v4, sp, 440
+; vload128le_o32 v5, sp, 424
+; vload128le_o32 v6, sp, 408
+; vload128le_o32 v7, sp, 392
+; vload128le_o32 v8, sp, 376
+; vload128le_o32 v9, sp, 360
+; vload128le_o32 v10, sp, 344
+; vload128le_o32 v11, sp, 328
+; vload128le_o32 v12, sp, 312
+; vload128le_o32 v13, sp, 296
+; vload128le_o32 v14, sp, 280
+; vload128le_o32 v15, sp, 264
+; vload128le_o32 v16, sp, 248
+; vload128le_o32 v17, sp, 232
+; vload128le_o32 v18, sp, 216
+; vload128le_o32 v19, sp, 200
+; vload128le_o32 v20, sp, 184
+; vload128le_o32 v21, sp, 168
+; vload128le_o32 v22, sp, 152
+; vload128le_o32 v23, sp, 136
+; vload128le_o32 v24, sp, 120
+; vload128le_o32 v25, sp, 104
+; vload128le_o32 v26, sp, 88
+; vload128le_o32 v27, sp, 72
+; vload128le_o32 v28, sp, 56
+; vload128le_o32 v29, sp, 40
+; vload128le_o32 v30, sp, 24
+; vload128le_o32 v31, sp, 8
; pop_frame_restore 912, x16
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/preserve-all-probestack.clif b/cranelift/filetests/filetests/isa/riscv64/preserve-all-probestack.clif
new file mode 100644
index 0000000000..b59c8ff119
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/preserve-all-probestack.clif
@@ -0,0 +1,140 @@
+; Inline stack probes in a `preserve_all` function must not clobber any
+; allocatable register, since they run before registers are saved. Other ABIs
+; use %t3 (x28, allocatable) as the probe temporary, and a probe loop for
+; four or more probes; `preserve_all` always uses unrolled probes with the
+; non-allocatable spill temporary %t5 (x30).
+test compile precise-output
+set enable_probestack=true
+set probestack_strategy=inline
+target riscv64
+
+function %preserve_all(i64) preserve_all {
+ ss0 = explicit_slot 20480
+block0(v0: i64):
+ v1 = stack_addr.i64 ss0
+ store v0, v1
+ return
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
+; lui t5,-1
+; add sp,sp,t5
+; sw zero,0(sp)
+; add sp,sp,t5
+; sw zero,0(sp)
+; add sp,sp,t5
+; sw zero,0(sp)
+; add sp,sp,t5
+; sw zero,0(sp)
+; add sp,sp,t5
+; sw zero,0(sp)
+; lui t6,5
+; add sp,sp,t6
+; lui t6,-5
+; addi t6,t6,-16
+; add sp,sp,t6
+; sd a0,20488(sp)
+; block0:
+; sd a0,0(slot)
+; ld a0,20488(sp)
+; lui t6,5
+; addi t6,t6,16
+; add sp,sp,t6
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; lui t5, 0xfffff
+; add sp, sp, t5
+; sw zero, 0(sp)
+; add sp, sp, t5
+; sw zero, 0(sp)
+; add sp, sp, t5
+; sw zero, 0(sp)
+; add sp, sp, t5
+; sw zero, 0(sp)
+; add sp, sp, t5
+; sw zero, 0(sp)
+; lui t6, 5
+; add sp, sp, t6
+; lui t6, 0xffffb
+; addi t6, t6, -0x10
+; add sp, sp, t6
+; lui t6, 5
+; addi t6, t6, 8
+; add t6, t6, sp
+; sd a0, 0(t6)
+; block1: ; offset 0x60
+; sd a0, 0(sp) ; trap: heap_oob
+; lui t6, 5
+; addi t6, t6, 8
+; add t6, t6, sp
+; ld a0, 0(t6)
+; lui t6, 5
+; addi t6, t6, 0x10
+; add sp, sp, t6
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
+function %system_v(i64) system_v {
+ ss0 = explicit_slot 20480
+block0(v0: i64):
+ v1 = stack_addr.i64 ss0
+ store v0, v1
+ return
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
+; inline_stack_probe##guard_size=4096 probe_count=5 tmp=t3
+; lui t6,-5
+; add sp,sp,t6
+; block0:
+; sd a0,0(slot)
+; lui t6,5
+; add sp,sp,t6
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; lui t6, 5
+; lui t3, 1
+; bgeu t3, t6, 0x14
+; sub t5, sp, t6
+; sb zero, 0(t5) ; trap: heap_oob
+; sub t6, t6, t3
+; j -0x10
+; lui t6, 0xffffb
+; add sp, sp, t6
+; block1: ; offset 0x34
+; sd a0, 0(sp) ; trap: heap_oob
+; lui t6, 5
+; add sp, sp, t6
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/x64/preserve-all-probestack.clif b/cranelift/filetests/filetests/isa/x64/preserve-all-probestack.clif
new file mode 100644
index 0000000000..1c3236fcef
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/preserve-all-probestack.clif
@@ -0,0 +1,105 @@
+; Inline stack probes in a `preserve_all` function must not clobber any
+; registers, since they run before registers are saved. With five or more
+; probes, other ABIs use a probe loop with %r11 as a temporary;
+; `preserve_all` always uses the unrolled probes, which use no registers.
+test compile precise-output
+set enable_probestack=true
+set probestack_strategy=inline
+target x86_64
+
+function %preserve_all(i64) preserve_all {
+ ss0 = explicit_slot 20480
+block0(v0: i64):
+ v1 = stack_addr.i64 ss0
+ store v0, v1
+ return
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x1000, %rsp
+; movl $0x0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0x0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0x0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0x0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0x0, (%rsp)
+; addq $0x5000, %rsp
+; subq $0x5010, %rsp
+; movq %rdi, 0x5000(%rsp)
+; block0:
+; movq %rdi, <offset:1>+(%rsp)
+; movq 0x5000(%rsp), %rdi
+; addq $0x5010, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; addq $0x5000, %rsp
+; subq $0x5010, %rsp
+; movq %rdi, 0x5000(%rsp)
+; block1: ; offset 0x60
+; movq %rdi, (%rsp)
+; movq 0x5000(%rsp), %rdi
+; addq $0x5010, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %system_v(i64) system_v {
+ ss0 = explicit_slot 20480
+block0(v0: i64):
+ v1 = stack_addr.i64 ss0
+ store v0, v1
+ return
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; stack_probe_loop %r11, frame_size=20480, guard_size=4096
+; subq $0x5000, %rsp
+; block0:
+; movq %rdi, <offset:1>+(%rsp)
+; addq $0x5000, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; movq %rsp, %r11
+; subq $0x5000, %r11
+; subq $0x1000, %rsp
+; movl $0, (%rsp)
+; cmpq %rsp, %r11
+; jne 0xe
+; addq $0x5000, %rsp
+; subq $0x5000, %rsp
+; block1: ; offset 0x33
+; movq %rdi, (%rsp)
+; addq $0x5000, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/runtests/preserve-all-probestack.clif b/cranelift/filetests/filetests/runtests/preserve-all-probestack.clif
new file mode 100644
index 0000000000..1beb2cba70
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/preserve-all-probestack.clif
@@ -0,0 +1,65 @@
+; A `preserve_all` callee with a frame large enough for an inline stack-probe
+; loop must not clobber any of the caller's registers: the caller keeps
+; values live across the call in registers, including ones that other ABIs
+; treat as caller-saved scratch registers (e.g. %r11 on x86-64).
+test interpret
+test run
+set enable_probestack=true
+set probestack_strategy=inline
+target x86_64
+target aarch64
+
+function %callee(i64) preserve_all {
+ ss0 = explicit_slot 100000
+block0(v0: i64):
+ v1 = stack_addr.i64 ss0
+ store v0, v1
+ return
+}
+
+function %caller(i64) -> i64 {
+ fn0 = colocated %callee(i64) preserve_all
+block0(v0: i64):
+ v101 = iconst.i64 4369
+ v1 = imul v0, v101
+ v102 = iconst.i64 8738
+ v2 = imul v0, v102
+ v103 = iconst.i64 13107
+ v3 = imul v0, v103
+ v104 = iconst.i64 17476
+ v4 = imul v0, v104
+ v105 = iconst.i64 21845
+ v5 = imul v0, v105
+ v106 = iconst.i64 26214
+ v6 = imul v0, v106
+ v107 = iconst.i64 30583
+ v7 = imul v0, v107
+ v108 = iconst.i64 34952
+ v8 = imul v0, v108
+ v109 = iconst.i64 39321
+ v9 = imul v0, v109
+ v110 = iconst.i64 43690
+ v10 = imul v0, v110
+ v111 = iconst.i64 48059
+ v11 = imul v0, v111
+ v112 = iconst.i64 52428
+ v12 = imul v0, v112
+ v113 = iconst.i64 56797
+ v13 = imul v0, v113
+ call fn0(v0)
+ v200 = iadd v1, v2
+ v201 = iadd v200, v3
+ v202 = iadd v201, v4
+ v203 = iadd v202, v5
+ v204 = iadd v203, v6
+ v205 = iadd v204, v7
+ v206 = iadd v205, v8
+ v207 = iadd v206, v9
+ v208 = iadd v207, v10
+ v209 = iadd v208, v11
+ v210 = iadd v209, v12
+ v211 = iadd v210, v13
+ return v211
+}
+; run: %caller(1) == 397579
+; run: %caller(3) == 1192737
diff --git a/cranelift/filetests/filetests/runtests/preserve-all-vector-saves.clif b/cranelift/filetests/filetests/runtests/preserve-all-vector-saves.clif
new file mode 100644
index 0000000000..aaaeb97f61
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/preserve-all-vector-saves.clif
@@ -0,0 +1,168 @@
+; A `preserve_all` callee that clobbers every vector register must preserve
+; the caller's float and vector values, including all 16 bytes of each
+; vector register.
+test run
+target x86_64
+target aarch64
+target pulley64
+target pulley64be
+
+function %callee(i64) preserve_all {
+block0(v0: i64):
+ v10 = iconst.i64 1
+ v100 = splat.i64x2 v10
+ v11 = iconst.i64 2
+ v101 = splat.i64x2 v11
+ v12 = iconst.i64 3
+ v102 = splat.i64x2 v12
+ v13 = iconst.i64 4
+ v103 = splat.i64x2 v13
+ v14 = iconst.i64 5
+ v104 = splat.i64x2 v14
+ v15 = iconst.i64 6
+ v105 = splat.i64x2 v15
+ v16 = iconst.i64 7
+ v106 = splat.i64x2 v16
+ v17 = iconst.i64 8
+ v107 = splat.i64x2 v17
+ v18 = iconst.i64 9
+ v108 = splat.i64x2 v18
+ v19 = iconst.i64 10
+ v109 = splat.i64x2 v19
+ v20 = iconst.i64 11
+ v110 = splat.i64x2 v20
+ v21 = iconst.i64 12
+ v111 = splat.i64x2 v21
+ v22 = iconst.i64 13
+ v112 = splat.i64x2 v22
+ v23 = iconst.i64 14
+ v113 = splat.i64x2 v23
+ v24 = iconst.i64 15
+ v114 = splat.i64x2 v24
+ v25 = iconst.i64 16
+ v115 = splat.i64x2 v25
+ v26 = iconst.i64 17
+ v116 = splat.i64x2 v26
+ v27 = iconst.i64 18
+ v117 = splat.i64x2 v27
+ v28 = iconst.i64 19
+ v118 = splat.i64x2 v28
+ v29 = iconst.i64 20
+ v119 = splat.i64x2 v29
+ v30 = iconst.i64 21
+ v120 = splat.i64x2 v30
+ v31 = iconst.i64 22
+ v121 = splat.i64x2 v31
+ v32 = iconst.i64 23
+ v122 = splat.i64x2 v32
+ v33 = iconst.i64 24
+ v123 = splat.i64x2 v33
+ v34 = iconst.i64 25
+ v124 = splat.i64x2 v34
+ v35 = iconst.i64 26
+ v125 = splat.i64x2 v35
+ v36 = iconst.i64 27
+ v126 = splat.i64x2 v36
+ v37 = iconst.i64 28
+ v127 = splat.i64x2 v37
+ v38 = iconst.i64 29
+ v128 = splat.i64x2 v38
+ v39 = iconst.i64 30
+ v129 = splat.i64x2 v39
+ v40 = iconst.i64 31
+ v130 = splat.i64x2 v40
+ v41 = iconst.i64 32
+ v131 = splat.i64x2 v41
+ v201 = iadd v100, v101
+ v202 = iadd v201, v102
+ v203 = iadd v202, v103
+ v204 = iadd v203, v104
+ v205 = iadd v204, v105
+ v206 = iadd v205, v106
+ v207 = iadd v206, v107
+ v208 = iadd v207, v108
+ v209 = iadd v208, v109
+ v210 = iadd v209, v110
+ v211 = iadd v210, v111
+ v212 = iadd v211, v112
+ v213 = iadd v212, v113
+ v214 = iadd v213, v114
+ v215 = iadd v214, v115
+ v216 = iadd v215, v116
+ v217 = iadd v216, v117
+ v218 = iadd v217, v118
+ v219 = iadd v218, v119
+ v220 = iadd v219, v120
+ v221 = iadd v220, v121
+ v222 = iadd v221, v122
+ v223 = iadd v222, v123
+ v224 = iadd v223, v124
+ v225 = iadd v224, v125
+ v226 = iadd v225, v126
+ v227 = iadd v226, v127
+ v228 = iadd v227, v128
+ v229 = iadd v228, v129
+ v230 = iadd v229, v130
+ v231 = iadd v230, v131
+ v300 = extractlane v231, 0
+ store notrap aligned v300, v0
+ return
+}
+
+function %caller(i64, f64) -> f64 {
+ ss0 = explicit_slot 16
+ fn0 = colocated %callee(i64) preserve_all
+block0(v0: i64, v1: f64):
+ v2 = stack_addr.i64 ss0
+ v10 = f64const 0x1.0
+ v20 = fadd v1, v10
+ v11 = f64const 0x2.0
+ v21 = fadd v1, v11
+ v12 = f64const 0x3.0
+ v22 = fadd v1, v12
+ v13 = f64const 0x4.0
+ v23 = fadd v1, v13
+ v14 = f64const 0x5.0
+ v24 = fadd v1, v14
+ v15 = f64const 0x6.0
+ v25 = fadd v1, v15
+ v16 = f64const 0x7.0
+ v26 = fadd v1, v16
+ v17 = f64const 0x8.0
+ v27 = fadd v1, v17
+ v30 = scalar_to_vector.f64x2 v20
+ v40 = splat.f64x2 v20
+ v31 = scalar_to_vector.f64x2 v21
+ v41 = splat.f64x2 v21
+ v32 = scalar_to_vector.f64x2 v22
+ v42 = splat.f64x2 v22
+ v33 = scalar_to_vector.f64x2 v23
+ v43 = splat.f64x2 v23
+ v34 = scalar_to_vector.f64x2 v24
+ v44 = splat.f64x2 v24
+ v35 = scalar_to_vector.f64x2 v25
+ v45 = splat.f64x2 v25
+ v36 = scalar_to_vector.f64x2 v26
+ v46 = splat.f64x2 v26
+ v37 = scalar_to_vector.f64x2 v27
+ v47 = splat.f64x2 v27
+ call fn0(v2)
+ v50 = fadd v20, v21
+ v51 = fadd v50, v22
+ v52 = fadd v51, v23
+ v53 = fadd v52, v24
+ v54 = fadd v53, v25
+ v55 = fadd v54, v26
+ v56 = fadd v55, v27
+ v60 = fadd v40, v41
+ v61 = fadd v60, v42
+ v62 = fadd v61, v43
+ v63 = fadd v62, v44
+ v64 = fadd v63, v45
+ v65 = fadd v64, v46
+ v66 = fadd v65, v47
+ v70 = extractlane v66, 0
+ v71 = fadd v56, v70
+ return v71
+}
+; run: %caller(0, 0x1.0) == 0x58.0
diff --git a/cranelift/filetests/filetests/verifier/preserve-all-abi.clif b/cranelift/filetests/filetests/verifier/preserve-all-abi.clif
index be076884bc..6c8589f746 100644
--- a/cranelift/filetests/filetests/verifier/preserve-all-abi.clif
+++ b/cranelift/filetests/filetests/verifier/preserve-all-abi.clif
@@ -20,3 +20,27 @@ block0():
v1, v2, v3, v4 = call fn0()
return
}
+
+; The stack-limit check runs before any registers are saved and needs scratch
+; registers, which a `preserve_all` function can't clobber.
+function %f3(i64 vmctx) preserve_all { ; error: functions with the `preserve_all` ABI cannot have a `stack_limit`
+ gv0 = vmctx
+ gv1 = load.i64 notrap aligned gv0
+ stack_limit = gv1
+
+block0(v0: i64):
+ return
+}
+
+; Other ABIs can have a `stack_limit`, and calling a `preserve_all` function
+; from such a function is fine.
+function %f4(i64 vmctx) {
+ gv0 = vmctx
+ gv1 = load.i64 notrap aligned gv0
+ stack_limit = gv1
+ fn0 = %g(i64) preserve_all
+
+block0(v0: i64):
+ call fn0(v0)
+ return
+}
diff --git a/cranelift/filetests/filetests/verifier/preserve-all-probestack-inline.clif b/cranelift/filetests/filetests/verifier/preserve-all-probestack-inline.clif
new file mode 100644
index 0000000000..e1ba88b3b3
--- /dev/null
+++ b/cranelift/filetests/filetests/verifier/preserve-all-probestack-inline.clif
@@ -0,0 +1,10 @@
+test verifier
+set enable_probestack=true
+set probestack_strategy=inline
+
+; Inline stack probes are allowed for `preserve_all` functions: backends
+; must emit them without clobbering any registers.
+function %f0() preserve_all {
+block0():
+ return
+}
diff --git a/cranelift/filetests/filetests/verifier/preserve-all-probestack-outline.clif b/cranelift/filetests/filetests/verifier/preserve-all-probestack-outline.clif
new file mode 100644
index 0000000000..2fc6b3cf3a
--- /dev/null
+++ b/cranelift/filetests/filetests/verifier/preserve-all-probestack-outline.clif
@@ -0,0 +1,21 @@
+test verifier
+set enable_probestack=true
+set probestack_strategy=outline
+
+; The outline stack probe runs before any registers are saved and clobbers
+; registers (the frame size argument and the probestack function's own
+; clobbers), which a `preserve_all` function can't do.
+function %f0() preserve_all { ; error: functions with the `preserve_all` ABI cannot use the outline stack probe
+block0():
+ return
+}
+
+; Other ABIs can use the outline stack probe, and calling a `preserve_all`
+; function from such a function is fine.
+function %f1(i64) {
+ fn0 = %g(i64) preserve_all
+
+block0(v0: i64):
+ call fn0(v0)
+ return
+}