Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch: support tail calls with callee cleanup (#14349)

Daniel Macoveic46ba0216226
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index 171549ceb1..8659561704 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -713,10 +713,13 @@ impl ABIMachineSpec for AArch64MachineDeps {
});
}
- if call_conv == isa::CallConv::Tail && frame_layout.tail_args_size > 0 {
- insts.extend(Self::gen_sp_reg_adjust(
- frame_layout.tail_args_size.try_into().unwrap(),
- ));
+ let callee_pop_size = match call_conv {
+ isa::CallConv::Tail => frame_layout.tail_args_size,
+ isa::CallConv::Winch => frame_layout.incoming_args_size,
+ _ => 0,
+ };
+ if callee_pop_size > 0 {
+ insts.extend(Self::gen_sp_reg_adjust(callee_pop_size.try_into().unwrap()));
}
insts
diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs
index 6eb1a5e012..6b9e553813 100644
--- a/cranelift/codegen/src/isa/call_conv.rs
+++ b/cranelift/codegen/src/isa/call_conv.rs
@@ -37,11 +37,15 @@ pub enum CallConv {
AppleAarch64,
/// Specialized convention for the probestack function.
Probestack,
- /// The winch calling convention, not ABI-stable.
+ /// Internal calling convention for Winch-generated functions.
+ /// Cranelift-generated trampolines use the same convention to
+ /// interoperate with them. This convention is not ABI-stable.
///
- /// The main difference to SystemV is that the winch calling convention
- /// defines no callee-save registers, and restricts the number of return
- /// registers to one integer, and one floating point.
+ /// Callees reclaim their aligned incoming stack-argument area
+ /// before returning.
+ ///
+ /// Only the last result is returned in a register; any preceding
+ /// results are written to the return area in memory.
Winch,
/// Calling convention optimized for callsite efficiency, at the
/// cost of the callee. It does so by not clobbering any
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 2616737bfc..88117cdd8b 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -595,10 +595,10 @@ impl ABIMachineSpec for X64ABIMachineSpec {
frame_layout: &FrameLayout,
) -> SmallInstVec<Self::I> {
// Emit return instruction.
- let stack_bytes_to_pop = if call_conv == CallConv::Tail {
- frame_layout.tail_args_size
- } else {
- 0
+ let stack_bytes_to_pop = match call_conv {
+ CallConv::Tail => frame_layout.tail_args_size,
+ CallConv::Winch => frame_layout.incoming_args_size,
+ _ => 0,
};
let inst = if stack_bytes_to_pop == 0 {
asm::inst::retq_zo::new().into()
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 996310db7d..915720ed64 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -2168,8 +2168,9 @@ impl<M: ABIMachineSpec> Callee<M> {
// after the register allocator has run and thus cannot have register allocator-inserted
// references to SP offsets.)
- let callee_pop_size = if callee_conv == isa::CallConv::Tail {
- // The tail calling convention has callees pop stack arguments.
+ let callee_pop_size = if matches!(callee_conv, isa::CallConv::Tail | isa::CallConv::Winch) {
+ // The tail and Winch calling conventions have callees pop stack
+ // arguments.
stack_arg_space
} else {
0
diff --git a/cranelift/filetests/filetests/isa/aarch64/winch-callee-pop.clif b/cranelift/filetests/filetests/isa/aarch64/winch-callee-pop.clif
new file mode 100644
index 0000000000..2f4728d18e
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/winch-callee-pop.clif
@@ -0,0 +1,212 @@
+; Cranelift-generated trampolines must agree with Winch about who pops
+; incoming stack arguments, including indirect calls.
+test compile precise-output
+target aarch64
+
+function %callee(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch {
+block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64, v5: i64, v6: i64, v7: i64, v8: i64, v9: i64):
+ v10 = iadd v0, v9
+ return v10
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; block0:
+; ldr x2, [sp, #24]
+; add x0, x0, x2
+; ldp fp, lr, [sp], #16
+; add sp, sp, #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; block1: ; offset 0x8
+; ldur x2, [sp, #0x18]
+; add x0, x0, x2
+; ldp x29, x30, [sp], #0x10
+; add sp, sp, #0x10
+; ret
+
+function %caller() -> i64 system_v {
+ sig0 = (i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch
+ fn0 = colocated %callee sig0
+block0:
+ v0 = iconst.i64 42
+ v1 = call fn0(v0, v0, v0, v0, v0, v0, v0, v0, v0, v0)
+ v2 = iadd v1, v0
+ return v2
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; str x27, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; sub sp, sp, #16
+; block0:
+; movz x7, #42
+; str x7, [sp]
+; str x7, [sp, #8]
+; mov x0, x7
+; mov x1, x7
+; mov x2, x7
+; mov x3, x7
+; mov x4, x7
+; mov x5, x7
+; mov x6, x7
+; bl 0
+; add x0, x0, #42
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldr x27, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; str x27, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; sub sp, sp, #0x10
+; block1: ; offset 0x30
+; mov x7, #0x2a
+; stur x7, [sp]
+; stur x7, [sp, #8]
+; mov x0, x7
+; mov x1, x7
+; mov x2, x7
+; mov x3, x7
+; mov x4, x7
+; mov x5, x7
+; mov x6, x7
+; bl #0x58 ; reloc_external Call %callee 0
+; sub sp, sp, #0x10
+; add x0, x0, #0x2a
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldr x27, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
+function %indirect_caller(i64) -> i64 system_v {
+ sig0 = (i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch
+block0(v0: i64):
+ v1 = iconst.i64 42
+ v2 = call_indirect sig0, v0(v1, v1, v1, v1, v1, v1, v1, v1, v1, v1)
+ v3 = iadd v2, v1
+ return v3
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; str x27, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; sub sp, sp, #16
+; block0:
+; mov x8, x0
+; movz x7, #42
+; str x7, [sp]
+; str x7, [sp, #8]
+; mov x0, x7
+; mov x1, x7
+; mov x2, x7
+; mov x3, x7
+; mov x4, x7
+; mov x5, x7
+; mov x6, x7
+; blr x8
+; add x0, x0, #42
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldr x27, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; str x27, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; sub sp, sp, #0x10
+; block1: ; offset 0x30
+; mov x8, x0
+; mov x7, #0x2a
+; stur x7, [sp]
+; stur x7, [sp, #8]
+; mov x0, x7
+; mov x1, x7
+; mov x2, x7
+; mov x3, x7
+; mov x4, x7
+; mov x5, x7
+; mov x6, x7
+; blr x8
+; sub sp, sp, #0x10
+; add x0, x0, #0x2a
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldr x27, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
diff --git a/cranelift/filetests/filetests/isa/aarch64/winch.clif b/cranelift/filetests/filetests/isa/aarch64/winch.clif
new file mode 100644
index 0000000000..5fb824ea75
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/winch.clif
@@ -0,0 +1,151 @@
+test compile precise-output
+target aarch64
+
+function %direct_winch_call() winch {
+ fn0 = colocated %direct_callee() winch
+
+block0:
+ call fn0()
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; block0:
+; bl 0
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; block1: ; offset 0x8
+; bl #8 ; reloc_external Call %direct_callee 0
+; ldp x29, x30, [sp], #0x10
+; ret
+
+function %indirect_winch_call(i64) winch {
+ sig0 = () winch
+
+block0(v0: i64):
+ call_indirect sig0, v0()
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; block0:
+; blr x0
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; block1: ; offset 0x8
+; blr x0
+; ldp x29, x30, [sp], #0x10
+; ret
+
+function %try_winch_call() -> i32 winch {
+ sig0 = () winch
+ fn0 = colocated %try_callee sig0
+
+block0:
+ try_call fn0(), sig0, block1(), [ default: block2 ]
+
+block1:
+ v0 = iconst.i32 0
+ return v0
+
+block2:
+ v1 = iconst.i32 1
+ return v1
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; block0:
+; bl 0; b MachLabel(1); catch [default: MachLabel(2)]
+; block1:
+; movz w0, #0
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block2:
+; movz w0, #1
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; block1: ; offset 0x2c
+; bl #0x2c ; reloc_external Call %try_callee 0
+; block2: ; offset 0x30
+; mov w0, #0
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block3: ; offset 0x60
+; mov w0, #1
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
diff --git a/cranelift/filetests/filetests/isa/x64/winch-callee-pop.clif b/cranelift/filetests/filetests/isa/x64/winch-callee-pop.clif
new file mode 100644
index 0000000000..6db5712bde
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/winch-callee-pop.clif
@@ -0,0 +1,184 @@
+; Cranelift-generated trampolines must agree with Winch about who pops
+; incoming stack arguments, including indirect calls.
+test compile precise-output
+target x86_64
+
+function %callee(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch {
+block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64, v5: i64, v6: i64, v7: i64, v8: i64, v9: i64):
+ v10 = iadd v0, v9
+ return v10
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq <offset:0>+-8(%rbp), %rdx
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq $0x20
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq 0x28(%rbp), %rdx
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq $0x20
+
+function %caller() -> i64 system_v {
+ sig0 = (i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch
+ fn0 = colocated %callee sig0
+block0:
+ v0 = iconst.i64 42
+ v1 = call fn0(v0, v0, v0, v0, v0, v0, v0, v0, v0, v0)
+ v2 = iadd v1, v0
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x20(%rsp)
+; movq %r12, 0x28(%rsp)
+; movq %r13, 0x30(%rsp)
+; movq %r14, 0x38(%rsp)
+; movq %r15, 0x40(%rsp)
+; block0:
+; movl $0x2a, %r9d
+; movq %r9, (%rsp)
+; movq %r9, 8(%rsp)
+; movq %r9, 0x10(%rsp)
+; movq %r9, 0x18(%rsp)
+; movq %r9, %rcx
+; movq %r9, %rdx
+; movq %r9, %rsi
+; movq %r9, %rdi
+; movq %r9, %r8
+; call TestCase(%callee)
+; leaq 0x2a(%rax), %rax
+; movq 0x20(%rsp), %rbx
+; movq 0x28(%rsp), %r12
+; movq 0x30(%rsp), %r13
+; movq 0x38(%rsp), %r14
+; movq 0x40(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x20(%rsp)
+; movq %r12, 0x28(%rsp)
+; movq %r13, 0x30(%rsp)
+; movq %r14, 0x38(%rsp)
+; movq %r15, 0x40(%rsp)
+; block1: ; offset 0x21
+; movl $0x2a, %r9d
+; movq %r9, (%rsp)
+; movq %r9, 8(%rsp)
+; movq %r9, 0x10(%rsp)
+; movq %r9, 0x18(%rsp)
+; movq %r9, %rcx
+; movq %r9, %rdx
+; movq %r9, %rsi
+; movq %r9, %rdi
+; movq %r9, %r8
+; callq 0x4e ; reloc_external CallPCRel4 %callee -4
+; subq $0x20, %rsp
+; addq $0x2a, %rax
+; movq 0x20(%rsp), %rbx
+; movq 0x28(%rsp), %r12
+; movq 0x30(%rsp), %r13
+; movq 0x38(%rsp), %r14
+; movq 0x40(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %indirect_caller(i64) -> i64 system_v {
+ sig0 = (i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 winch
+block0(v0: i64):
+ v1 = iconst.i64 42
+ v2 = call_indirect sig0, v0(v1, v1, v1, v1, v1, v1, v1, v1, v1, v1)
+ v3 = iadd v2, v1
+ return v3
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x20(%rsp)
+; movq %r12, 0x28(%rsp)
+; movq %r13, 0x30(%rsp)
+; movq %r14, 0x38(%rsp)
+; movq %r15, 0x40(%rsp)
+; block0:
+; movl $0x2a, %r9d
+; movq %r9, (%rsp)
+; movq %r9, 8(%rsp)
+; movq %r9, 0x10(%rsp)
+; movq %r9, 0x18(%rsp)
+; movq %rdi, %r10
+; movq %r9, %rcx
+; movq %r9, %rdx
+; movq %r9, %rsi
+; movq %r9, %rdi
+; movq %r9, %r8
+; call *%r10
+; leaq 0x2a(%rax), %rax
+; movq 0x20(%rsp), %rbx
+; movq 0x28(%rsp), %r12
+; movq 0x30(%rsp), %r13
+; movq 0x38(%rsp), %r14
+; movq 0x40(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x20(%rsp)
+; movq %r12, 0x28(%rsp)
+; movq %r13, 0x30(%rsp)
+; movq %r14, 0x38(%rsp)
+; movq %r15, 0x40(%rsp)
+; block1: ; offset 0x21
+; movl $0x2a, %r9d
+; movq %r9, (%rsp)
+; movq %r9, 8(%rsp)
+; movq %r9, 0x10(%rsp)
+; movq %r9, 0x18(%rsp)
+; movq %rdi, %r10
+; movq %r9, %rcx
+; movq %r9, %rdx
+; movq %r9, %rsi
+; movq %r9, %rdi
+; movq %r9, %r8
+; callq *%r10
+; subq $0x20, %rsp
+; addq $0x2a, %rax
+; movq 0x20(%rsp), %rbx
+; movq 0x28(%rsp), %r12
+; movq 0x30(%rsp), %r13
+; movq 0x38(%rsp), %r14
+; movq 0x40(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
diff --git a/cranelift/filetests/filetests/isa/x64/winch.clif b/cranelift/filetests/filetests/isa/x64/winch.clif
index 0067b8fc17..32ee94e3b8 100644
--- a/cranelift/filetests/filetests/isa/x64/winch.clif
+++ b/cranelift/filetests/filetests/isa/x64/winch.clif
@@ -278,6 +278,83 @@ block0:
; popq %rbp
; retq
+function %direct_winch_call() winch {
+ fn0 = colocated %direct_callee() winch
+
+block0:
+ call fn0()
+ return
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; call TestCase(%direct_callee)
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; callq 9 ; reloc_external CallPCRel4 %direct_callee -4
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %try_winch_call() -> i32 winch {
+ sig0 = () winch
+ fn0 = colocated %try_callee sig0
+
+block0:
+ try_call fn0(), sig0, block1(), [ default: block2 ]
+
+block1:
+ v0 = iconst.i32 0
+ return v0
+
+block2:
+ v1 = iconst.i32 1
+ return v1
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; call TestCase(%try_callee); jmp MachLabel(1); catch [default: MachLabel(2)]
+; block1:
+; uninit %rax
+; xorl %eax, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movl $0x1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; callq 9 ; reloc_external CallPCRel4 %try_callee -4
+; block2: ; offset 0x9
+; xorl %eax, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x10
+; movl $1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
function %f6(i64) -> i32 {
sig0 = () -> i32, i32, f64 winch
fn0 = %g sig0
@@ -448,4 +525,3 @@ block0:
; movq %rbp, %rsp
; popq %rbp
; retq
-
diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs
index 4c4980df9c..7ee400a3df 100644
--- a/crates/fuzzing/src/generators/config.rs
+++ b/crates/fuzzing/src/generators/config.rs
@@ -692,7 +692,6 @@ impl WasmtimeConfig {
// module that wasm-smith generates.
config.config.relaxed_simd_enabled = false;
config.config.gc_enabled = false;
- config.config.tail_call_enabled = false;
config.config.reference_types_enabled = false;
config.config.exceptions_enabled = false;
config.function_references_enabled = false;
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index f328795cb9..35d3010a25 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -462,7 +462,6 @@ impl Compiler {
Compiler::Winch => {
if config.gc()
- || config.tail_call()
|| config.function_references()
|| config.relaxed_simd()
|| config.legacy_exceptions()
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index b8635634ea..310faf5067 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -938,7 +938,7 @@ impl Config {
/// programs to implement some recursive algorithms with *O(1)* stack space
/// usage.
///
- /// This is `true` by default except when the Winch compiler is enabled.
+ /// This is `true` by default.
///
/// [WebAssembly tail calls proposal]: https://github.com/WebAssembly/tail-call
pub fn wasm_tail_call(&mut self, enable: bool) -> &mut Self {
@@ -2485,7 +2485,6 @@ impl Config {
unsupported |= WasmFeatures::GC
| WasmFeatures::FUNCTION_REFERENCES
| WasmFeatures::RELAXED_SIMD
- | WasmFeatures::TAIL_CALL
| WasmFeatures::LEGACY_EXCEPTIONS
| WasmFeatures::STACK_SWITCHING;
diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md
index d9ca974d82..cb8fb2f869 100644
--- a/docs/stability-tiers.md
+++ b/docs/stability-tiers.md
@@ -225,7 +225,7 @@ here is:
| [`relaxed-simd`] | ✅ | ❌ |
| [`multi-memory`] | ✅ | ✅ |
| [`threads`] | ✅ | ✅ |
-| [`tail-call`] | ✅ | ❌ |
+| [`tail-call`] | ✅ | ✅ |
| [`extended-const`] | ✅ | ✅ |
| [`memory64`] | ✅ | ✅ |
| [`function-references`] | ✅ | ❌ |
@@ -250,7 +250,7 @@ here is:
| [`relaxed-simd`] | ✅ | ❌ |
| [`multi-memory`] | ✅ | ✅ |
| [`threads`] | ✅ | ❌ |
-| [`tail-call`] | ✅ | ❌ |
+| [`tail-call`] | ✅ | ✅ |
| [`extended-const`] | ✅ | ✅ |
| [`memory64`] | ✅ | ✅ |
| [`function-references`] | ✅ | ❌ |
diff --git a/tests/all/defaults.rs b/tests/all/defaults.rs
index 28b34c4533..e76f341553 100644
--- a/tests/all/defaults.rs
+++ b/tests/all/defaults.rs
@@ -34,14 +34,30 @@ fn test_tail_call_default() -> Result<()> {
),
(
line!(),
- false,
+ true,
Config::new().strategy(Strategy::Winch).target("x86_64")?,
),
(
line!(),
- false,
+ true,
Config::new().strategy(Strategy::Winch).target("aarch64")?,
),
+ (
+ line!(),
+ false,
+ Config::new()
+ .strategy(Strategy::Winch)
+ .wasm_tail_call(false)
+ .target("x86_64")?,
+ ),
+ (
+ line!(),
+ false,
+ Config::new()
+ .strategy(Strategy::Winch)
+ .wasm_tail_call(false)
+ .target("aarch64")?,
+ ),
(
line!(),
false,
diff --git a/tests/all/main.rs b/tests/all/main.rs
index 33448abe92..ed8c845a6e 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -68,6 +68,7 @@ mod traps;
mod types;
mod wait_notify;
mod winch_engine_features;
+mod winch_tail_calls;
/// A helper to compile a module in a new store with reference types enabled.
pub(crate) fn ref_types_module(
diff --git a/tests/all/module.rs b/tests/all/module.rs
index 2920aadb13..3d5764797c 100644
--- a/tests/all/module.rs
+++ b/tests/all/module.rs
@@ -291,13 +291,12 @@ fn tail_call_defaults() -> Result<()> {
wasm_with_tail_calls,
)?;
- if cfg!(target_arch = "x86_64") {
- // off by default for winch
- let err = Module::new(
+ if cfg!(any(target_arch = "x86_64", target_arch = "aarch64")) {
+ // on by default for Winch on its supported targets
+ Module::new(
&Engine::new(Config::new().strategy(Strategy::Winch))?,
wasm_with_tail_calls,
- );
- assert!(err.is_err());
+ )?;
}
Ok(())
}
diff --git a/tests/all/traps.rs b/tests/all/traps.rs
index b80bfa7059..14dd27eaf7 100644
--- a/tests/all/traps.rs
+++ b/tests/all/traps.rs
@@ -1767,9 +1767,9 @@ fn return_call_indirect_to_imported_function(config: &mut Config) -> Result<()>
Ok(())
}
-#[test]
-fn return_call_to_aborting_wasm_function_with_stack_adjustments() -> Result<()> {
- let engine = Engine::default();
+#[wasmtime_test(wasm_features(tail_call))]
+fn return_call_to_aborting_wasm_function_with_stack_adjustments(config: &mut Config) -> Result<()> {
+ let engine = Engine::new(config)?;
let module = Module::new(
&engine,
r#"
diff --git a/tests/all/winch_tail_calls.rs b/tests/all/winch_tail_calls.rs
new file mode 100644
index 0000000000..6751ed0682
--- /dev/null
+++ b/tests/all/winch_tail_calls.rs
@@ -0,0 +1,406 @@
+//! Tail-call returns, synchronous frame walking, and recovery after traps.
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use wasmtime::*;
+
+use wasmtime_test_macros::wasmtime_test;
+
+#[wasmtime_test(wasm_features(tail_call))]
+#[cfg_attr(miri, ignore)]
+fn tail_calls_to_host_functions(config: &mut Config) -> Result<()> {
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"(module
+ (type $no-args (func (result i32)))
+ (type $with-args (func (param i32 i32 i32 i32 i32) (result i32)))
+ (import "" "no-args" (func $no-args (type $no-args)))
+ (import "" "with-args" (func $with-args (type $with-args)))
+ (table funcref (elem $no-args $with-args))
+ (func (export "direct-no-args") (result i32)
+ return_call $no-args)
+ (func (export "indirect-no-args") (result i32)
+ i32.const 0
+ return_call_indirect (type $no-args))
+ (func (export "direct-with-args") (param i32) (result i32)
+ local.get 0
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ i32.const 5
+ return_call $with-args)
+ (func (export "indirect-with-args") (param i32) (result i32)
+ local.get 0
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ i32.const 5
+ i32.const 1
+ return_call_indirect (type $with-args)))"#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ let no_args = Func::wrap(&mut store, || 42i32);
+ let with_args = Func::wrap(&mut store, |first: i32, _: i32, _: i32, _: i32, _: i32| {
+ first
+ });
+ let instance = Instance::new(&mut store, &module, &[no_args.into(), with_args.into()])?;
+ for name in ["direct-no-args", "indirect-no-args"] {
+ let run = instance.get_typed_func::<(), i32>(&mut store, name)?;
+ assert_eq!(run.call(&mut store, ())?, 42);
+ }
+ for name in ["direct-with-args", "indirect-with-args"] {
+ let run = instance.get_typed_func::<i32, i32>(&mut store, name)?;
+ assert_eq!(run.call(&mut store, 84)?, 84);
+ }
+ Ok(())
+}
+
+#[wasmtime_test(wasm_features(tail_call))]
+#[cfg_attr(miri, ignore)]
+fn tail_calls_skip_unreachable_loops(config: &mut Config) -> Result<()> {
+ for (fuel, epoch) in [(true, false), (false, true), (true, true)] {
+ let mut config = config.clone();
+ config.consume_fuel(fuel).epoch_interruption(epoch);
+ let engine = Engine::new(&config)?;
+ for call in [
+ "return_call $leaf",
+ "i32.const 0 return_call_indirect (type $t)",
+ ] {
+ let module = Module::new(
+ &engine,
+ format!(
+ r#"(module
+ (type $t (func (result i32)))
+ (import "" "interrupt" (func $interrupt))
+ (func $leaf (type $t) i32.const 42)
+ (table funcref (elem $leaf))
+ ;; Tail recursion followed by an unreachable loop.
+ (func $recursive return_call $recursive (loop))
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ if
+ {call}
+ (loop (loop))
+ end
+ ;; The join restores reachability and the frame.
+ (loop)
+ i32.const 7)
+ (func (export "interrupt") (param i32) (result i32)
+ local.get 0
+ if
+ {call}
+ (loop)
+ end
+ call $interrupt
+ ;; This reachable loop must still check for interruption.
+ (loop)
+ i32.const 7))"#
+ ),
+ )?;
+ let mut store = Store::new(&engine, ());
+ if fuel {
+ store.set_fuel(1_000_000)?;
+ }
+ if epoch {
+ store.set_epoch_deadline(1);
+ }
+ let interrupt = Func::wrap(
+ &mut store,
+ move |mut caller: Caller<'_, ()>| -> Result<()> {
+ if fuel {
+ caller.set_fuel(0)?;
+ } else {
+ caller.engine().increment_epoch();
+ }
+ Ok(())
+ },
+ );
+ let instance = Instance::new(&mut store, &module, &[interrupt.into()])?;
+ let run = instance.get_typed_func::<i32, i32>(&mut store, "run")?;
+ assert_eq!(run.call(&mut store, 1)?, 42);
+ assert_eq!(run.call(&mut store, 0)?, 7);
+ let interrupt = instance.get_typed_func::<i32, i32>(&mut store, "interrupt")?;
+ assert_eq!(interrupt.call(&mut store, 1)?, 42);
+ let trap = interrupt
+ .call(&mut store, 0)
+ .unwrap_err()
+ .downcast::<Trap>()?;
+ assert_eq!(
+ trap,
+ if fuel {
+ Trap::OutOfFuel
+ } else {
+ Trap::Interrupt
+ }
+ );
+ }
+ }
+ Ok(())
+}
+
+// Keep an operand live across each call and consume every argument and result.
+// This exercises combined padding/spill cleanup as well as the stack-result
+// path, where result movement must precede the final cleanup.
+#[wasmtime_test]
+#[cfg_attr(miri, ignore)]
+fn cleanup_preserves_live_values(config: &mut Config) -> Result<()> {
+ config.wasm_tail_call(false);
+ let engine = Engine::new(config)?;
+ for n in [0, 1, 5, 10, 32, 64, 80] {
+ for multi in [false, true] {
+ let params = " i64".repeat(n);
+ let sum = (0..n)
+ .map(|i| format!("local.get {i} i64.add "))
+ .collect::<String>();
+ let args = (1..=n)
+ .map(|i| format!("local.get 0 i64.const {i} i64.add "))
+ .collect::<String>();
+ let results = if multi { "i64 i64 f64" } else { "i64" };
+ let extra = if multi {
+ "i64.const 123 f64.const 3.5"
+ } else {
+ ""
+ };
+ let consume = if multi {
+ "i64.trunc_f64_s i64.add i64.add"
+ } else {
+ ""
+ };
+ let mut store = Store::new(&engine, ());
+ let provider = Module::new(
+ &engine,
+ format!(
+ "(module (func (export \"leaf\") (param {params}) (result {results})
+ i64.const 0 {sum} {extra}))"
+ ),
+ )?;
+ let provider = Instance::new(&mut store, &provider, &[])?;
+ let imported = provider.get_func(&mut store, "leaf").unwrap();
+ let module = Module::new(
+ &engine,
+ format!(
+ r#"(module
+ (type $t (func (param {params}) (result {results})))
+ (import "p" "leaf" (func $imported (type $t)))
+ (func $local (type $t) i64.const 0 {sum} {extra})
+ (table funcref (elem $local $imported))
+ (func (export "local") (param i64) (result i64)
+ local.get 0 {args} call $local {consume} i64.add)
+ (func (export "imported") (param i64) (result i64)
+ local.get 0 {args} call $imported {consume} i64.add)
+ (func (export "indirect") (param i64 i32) (result i64)
+ local.get 0 {args} local.get 1 call_indirect (type $t)
+ {consume} i64.add))"#
+ ),
+ )?;
+ let instance = Instance::new(&mut store, &module, &[imported.into()])?;
+ let local = instance.get_typed_func::<i64, i64>(&mut store, "local")?;
+ let imported = instance.get_typed_func::<i64, i64>(&mut store, "imported")?;
+ let indirect = instance.get_typed_func::<(i64, i32), i64>(&mut store, "indirect")?;
+ for seed in [-17, 0, 42] {
+ let n = n as i64;
+ let expected = (n + 1) * seed + n * (n + 1) / 2 + if multi { 126 } else { 0 };
+ for _ in 0..3 {
+ assert_eq!(local.call(&mut store, seed)?, expected);
+ assert_eq!(imported.call(&mut store, seed)?, expected);
+ assert_eq!(indirect.call(&mut store, (seed, 0))?, expected);
+ assert_eq!(indirect.call(&mut store, (seed, 1))?, expected);
+ }
+ }
+ }
+ }
+ Ok(())
+}
+
+#[derive(Default)]
+struct State {
+ mode: u32,
+ captures: Vec<Vec<String>>,
+}
+
+fn names(bt: &WasmBacktrace) -> Vec<String> {
+ bt.frames()
+ .iter()
+ .map(|f| f.func_name().unwrap_or("?").to_owned())
+ .collect()
+}
+
+#[wasmtime_test(wasm_features(tail_call))]
+#[cfg_attr(miri, ignore)]
+fn returns_and_traps(config: &mut Config) -> Result<()> {
+ for fuel in [false, true] {
+ let mut config = config.clone();
+ config.consume_fuel(fuel);
+ config.max_wasm_stack(64 * 1024);
+ let engine = Engine::new(&config)?;
+ // Cover register and stack arguments, AArch64's signed load/store-offset
+ // boundary, and entry-SP deltas beyond a single add/sub immediate.
+ for n in [0, 1, 5, 10, 32, 38, 39, 40, 64, 80, 528] {
+ let params = " i64".repeat(n);
+ let args = (1..=n)
+ .map(|i| format!("i64.const {i} "))
+ .collect::<String>();
+ let gets = (0..n)
+ .map(|i| format!("local.get {i} "))
+ .collect::<String>();
+ let sum = (0..n)
+ .map(|i| format!("local.get {i} i64.add "))
+ .collect::<String>();
+ let provider = format!(
+ r#"(module
+ (import "h" "observe" (func $observe))
+ (func $reenter (export "reenter") (param {params}) (result i64)
+ i64.const 0 {sum})
+ (func $leaf (export "leaf") (param {params}) (result i64 i64 f64)
+ {gets} call $reenter drop
+ call $observe
+ i64.const 0 {sum} i64.const 123 f64.const 3.5))"#
+ );
+ for indirect in [false, true] {
+ for tail in [false, true] {
+ let mut store = Store::new(&engine, State::default());
+ if fuel {
+ store.set_fuel(1_000_000_000)?;
+ }
+ let observe = Func::wrap(
+ &mut store,
+ move |mut caller: Caller<'_, State>| -> Result<()> {
+ let bt = names(&WasmBacktrace::force_capture(&mut caller));
+ caller.data_mut().captures.push(bt);
+ match caller.data().mode {
+ 1 => bail!("expected host error"),
+ 2 => panic!("expected host panic"),
+ 3 => {
+ let f =
+ caller.get_export("reenter").unwrap().into_func().unwrap();
+ let input =
+ (1..=n).map(|i| Val::I64(i as i64)).collect::<Vec<_>>();
+ let mut result = [Val::I64(0)];
+ f.call(&mut caller, &input, &mut result)?;
+ assert_eq!(result[0].i64(), Some((n * (n + 1) / 2) as i64));
+ let bt = names(&WasmBacktrace::force_capture(&mut caller));
+ caller.data_mut().captures.push(bt);
+ }
+ _ => {}
+ }
+ Ok(())
+ },
+ );
+ let provider = Instance::new(
+ &mut store,
+ &Module::new(&engine, &provider)?,
+ &[observe.into()],
+ )?;
+ let leaf = provider.get_func(&mut store, "leaf").unwrap();
+ let op = if tail { "return_call" } else { "call" };
+ let leaf_call = if indirect {
+ format!("i32.const 0 {op}_indirect (type $t)")
+ } else {
+ format!("{op} $leaf")
+ };
+ let source = format!(
+ r#"(module
+ (type $t (func (param {params}) (result i64 i64 f64)))
+ (import "p" "leaf" (func $leaf (type $t)))
+ (table 1 funcref) (elem (i32.const 0) $leaf)
+ (func $thin (result i64 i64 f64) {args} {leaf_call})
+ (func $wide (param {params}) (result i64 i64 f64) {op} $thin)
+ (func $run (export "run") (result i64 i64 f64) {args} call $wide)
+ (func $after (export "after") call $run drop drop drop unreachable))"#
+ );
+ let module = Module::new(&engine, &source)?;
+ let instance = Instance::new(&mut store, &module, &[leaf.into()])?;
+ let run = instance.get_typed_func::<(), (i64, i64, f64)>(&mut store, "run")?;
+ let after = instance.get_typed_func::<(), ()>(&mut store, "after")?;
+ let expected_frames = if tail {
+ vec!["leaf", "run"]
+ } else {
+ vec!["leaf", "thin", "wide", "run"]
+ };
+ for _ in 0..2 {
+ for mode in [0, 1, 2, 3] {
+ store.data_mut().mode = mode;
+ store.data_mut().captures.clear();
+ match mode {
+ 1 => {
+ let err = run.call(&mut store, ()).unwrap_err();
+ assert!(format!("{err:#}").contains("expected host error"));
+ assert_eq!(
+ names(err.downcast_ref::<WasmBacktrace>().unwrap()),
+ expected_frames
+ );
+ }
+ 2 => {
+ let panic =
+ catch_unwind(AssertUnwindSafe(|| run.call(&mut store, ())))
+ .unwrap_err();
+ assert_eq!(
+ panic.downcast_ref::<&str>(),
+ Some(&"expected host panic")
+ );
+ }
+ _ => assert_eq!(
+ run.call(&mut store, ())?,
+ ((n * (n + 1) / 2) as i64, 123, 3.5)
+ ),
+ }
+ assert_eq!(store.data().captures.len(), if mode == 3 { 2 } else { 1 });
+ for frames in &store.data().captures {
+ assert_eq!(*frames, expected_frames);
+ }
+ }
+ store.data_mut().mode = 0;
+ let err = after.call(&mut store, ()).unwrap_err();
+ assert_eq!(
+ err.downcast_ref::<Trap>(),
+ Some(&Trap::UnreachableCodeReached)
+ );
+ assert_eq!(
+ names(err.downcast_ref::<WasmBacktrace>().unwrap()),
+ ["after"]
+ );
+ }
+ }
+ }
+ }
+ // Ordinary recursion must trap; equivalent tail recursion must not grow
+ // the stack. A successful subsequent call checks trap-state restoration.
+ let args = " i64.const 7".repeat(12);
+ let params = " i64".repeat(12);
+ let gets = (1..=12)
+ .map(|i| format!("local.get {i} "))
+ .collect::<String>();
+ let source = format!(
+ r#"(module
+ (func $ordinary (param i32 {params}) (result i32)
+ local.get 0 i32.eqz if (result i32) i32.const 42 else
+ local.get 0 i32.const 1 i32.sub {gets} call $ordinary
+ end)
+ (func $tail (param i32 {params}) (result i32)
+ local.get 0 i32.eqz if (result i32) i32.const 42 else
+ local.get 0 i32.const 1 i32.sub {gets} return_call $tail
+ end)
+ (func (export "ordinary") (param i32) (result i32) local.get 0 {args} call $ordinary)
+ (func (export "tail") (param i32) (result i32) local.get 0 {args} call $tail))"#
+ );
+ let mut store = Store::new(&engine, ());
+ if fuel {
+ store.set_fuel(1_000_000_000)?;
+ }
+ let instance = Instance::new(&mut store, &Module::new(&engine, source)?, &[])?;
+ let ordinary = instance.get_typed_func::<i32, i32>(&mut store, "ordinary")?;
+ let tail = instance.get_typed_func::<i32, i32>(&mut store, "tail")?;
+ for _ in 0..10 {
+ let err = ordinary.call(&mut store, 100_000).unwrap_err();
+ assert_eq!(err.downcast_ref::<Trap>(), Some(&Trap::StackOverflow));
+ assert!(
+ !err.downcast_ref::<WasmBacktrace>()
+ .unwrap()
+ .frames()
+ .is_empty()
+ );
+ assert_eq!(tail.call(&mut store, 100_000)?, 42);
+ assert_eq!(ordinary.call(&mut store, 2)?, 42);
+ }
+ }
+ Ok(())
+}
diff --git a/tests/disas/winch/aarch64/call/params.wat b/tests/disas/winch/aarch64/call/params.wat
index 57059a49bb..b1c035c64b 100644
--- a/tests/disas/winch/aarch64/call/params.wat
+++ b/tests/disas/winch/aarch64/call/params.wat
@@ -48,7 +48,7 @@
;; movk x17, #0x40
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x14c
+;; b.lo #0x144
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -79,9 +79,8 @@
;; mov x16, #8
;; stur w16, [x28, #0x10]
;; bl #0x160
-;; a4: add x28, x28, #0x24
-;; mov sp, x28
-;; add x28, x28, #4
+;; a4: mov x28, sp
+;; add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; ldur w1, [x28, #4]
@@ -110,8 +109,7 @@
;; mov x16, #8
;; stur w16, [x28, #0x10]
;; bl #0x160
-;; 120: add x28, x28, #0x20
-;; mov sp, x28
+;; 11c: mov x28, sp
;; add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
@@ -121,7 +119,7 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 14c: udf #0xc11f
+;; 144: udf #0xc11f
;;
;; wasm[0]::function[1]::add:
;; stp x29, x30, [sp, #-0x10]!
@@ -134,7 +132,7 @@
;; movk x17, #0x28
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x218
+;; b.lo #0x21c
;; 18c: mov x9, x0
;; sub x28, x28, #0x28
;; mov sp, x28
@@ -169,5 +167,6 @@
;; mov sp, x28
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
+;; add sp, sp, #0x20
;; ret
-;; 218: udf #0xc11f
+;; 21c: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/call/recursive.wat b/tests/disas/winch/aarch64/call/recursive.wat
index d713786075..6b8db0ec04 100644
--- a/tests/disas/winch/aarch64/call/recursive.wat
+++ b/tests/disas/winch/aarch64/call/recursive.wat
@@ -35,7 +35,7 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x10c
+;; b.lo #0x104
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -49,7 +49,7 @@
;; b.eq #0x64
;; b #0x5c
;; 5c: ldur w0, [x28, #4]
-;; b #0xf4
+;; b #0xec
;; 64: ldur w0, [x28, #4]
;; sub w0, w0, #1
;; sub x28, x28, #4
@@ -61,9 +61,7 @@
;; mov x1, x9
;; ldur w2, [x28, #4]
;; bl #0
-;; 90: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; 90: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; ldur w1, [x28, #4]
@@ -78,7 +76,7 @@
;; mov x1, x9
;; ldur w2, [x28]
;; bl #0
-;; d4: add x28, x28, #4
+;; cc: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; ldur w1, [x28]
@@ -92,4 +90,4 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 10c: udf #0xc11f
+;; 104: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
index 6c26d05746..eed3fa344b 100644
--- a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
@@ -40,7 +40,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x28c
+;; b.lo #0x27c
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -54,7 +54,7 @@
;; b.eq #0x64
;; b #0x5c
;; 5c: mov x0, #1
-;; b #0x274
+;; b #0x264
;; 64: ldur w0, [x28, #4]
;; sub w0, w0, #2
;; sub x28, x28, #4
@@ -65,7 +65,7 @@
;; ldur x3, [x2, #0x38]
;; cmp x1, x3, uxtx
;; sub sp, x28, #4
-;; b.hs #0x290
+;; b.hs #0x280
;; 90: mov sp, x28
;; mov x16, x1
;; mov x17, #8
@@ -85,21 +85,21 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x408
+;; bl #0x3f8
;; e0: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; b #0xf4
;; f0: and x0, x0, #0xfffffffffffffffe
;; sub sp, x28, #4
-;; cbz x0, #0x294
+;; cbz x0, #0x284
;; fc: mov sp, x28
;; ldur x16, [x9, #0x28]
;; ldur w1, [x16]
;; ldur w2, [x0, #0x10]
;; cmp w1, w2, uxtx
;; sub sp, x28, #4
-;; b.ne #0x298
+;; b.ne #0x288
;; 118: mov sp, x28
;; sub x28, x28, #8
;; mov sp, x28
@@ -115,9 +115,7 @@
;; mov x1, x9
;; ldur w2, [x28, #4]
;; blr x4
-;; 154: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; 154: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; ldur w1, [x28, #4]
@@ -132,8 +130,8 @@
;; mov x2, x9
;; ldur x3, [x2, #0x38]
;; cmp x1, x3, uxtx
-;; b.hs #0x29c
-;; 19c: mov x16, x1
+;; b.hs #0x28c
+;; 194: mov x16, x1
;; mov x17, #8
;; mul x16, x16, x17
;; ldur x2, [x2, #0x30]
@@ -143,9 +141,9 @@
;; csel x2, x4, x2, hs
;; ldur x0, [x2]
;; tst x0, x0
-;; b.ne #0x208
-;; b #0x1cc
-;; 1cc: sub x28, x28, #4
+;; b.ne #0x1f8
+;; b #0x1c4
+;; 1c4: sub x28, x28, #4
;; mov sp, x28
;; stur w1, [x28]
;; sub x28, x28, #0xc
@@ -153,21 +151,19 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28, #0xc]
-;; bl #0x408
-;; 1f0: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x3f8
+;; 1e8: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #0x18]
-;; b #0x20c
-;; 208: and x0, x0, #0xfffffffffffffffe
-;; cbz x0, #0x2a0
-;; 210: ldur x16, [x9, #0x28]
+;; b #0x1fc
+;; 1f8: and x0, x0, #0xfffffffffffffffe
+;; cbz x0, #0x290
+;; 200: ldur x16, [x9, #0x28]
;; ldur w1, [x16]
;; ldur w2, [x0, #0x10]
;; cmp w1, w2, uxtx
-;; b.ne #0x2a4
-;; 224: sub x28, x28, #8
+;; b.ne #0x294
+;; 214: sub x28, x28, #8
;; mov sp, x28
;; stur x0, [x28]
;; ldur x3, [x28]
@@ -179,7 +175,7 @@
;; mov x1, x9
;; ldur w2, [x28]
;; blr x4
-;; 254: add x28, x28, #4
+;; 244: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; ldur w1, [x28]
@@ -193,10 +189,10 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
+;; 27c: udf #0xc11f
+;; 280: udf #0xc11f
+;; 284: udf #0xc11f
+;; 288: udf #0xc11f
;; 28c: udf #0xc11f
;; 290: udf #0xc11f
;; 294: udf #0xc11f
-;; 298: udf #0xc11f
-;; 29c: udf #0xc11f
-;; 2a0: udf #0xc11f
-;; 2a4: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/call_indirect/local_arg.wat b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
index 1675b9bbd6..d81b97733c 100644
--- a/tests/disas/winch/aarch64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
@@ -54,7 +54,7 @@
;; movk x17, #0x24
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1c0
+;; b.lo #0x1b8
;; 8c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -71,7 +71,7 @@
;; ldur x3, [x2, #0x38]
;; cmp x1, x3, uxtx
;; sub sp, x28, #4
-;; b.hs #0x1c4
+;; b.hs #0x1bc
;; d0: mov sp, x28
;; mov x16, x1
;; mov x17, #8
@@ -91,21 +91,21 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x440
+;; bl #0x438
;; 120: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; b #0x134
;; 130: and x0, x0, #0xfffffffffffffffe
;; sub sp, x28, #4
-;; cbz x0, #0x1c8
+;; cbz x0, #0x1c0
;; 13c: mov sp, x28
;; ldur x16, [x9, #0x28]
;; ldur w1, [x16]
;; ldur w2, [x0, #0x10]
;; cmp w1, w2, uxtx
;; sub sp, x28, #4
-;; b.ne #0x1cc
+;; b.ne #0x1c4
;; 158: mov sp, x28
;; sub x28, x28, #8
;; mov sp, x28
@@ -121,9 +121,7 @@
;; mov x1, x9
;; ldur w2, [x28, #4]
;; blr x4
-;; 194: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; 194: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
@@ -132,7 +130,7 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
+;; 1b8: udf #0xc11f
+;; 1bc: udf #0xc11f
;; 1c0: udf #0xc11f
;; 1c4: udf #0xc11f
-;; 1c8: udf #0xc11f
-;; 1cc: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/callee_pop/cleanup.wat b/tests/disas/winch/aarch64/callee_pop/cleanup.wat
new file mode 100644
index 0000000000..3522648d0a
--- /dev/null
+++ b/tests/disas/winch/aarch64/callee_pop/cleanup.wat
@@ -0,0 +1,482 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+;; Nonconstant arguments force spills. Keep a value live below the arguments,
+;; then consume both register results and a separate stack-result area.
+(module
+ (type $single (func (param i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) (result i64)))
+ (func $single (type $single) local.get 0 local.get 9 i64.add)
+ (func $multi (param i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) (result i64 i64 f64)
+ local.get 0 local.get 9 f64.const 3.5)
+ (table funcref (elem $single))
+ (func (export "direct") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ call $single i64.add)
+ (func (export "indirect") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ i32.const 0 call_indirect (type $single) i64.add)
+ (func (export "stack_results") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ call $multi i64.trunc_f64_s i64.add i64.add i64.add))
+;; wasm[0]::function[0]::single:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x40
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x84
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x40
+;; mov sp, x28
+;; stur x0, [x28, #0x38]
+;; stur x1, [x28, #0x30]
+;; stur x2, [x28, #0x28]
+;; stur x3, [x28, #0x20]
+;; stur x4, [x28, #0x18]
+;; stur x5, [x28, #0x10]
+;; stur x6, [x28, #8]
+;; stur x7, [x28]
+;; ldur x0, [x29, #0x28]
+;; ldur x1, [x28, #0x28]
+;; add x1, x1, x0, uxtx
+;; mov x0, x1
+;; add x28, x28, #0x40
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; add sp, sp, #0x20
+;; ret
+;; 84: udf #0xc11f
+;;
+;; wasm[0]::function[1]::multi:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x1, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x50
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x15c
+;; cc: mov x9, x1
+;; sub x28, x28, #0x40
+;; mov sp, x28
+;; stur x1, [x28, #0x38]
+;; stur x2, [x28, #0x30]
+;; stur x3, [x28, #0x28]
+;; stur x4, [x28, #0x20]
+;; stur x5, [x28, #0x18]
+;; stur x6, [x28, #0x10]
+;; stur x7, [x28, #8]
+;; stur x0, [x28]
+;; fmov d0, #3.50000000
+;; ldur x16, [x28, #0x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x29, #0x30]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x0, [x28, #0x10]
+;; ldur x16, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x0]
+;; ldur x16, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x0, #8]
+;; add x28, x28, #0x40
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; add sp, sp, #0x30
+;; ret
+;; 15c: udf #0xc11f
+;;
+;; wasm[0]::function[2]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x90
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x2dc
+;; 18c: mov x9, x0
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x0, [x28, #0x10]
+;; stur x1, [x28, #8]
+;; stur x2, [x28]
+;; ldur x16, [x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #8]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x10]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x18]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x20]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x30]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x38]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x40]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x48]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x50]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; mov x0, x9
+;; mov x1, x9
+;; ldur x2, [x28, #0x68]
+;; ldur x3, [x28, #0x60]
+;; ldur x4, [x28, #0x58]
+;; ldur x5, [x28, #0x50]
+;; ldur x6, [x28, #0x48]
+;; ldur x7, [x28, #0x40]
+;; ldur x16, [x28, #0x38]
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x30]
+;; stur x16, [x28, #8]
+;; ldur x16, [x28, #0x28]
+;; stur x16, [x28, #0x10]
+;; ldur x16, [x28, #0x20]
+;; stur x16, [x28, #0x18]
+;; bl #0
+;; 2a0: mov x28, sp
+;; add x28, x28, #0x50
+;; mov sp, x28
+;; ldur x9, [x28, #0x18]
+;; ldur x1, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; add x1, x1, x0, uxtx
+;; mov x0, x1
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 2dc: udf #0xc11f
+;;
+;; wasm[0]::function[3]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x90
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x510
+;; 30c: mov x9, x0
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x0, [x28, #0x10]
+;; stur x1, [x28, #8]
+;; stur x2, [x28]
+;; ldur x16, [x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #8]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x10]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x18]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x20]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x30]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x38]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x40]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x48]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x50]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; mov x1, #0
+;; mov x2, x9
+;; ldur x3, [x2, #0x38]
+;; cmp x1, x3, uxtx
+;; b.hs #0x514
+;; 3e8: mov x16, x1
+;; mov x17, #8
+;; mul x16, x16, x17
+;; ldur x2, [x2, #0x30]
+;; mov x4, x2
+;; add x2, x2, x16, uxtx
+;; cmp x1, x3, uxtx
+;; csel x2, x4, x2, hs
+;; ldur x0, [x2]
+;; tst x0, x0
+;; b.ne #0x44c
+;; b #0x418
+;; 418: sub x28, x28, #4
+;; mov sp, x28
+;; stur w1, [x28]
+;; sub x28, x28, #0xc
+;; mov sp, x28
+;; mov x0, x9
+;; mov x1, #0
+;; ldur w2, [x28, #0xc]
+;; bl #0xce0
+;; 43c: add x28, x28, #0x10
+;; mov sp, x28
+;; ldur x9, [x28, #0x68]
+;; b #0x450
+;; 44c: and x0, x0, #0xfffffffffffffffe
+;; cbz x0, #0x518
+;; 454: ldur x16, [x9, #0x28]
+;; ldur w1, [x16]
+;; ldur w2, [x0, #0x10]
+;; cmp w1, w2, uxtx
+;; b.ne #0x51c
+;; 468: sub x28, x28, #8
+;; mov sp, x28
+;; stur x0, [x28]
+;; ldur x8, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; ldur x11, [x8, #0x18]
+;; ldur x10, [x8, #8]
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; mov x0, x11
+;; mov x1, x9
+;; ldur x2, [x28, #0x68]
+;; ldur x3, [x28, #0x60]
+;; ldur x4, [x28, #0x58]
+;; ldur x5, [x28, #0x50]
+;; ldur x6, [x28, #0x48]
+;; ldur x7, [x28, #0x40]
+;; ldur x16, [x28, #0x38]
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x30]
+;; stur x16, [x28, #8]
+;; ldur x16, [x28, #0x28]
+;; stur x16, [x28, #0x10]
+;; ldur x16, [x28, #0x20]
+;; stur x16, [x28, #0x18]
+;; blr x10
+;; 4d4: mov x28, sp
+;; add x28, x28, #0x50
+;; mov sp, x28
+;; ldur x9, [x28, #0x18]
+;; ldur x1, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; add x1, x1, x0, uxtx
+;; mov x0, x1
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 510: udf #0xc11f
+;; 514: udf #0xc11f
+;; 518: udf #0xc11f
+;; 51c: udf #0xc11f
+;;
+;; wasm[0]::function[4]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0xb0
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x700
+;; 54c: mov x9, x0
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x0, [x28, #0x10]
+;; stur x1, [x28, #8]
+;; stur x2, [x28]
+;; ldur x16, [x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #8]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x10]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x18]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x20]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x28]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x30]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x38]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x40]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x48]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x50]
+;; sub x28, x28, #8
+;; mov sp, x28
+;; stur x16, [x28]
+;; sub x28, x28, #0x10
+;; mov sp, x28
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; mov x1, x9
+;; mov x2, x9
+;; ldur x3, [x28, #0x88]
+;; ldur x4, [x28, #0x80]
+;; ldur x5, [x28, #0x78]
+;; ldur x6, [x28, #0x70]
+;; ldur x7, [x28, #0x68]
+;; ldur x16, [x28, #0x60]
+;; stur x16, [x28]
+;; ldur x16, [x28, #0x58]
+;; stur x16, [x28, #8]
+;; ldur x16, [x28, #0x50]
+;; stur x16, [x28, #0x10]
+;; ldur x16, [x28, #0x48]
+;; stur x16, [x28, #0x18]
+;; ldur x16, [x28, #0x40]
+;; stur x16, [x28, #0x20]
+;; add x0, x28, #0x30
+;; bl #0xa0
+;; 670: mov x28, sp
+;; ldur x16, [x28, #8]
+;; stur x16, [x28, #0x58]
+;; ldur x16, [x28]
+;; stur x16, [x28, #0x50]
+;; add x28, x28, #0x50
+;; mov sp, x28
+;; ldur x9, [x28, #0x28]
+;; fcmp d0, d0
+;; b.vs #0x704
+;; 698: ldr d31, #0x710
+;; fcmp d0, d31
+;; b.le #0x708
+;; 6a4: ldr d31, #0x718
+;; fcmp d0, d31
+;; b.ge #0x70c
+;; 6b0: fcvtzs x0, d0
+;; ldur x1, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; add x1, x1, x0, uxtx
+;; ldur x0, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; add x0, x0, x1, uxtx
+;; ldur x1, [x28]
+;; add x28, x28, #8
+;; mov sp, x28
+;; add x1, x1, x0, uxtx
+;; mov x0, x1
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 700: udf #0xc11f
+;; 704: udf #0xc11f
+;; 708: udf #0xc11f
+;; 70c: udf #0xc11f
+;; 710: udf #1
+;; 714: .byte 0x00, 0x00, 0xe0, 0xc3
+;; 718: udf #0
+;; 71c: .byte 0x00, 0x00, 0xe0, 0x43
diff --git a/tests/disas/winch/aarch64/exceptions/catch-ref-drc.wat b/tests/disas/winch/aarch64/exceptions/catch-ref-drc.wat
index b06f1bb935..0c53fd53e4 100644
--- a/tests/disas/winch/aarch64/exceptions/catch-ref-drc.wat
+++ b/tests/disas/winch/aarch64/exceptions/catch-ref-drc.wat
@@ -24,16 +24,16 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x2cc
+;; b.lo #0x2bc
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x48c
+;; bl #0x47c
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x158
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x148
;; 48: ldur x9, [x28, #8]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #0xc]
@@ -50,12 +50,10 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x28
;; mov x4, #8
-;; bl #0x43c
+;; bl #0x42c
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x158
-;; 8c: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x148
+;; 8c: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0xc]
;; ldur x1, [x9, #8]
@@ -71,21 +69,21 @@
;; stur w16, [x2, #0x1c]
;; mov x1, #0
;; tst w1, w1
-;; b.eq #0x120
-;; b #0xdc
-;; dc: mov w16, w1
+;; b.eq #0x118
+;; b #0xd4
+;; d4: mov w16, w1
;; and w16, w16, #1
;; tst w16, w16
-;; b.ne #0x120
-;; b #0xf0
-;; f0: ldur x3, [x9, #8]
+;; b.ne #0x118
+;; b #0xe8
+;; e8: ldur x3, [x9, #8]
;; ldur x4, [x3, #0x28]
;; ldur x3, [x3, #0x20]
;; mov x16, x1
;; add x16, x16, #0x10
;; cmp x16, x4, uxtx
-;; b.hi #0x2d0
-;; 10c: mov x5, x3
+;; b.hi #0x2c0
+;; 104: mov x5, x3
;; add x5, x5, x1, uxtx
;; ldur x6, [x5, #8]
;; add x6, x6, #1
@@ -98,12 +96,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x4bc
+;; bl #0x4ac
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x158
-;; 144: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x148
+;; 13c: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -118,8 +114,8 @@
;; mov x16, x0
;; add x16, x16, #0x28
;; cmp x16, x2, uxtx
-;; b.hi #0x2d4
-;; 18c: mov x2, x1
+;; b.hi #0x2c4
+;; 17c: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w1, [x2, #0x20]
;; sub x28, x28, #8
@@ -133,28 +129,28 @@
;; stur w1, [x28]
;; ldur w0, [x28]
;; tst w0, w0
-;; b.eq #0x278
-;; b #0x1cc
-;; 1cc: mov w16, w0
+;; b.eq #0x268
+;; b #0x1bc
+;; 1bc: mov w16, w0
;; and w16, w16, #1
;; tst w16, w16
-;; b.ne #0x278
-;; b #0x1e0
-;; 1e0: ldur x1, [x9, #8]
+;; b.ne #0x268
+;; b #0x1d0
+;; 1d0: ldur x1, [x9, #8]
;; ldur x2, [x1, #0x28]
;; ldur x1, [x1, #0x20]
;; mov x16, x0
;; add x16, x16, #0x14
;; cmp x16, x2, uxtx
-;; b.hi #0x2d8
-;; 1fc: mov x2, x1
+;; b.hi #0x2c8
+;; 1ec: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w16, [x2]
;; and w16, w16, #2
;; tst w16, w16
-;; b.ne #0x278
-;; b #0x218
-;; 218: ldur x3, [x9, #0x20]
+;; b.ne #0x268
+;; b #0x208
+;; 208: ldur x3, [x9, #0x20]
;; ldur w16, [x3]
;; stur w16, [x2, #0x10]
;; ldur w16, [x2]
@@ -170,14 +166,14 @@
;; ldur w16, [x3, #8]
;; add w16, w16, w16, uxtx
;; cmp w4, w16, uxtx
-;; b.lo #0x278
-;; b #0x260
-;; 260: cmp w4, #0x400
-;; b.lo #0x278
-;; b #0x26c
-;; 26c: mov x0, x9
-;; bl #0x510
-;; 274: ldur x9, [x28, #0x18]
+;; b.lo #0x268
+;; b #0x250
+;; 250: cmp w4, #0x400
+;; b.lo #0x268
+;; b #0x25c
+;; 25c: mov x0, x9
+;; bl #0x500
+;; 264: ldur x9, [x28, #0x18]
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[0, 4]
;; ldur w0, [x28]
;; add x28, x28, #4
@@ -200,7 +196,7 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 2cc: udf #0xc11f
-;; 2d0: udf #0xc11f
-;; 2d4: udf #0xc11f
-;; 2d8: udf #0xc11f
+;; 2bc: udf #0xc11f
+;; 2c0: udf #0xc11f
+;; 2c4: udf #0xc11f
+;; 2c8: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/catch-ref-null.wat b/tests/disas/winch/aarch64/exceptions/catch-ref-null.wat
index 0418629e4c..96b17173b3 100644
--- a/tests/disas/winch/aarch64/exceptions/catch-ref-null.wat
+++ b/tests/disas/winch/aarch64/exceptions/catch-ref-null.wat
@@ -23,25 +23,25 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x2d0
+;; b.lo #0x2b8
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x51c
+;; bl #0x504
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x1dc
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x1cc
;; 48: ldur x9, [x28, #8]
;; ldur x16, [x9, #0x20]
;; ldur w1, [x16]
;; adds w1, w1, #7
-;; b.hs #0x2d4
+;; b.hs #0x2bc
;; 5c: and w1, w1, #0xfffffff8
;; mov w2, w1
;; adds w2, w2, #0x18
-;; b.hs #0x2d8
+;; b.hs #0x2c0
;; 6c: sub x28, x28, #4
;; mov sp, x28
;; stur w0, [x28]
@@ -66,9 +66,9 @@
;; stur x0, [x28]
;; mov x0, x9
;; ldur x1, [x28]
-;; bl #0x440
+;; bl #0x428
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x1dc
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x1cc
;; d0: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
@@ -110,12 +110,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #0xc]
-;; bl #0x494
+;; bl #0x47c
;; ├─╼ exception frame offset: SP = FP - 0x40
-;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x1dc
-;; 178: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #8
+;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x1cc
+;; 178: add x28, x28, #0x14
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; ldur w1, [x28]
@@ -132,12 +130,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x54c
+;; bl #0x534
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x1dc
-;; 1c8: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x1cc
+;; 1c0: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -152,8 +148,8 @@
;; mov x16, x0
;; add x16, x16, #0x18
;; cmp x16, x2, uxtx
-;; b.hi #0x2dc
-;; 210: mov x2, x1
+;; b.hi #0x2c4
+;; 200: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w1, [x2, #0x10]
;; sub x28, x28, #8
@@ -168,8 +164,8 @@
;; mov x0, x9
;; ldur w1, [x28]
;; mov x2, #0xffffffff
-;; bl #0x4c4
-;; 250: add x28, x28, #4
+;; bl #0x4ac
+;; 240: add x28, x28, #4
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4]
;; mov sp, x28
;; ldur x9, [x28, #0x14]
@@ -190,12 +186,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x54c
-;; 2a4: add x28, x28, #4
+;; bl #0x534
+;; 294: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x10
;; mov sp, x28
@@ -203,7 +197,7 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 2d0: udf #0xc11f
-;; 2d4: udf #0xc11f
-;; 2d8: udf #0xc11f
-;; 2dc: udf #0xc11f
+;; 2b8: udf #0xc11f
+;; 2bc: udf #0xc11f
+;; 2c0: udf #0xc11f
+;; 2c4: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/catch-ref.wat b/tests/disas/winch/aarch64/exceptions/catch-ref.wat
index b2665f8395..fbd9b12f9f 100644
--- a/tests/disas/winch/aarch64/exceptions/catch-ref.wat
+++ b/tests/disas/winch/aarch64/exceptions/catch-ref.wat
@@ -23,16 +23,16 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x264
+;; b.lo #0x244
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x4a4
+;; bl #0x484
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x170
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x158
;; 48: ldur x9, [x28, #8]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #0xc]
@@ -50,12 +50,10 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x20
;; mov x4, #0x10
-;; bl #0x3cc
+;; bl #0x3ac
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x170
-;; 90: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x158
+;; 90: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0xc]
;; ldur x1, [x9, #8]
@@ -83,12 +81,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #0xc]
-;; bl #0x41c
+;; bl #0x3fc
;; ├─╼ exception frame offset: SP = FP - 0x40
-;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x170
-;; 10c: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #8
+;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x158
+;; 104: add x28, x28, #0x14
;; mov sp, x28
;; ldur x9, [x28, #0x14]
;; ldur w1, [x28]
@@ -105,12 +101,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x4d4
+;; bl #0x4b4
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x170
-;; 15c: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x158
+;; 14c: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -125,8 +119,8 @@
;; mov x16, x0
;; add x16, x16, #0x20
;; cmp x16, x2, uxtx
-;; b.hi #0x268
-;; 1a4: mov x2, x1
+;; b.hi #0x248
+;; 18c: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w1, [x2, #0x18]
;; sub x28, x28, #8
@@ -141,8 +135,8 @@
;; mov x0, x9
;; ldur w1, [x28]
;; mov x2, #0xffffffff
-;; bl #0x44c
-;; 1e4: add x28, x28, #4
+;; bl #0x42c
+;; 1cc: add x28, x28, #4
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4]
;; mov sp, x28
;; ldur x9, [x28, #0x14]
@@ -163,12 +157,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x4d4
-;; 238: add x28, x28, #4
+;; bl #0x4b4
+;; 220: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x10
;; mov sp, x28
@@ -176,5 +168,5 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 264: udf #0xc11f
-;; 268: udf #0xc11f
+;; 244: udf #0xc11f
+;; 248: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/landing-pads.wat b/tests/disas/winch/aarch64/exceptions/landing-pads.wat
index 3b28571619..8095eeb2d2 100644
--- a/tests/disas/winch/aarch64/exceptions/landing-pads.wat
+++ b/tests/disas/winch/aarch64/exceptions/landing-pads.wat
@@ -26,7 +26,7 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x118
+;; b.lo #0x110
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -46,22 +46,20 @@
;; ldur w2, [x28, #4]
;; blr x3
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ├─╼ exception handler: tag=0, context at [SP+0x18], handler=0xa8
-;; ╰─╼ exception handler: default handler, context at [SP+0x18], handler=0x8c
-;; 74: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; ├─╼ exception handler: tag=0, context at [SP+0x18], handler=0xa0
+;; ╰─╼ exception handler: default handler, context at [SP+0x18], handler=0x84
+;; 74: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
-;; b #0xf4
-;; 8c: mov x28, x29
+;; b #0xec
+;; 84: mov x28, x29
;; sub x28, x28, #0x10
;; mov sp, x28
;; sub x28, x28, #0x18
;; mov sp, x28
;; ldur x9, [x28, #0x10]
-;; b #0xfc
-;; a8: mov x28, x29
+;; b #0xf4
+;; a0: mov x28, x29
;; sub x28, x28, #0x10
;; mov sp, x28
;; sub x28, x28, #0x18
@@ -74,20 +72,20 @@
;; add x16, x16, #0x18
;; cmp x16, x2, uxtx
;; sub sp, x28, #8
-;; b.hi #0x11c
-;; e0: mov sp, x28
+;; b.hi #0x114
+;; d8: mov sp, x28
;; mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w0, [x2, #0x10]
-;; b #0x100
-;; f4: mov x0, #0
-;; b #0x100
-;; fc: mov x0, #0xffffffff
+;; b #0xf8
+;; ec: mov x0, #0
+;; b #0xf8
+;; f4: mov x0, #0xffffffff
;; add x28, x28, #0x18
;; mov sp, x28
;; mov sp, x28
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 118: udf #0xc11f
-;; 11c: udf #0xc11f
+;; 110: udf #0xc11f
+;; 114: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-drc.wat b/tests/disas/winch/aarch64/exceptions/throw-drc.wat
index 178ea237cf..af1cf92d9e 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-drc.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-drc.wat
@@ -21,16 +21,16 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x160
+;; b.lo #0x150
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x2a0
+;; bl #0x290
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x108
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xf8
;; 48: ldur x9, [x28, #8]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #8]
@@ -47,12 +47,10 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x28
;; mov x4, #8
-;; bl #0x250
+;; bl #0x240
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108
-;; 8c: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xf8
+;; 8c: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0xc]
;; ldur x1, [x9, #8]
@@ -75,12 +73,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x2d0
+;; bl #0x2c0
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108
-;; f4: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xf8
+;; ec: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -95,8 +91,8 @@
;; mov x16, x0
;; add x16, x16, #0x28
;; cmp x16, x2, uxtx
-;; b.hi #0x164
-;; 13c: mov x2, x1
+;; b.hi #0x154
+;; 12c: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w0, [x2, #0x20]
;; add x28, x28, #0x10
@@ -105,5 +101,5 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 160: udf #0xc11f
-;; 164: udf #0xc11f
+;; 150: udf #0xc11f
+;; 154: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-externref-drc.wat b/tests/disas/winch/aarch64/exceptions/throw-externref-drc.wat
index a85b3e8208..7a5af0e414 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-externref-drc.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-externref-drc.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1a4
+;; b.lo #0x194
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -32,7 +32,7 @@
;; sub x28, x28, #4
;; mov sp, x28
;; mov x0, x9
-;; bl #0x274
+;; bl #0x264
;; 64: add x28, x28, #4
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4, 12]
;; mov sp, x28
@@ -52,12 +52,10 @@
;; ldur w2, [x28, #0xc]
;; mov x3, #0x28
;; mov x4, #8
-;; bl #0x224
-;; b0: add x28, x28, #0xc
+;; bl #0x214
+;; b0: add x28, x28, #0x10
;; ╰─╼ stack_map: frame_size=64, frame_offsets=[20, 28]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x18]
;; ldur x1, [x9, #8]
;; ldur x2, [x1, #0x28]
@@ -74,22 +72,22 @@
;; add x28, x28, #4
;; mov sp, x28
;; tst w1, w1
-;; b.eq #0x154
-;; b #0x108
-;; 108: mov w16, w1
+;; b.eq #0x14c
+;; b #0x100
+;; 100: mov w16, w1
;; and w16, w16, #1
;; tst w16, w16
-;; b.ne #0x154
-;; b #0x11c
-;; 11c: ldur x3, [x9, #8]
+;; b.ne #0x14c
+;; b #0x114
+;; 114: ldur x3, [x9, #8]
;; ldur x4, [x3, #0x28]
;; ldur x3, [x3, #0x20]
;; mov x16, x1
;; add x16, x16, #0x10
;; cmp x16, x4, uxtx
;; sub sp, x28, #8
-;; b.hi #0x1a8
-;; 13c: mov sp, x28
+;; b.hi #0x198
+;; 134: mov sp, x28
;; mov x5, x3
;; add x5, x5, x1, uxtx
;; ldur x6, [x5, #8]
@@ -103,12 +101,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x2a4
-;; 178: add x28, x28, #4
+;; bl #0x294
+;; 170: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
;; mov sp, x28
@@ -116,5 +112,5 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 1a4: udf #0xc11f
-;; 1a8: udf #0xc11f
+;; 194: udf #0xc11f
+;; 198: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-externref-null.wat b/tests/disas/winch/aarch64/exceptions/throw-externref-null.wat
index 49d2a6e582..739b45f928 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-externref-null.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-externref-null.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1dc
+;; b.lo #0x1cc
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -32,7 +32,7 @@
;; sub x28, x28, #4
;; mov sp, x28
;; mov x0, x9
-;; bl #0x2b4
+;; bl #0x2a4
;; 64: add x28, x28, #4
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4, 12]
;; mov sp, x28
@@ -41,13 +41,13 @@
;; ldur w1, [x16]
;; sub sp, x28, #4
;; adds w1, w1, #7
-;; b.hs #0x1e0
+;; b.hs #0x1d0
;; 84: mov sp, x28
;; and w1, w1, #0xfffffff8
;; mov w2, w1
;; sub sp, x28, #4
;; adds w2, w2, #0x18
-;; b.hs #0x1e4
+;; b.hs #0x1d4
;; 9c: mov sp, x28
;; sub x28, x28, #4
;; mov sp, x28
@@ -65,7 +65,7 @@
;; ldur x2, [x1, #0x28]
;; ldur x1, [x1, #0x20]
;; cmp x0, x2, uxtx
-;; b.ls #0x120
+;; b.ls #0x118
;; b #0xe8
;; e8: sub x0, x0, x2, uxtx
;; sub x28, x28, #8
@@ -75,12 +75,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #4]
-;; bl #0x260
-;; 10c: add x28, x28, #4
+;; bl #0x250
+;; 10c: add x28, x28, #0xc
;; ╰─╼ stack_map: frame_size=64, frame_offsets=[20, 28]
;; mov sp, x28
-;; add x28, x28, #8
-;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur w0, [x28]
;; add x28, x28, #4
@@ -117,12 +115,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x2e4
-;; 1b0: add x28, x28, #4
+;; bl #0x2d4
+;; 1a8: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
;; mov sp, x28
@@ -130,6 +126,6 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 1dc: udf #0xc11f
-;; 1e0: udf #0xc11f
-;; 1e4: udf #0xc11f
+;; 1cc: udf #0xc11f
+;; 1d0: udf #0xc11f
+;; 1d4: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-externref.wat b/tests/disas/winch/aarch64/exceptions/throw-externref.wat
index 4d8dc429c9..89008923b3 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-externref.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-externref.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x150
+;; b.lo #0x140
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -32,7 +32,7 @@
;; sub x28, x28, #4
;; mov sp, x28
;; mov x0, x9
-;; bl #0x21c
+;; bl #0x20c
;; 64: add x28, x28, #4
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[4, 12]
;; mov sp, x28
@@ -53,12 +53,10 @@
;; ldur w2, [x28, #0xc]
;; mov x3, #0x20
;; mov x4, #0x10
-;; bl #0x1cc
-;; b4: add x28, x28, #0xc
+;; bl #0x1bc
+;; b4: add x28, x28, #0x10
;; ╰─╼ stack_map: frame_size=64, frame_offsets=[20, 28]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x18]
;; ldur x1, [x9, #8]
;; ldur x2, [x1, #0x28]
@@ -82,12 +80,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x24c
-;; 124: add x28, x28, #4
+;; bl #0x23c
+;; 11c: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
;; mov sp, x28
@@ -95,4 +91,4 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 150: udf #0xc11f
+;; 140: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-funcref-drc.wat b/tests/disas/winch/aarch64/exceptions/throw-funcref-drc.wat
index f641d62430..311a6314ff 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-funcref-drc.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-funcref-drc.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1a0
+;; b.lo #0x188
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -30,7 +30,7 @@
;; mov sp, x28
;; stur x16, [x28]
;; mov x0, x9
-;; bl #0x29c
+;; bl #0x284
;; 5c: ldur x9, [x28, #0x18]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #4]
@@ -47,10 +47,8 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x28
;; mov x4, #8
-;; bl #0x21c
-;; a0: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x204
+;; a0: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur x1, [x9, #8]
@@ -80,10 +78,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #4]
-;; bl #0x26c
-;; 124: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #8
+;; bl #0x254
+;; 11c: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur w1, [x28]
@@ -100,10 +96,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x2cc
-;; 174: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x2b4
+;; 164: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
@@ -112,4 +106,4 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 1a0: udf #0xc11f
+;; 188: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-funcref-null.wat b/tests/disas/winch/aarch64/exceptions/throw-funcref-null.wat
index 095e330ba0..f20ac0bfe5 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-funcref-null.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-funcref-null.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x210
+;; b.lo #0x200
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -30,16 +30,16 @@
;; mov sp, x28
;; stur x16, [x28]
;; mov x0, x9
-;; bl #0x318
+;; bl #0x308
;; 5c: ldur x9, [x28, #0x18]
;; ldur x16, [x9, #0x20]
;; ldur w1, [x16]
;; adds w1, w1, #7
-;; b.hs #0x214
+;; b.hs #0x204
;; 70: and w1, w1, #0xfffffff8
;; mov w2, w1
;; adds w2, w2, #0x18
-;; b.hs #0x218
+;; b.hs #0x208
;; 80: sub x28, x28, #4
;; mov sp, x28
;; stur w0, [x28]
@@ -64,7 +64,7 @@
;; stur x0, [x28]
;; mov x0, x9
;; ldur x1, [x28]
-;; bl #0x294
+;; bl #0x284
;; e4: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x20]
@@ -108,10 +108,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #4]
-;; bl #0x2e8
-;; 194: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #8
+;; bl #0x2d8
+;; 194: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur w1, [x28]
@@ -128,10 +126,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x348
-;; 1e4: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x338
+;; 1dc: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
@@ -140,6 +136,6 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 210: udf #0xc11f
-;; 214: udf #0xc11f
-;; 218: udf #0xc11f
+;; 200: udf #0xc11f
+;; 204: udf #0xc11f
+;; 208: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-funcref.wat b/tests/disas/winch/aarch64/exceptions/throw-funcref.wat
index c360dfa062..25c598d2c4 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-funcref.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-funcref.wat
@@ -18,7 +18,7 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1a4
+;; b.lo #0x18c
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -30,7 +30,7 @@
;; mov sp, x28
;; stur x16, [x28]
;; mov x0, x9
-;; bl #0x2a0
+;; bl #0x288
;; 5c: ldur x9, [x28, #0x18]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #4]
@@ -48,10 +48,8 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x20
;; mov x4, #0x10
-;; bl #0x220
-;; a4: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x208
+;; a4: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur x1, [x9, #8]
@@ -81,10 +79,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur x1, [x28, #4]
-;; bl #0x270
-;; 128: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #8
+;; bl #0x258
+;; 120: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0x1c]
;; ldur w1, [x28]
@@ -101,10 +97,8 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x2d0
-;; 178: add x28, x28, #4
-;; mov sp, x28
-;; add x28, x28, #4
+;; bl #0x2b8
+;; 168: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
@@ -113,4 +107,4 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 1a4: udf #0xc11f
+;; 18c: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw-null.wat b/tests/disas/winch/aarch64/exceptions/throw-null.wat
index d73446d07b..a3a10b9c56 100644
--- a/tests/disas/winch/aarch64/exceptions/throw-null.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw-null.wat
@@ -21,25 +21,25 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x1d0
+;; b.lo #0x1c8
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x31c
+;; bl #0x314
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x178
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x170
;; 48: ldur x9, [x28, #8]
;; ldur x16, [x9, #0x20]
;; ldur w1, [x16]
;; adds w1, w1, #7
-;; b.hs #0x1d4
+;; b.hs #0x1cc
;; 5c: and w1, w1, #0xfffffff8
;; mov w2, w1
;; adds w2, w2, #0x18
-;; b.hs #0x1d8
+;; b.hs #0x1d0
;; 6c: sub x28, x28, #4
;; mov sp, x28
;; stur w0, [x28]
@@ -64,9 +64,9 @@
;; stur x0, [x28]
;; mov x0, x9
;; ldur x1, [x28]
-;; bl #0x2c8
+;; bl #0x2c0
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x170
;; d0: add x28, x28, #8
;; mov sp, x28
;; ldur x9, [x28, #0x10]
@@ -103,12 +103,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x34c
+;; bl #0x344
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178
-;; 164: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x170
+;; 164: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -123,8 +121,8 @@
;; mov x16, x0
;; add x16, x16, #0x18
;; cmp x16, x2, uxtx
-;; b.hi #0x1dc
-;; 1ac: mov x2, x1
+;; b.hi #0x1d4
+;; 1a4: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w0, [x2, #0x10]
;; add x28, x28, #0x10
@@ -133,7 +131,7 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
+;; 1c8: udf #0xc11f
+;; 1cc: udf #0xc11f
;; 1d0: udf #0xc11f
;; 1d4: udf #0xc11f
-;; 1d8: udf #0xc11f
-;; 1dc: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/exceptions/throw.wat b/tests/disas/winch/aarch64/exceptions/throw.wat
index c0c4ceb155..30d6f6251b 100644
--- a/tests/disas/winch/aarch64/exceptions/throw.wat
+++ b/tests/disas/winch/aarch64/exceptions/throw.wat
@@ -21,16 +21,16 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x164
+;; b.lo #0x154
;; 2c: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
;; stur x1, [x28]
;; mov x0, x9
-;; bl #0x2a4
+;; bl #0x294
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x10c
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xfc
;; 48: ldur x9, [x28, #8]
;; ldur x1, [x9, #0x28]
;; ldur w1, [x1, #8]
@@ -48,12 +48,10 @@
;; ldur w2, [x28, #8]
;; mov x3, #0x20
;; mov x4, #0x10
-;; bl #0x254
+;; bl #0x244
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c
-;; 90: add x28, x28, #8
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xfc
+;; 90: add x28, x28, #0xc
;; mov sp, x28
;; ldur x9, [x28, #0xc]
;; ldur x1, [x9, #8]
@@ -76,12 +74,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #0xc]
-;; bl #0x2d4
+;; bl #0x2c4
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c
-;; f8: add x28, x28, #0xc
-;; mov sp, x28
-;; add x28, x28, #4
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xfc
+;; f0: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #8]
;; mov x28, x29
@@ -96,8 +92,8 @@
;; mov x16, x0
;; add x16, x16, #0x20
;; cmp x16, x2, uxtx
-;; b.hi #0x168
-;; 140: mov x2, x1
+;; b.hi #0x158
+;; 130: mov x2, x1
;; add x2, x2, x0, uxtx
;; ldur w0, [x2, #0x18]
;; add x28, x28, #0x10
@@ -106,5 +102,5 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 164: udf #0xc11f
-;; 168: udf #0xc11f
+;; 154: udf #0xc11f
+;; 158: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/global-set-externref-drc.wat b/tests/disas/winch/aarch64/global-set-externref-drc.wat
index 6e5875a579..fe0ef3dd2e 100644
--- a/tests/disas/winch/aarch64/global-set-externref-drc.wat
+++ b/tests/disas/winch/aarch64/global-set-externref-drc.wat
@@ -17,7 +17,7 @@
;; movk x17, #0x20
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x168
+;; b.lo #0x160
;; 2c: mov x9, x0
;; sub x28, x28, #0x18
;; mov sp, x28
@@ -47,7 +47,7 @@
;; add x16, x16, #0x10
;; cmp x16, x2, uxtx
;; sub sp, x28, #8
-;; b.hi #0x16c
+;; b.hi #0x164
;; a4: mov sp, x28
;; mov x4, x1
;; add x4, x4, x0, uxtx
@@ -56,18 +56,18 @@
;; stur x5, [x4, #8]
;; stur w0, [x9, #0x30]
;; tst w3, w3
-;; b.eq #0x150
+;; b.eq #0x148
;; b #0xcc
;; cc: mov w16, w3
;; and w16, w16, #1
;; tst w16, w16
-;; b.ne #0x150
+;; b.ne #0x148
;; b #0xe0
;; e0: mov x16, x3
;; add x16, x16, #0x10
;; cmp x16, x2, uxtx
;; sub sp, x28, #8
-;; b.hi #0x170
+;; b.hi #0x168
;; f4: mov sp, x28
;; mov x4, x1
;; add x4, x4, x3, uxtx
@@ -77,7 +77,7 @@
;; b.eq #0x11c
;; b #0x114
;; 114: stur x5, [x4, #8]
-;; b #0x150
+;; b #0x148
;; 11c: sub x28, x28, #4
;; mov sp, x28
;; stur w3, [x28]
@@ -85,12 +85,10 @@
;; mov sp, x28
;; mov x0, x9
;; ldur w1, [x28, #4]
-;; bl #0x1ec
-;; 13c: add x28, x28, #4
+;; bl #0x1e4
+;; 13c: add x28, x28, #8
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
;; mov sp, x28
-;; add x28, x28, #4
-;; mov sp, x28
;; ldur x9, [x28, #0x10]
;; add x28, x28, #0x18
;; mov sp, x28
@@ -98,6 +96,6 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
+;; 160: udf #0xc11f
+;; 164: udf #0xc11f
;; 168: udf #0xc11f
-;; 16c: udf #0xc11f
-;; 170: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/params/400_params.wat b/tests/disas/winch/aarch64/params/400_params.wat
index 5e72929bc6..4b54f3e4ed 100644
--- a/tests/disas/winch/aarch64/params/400_params.wat
+++ b/tests/disas/winch/aarch64/params/400_params.wat
@@ -60,7 +60,7 @@
;; movk x17, #0x28
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x74
+;; b.lo #0x78
;; 2c: mov x9, x0
;; sub x28, x28, #0x28
;; mov sp, x28
@@ -78,5 +78,6 @@
;; mov sp, x28
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
+;; add sp, sp, #0xc50
;; ret
-;; 74: udf #0xc11f
+;; 78: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_stack_params.wat b/tests/disas/winch/aarch64/v128_stack_params.wat
index 00e6fa7e31..08138a158f 100644
--- a/tests/disas/winch/aarch64/v128_stack_params.wat
+++ b/tests/disas/winch/aarch64/v128_stack_params.wat
@@ -29,7 +29,7 @@
;; movk x17, #0x90
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x9c
+;; b.lo #0xa0
;; 2c: mov x9, x0
;; sub x28, x28, #0x90
;; mov sp, x28
@@ -57,8 +57,9 @@
;; mov sp, x28
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
+;; add sp, sp, #0x20
;; ret
-;; 9c: udf #0xc11f
+;; a0: udf #0xc11f
;;
;; wasm[0]::function[1]:
;; stp x29, x30, [sp, #-0x10]!
@@ -71,8 +72,8 @@
;; movk x17, #0x30
;; add x16, x16, x17
;; cmp sp, x16
-;; b.lo #0x148
-;; cc: mov x9, x0
+;; b.lo #0x164
+;; ec: mov x9, x0
;; sub x28, x28, #0x10
;; mov sp, x28
;; stur x0, [x28, #8]
@@ -81,21 +82,20 @@
;; mov sp, x28
;; mov x0, x9
;; mov x1, x9
-;; ldr q0, #0x150
-;; ldr q1, #0x160
-;; ldr q2, #0x170
-;; ldr q3, #0x180
-;; ldr q4, #0x190
-;; ldr q5, #0x1a0
-;; ldr q6, #0x1b0
-;; ldr q7, #0x1c0
-;; ldr q31, #0x1d0
+;; ldr q0, #0x170
+;; ldr q1, #0x180
+;; ldr q2, #0x190
+;; ldr q3, #0x1a0
+;; ldr q4, #0x1b0
+;; ldr q5, #0x1c0
+;; ldr q6, #0x1d0
+;; ldr q7, #0x1e0
+;; ldr q31, #0x1f0
;; stur q31, [x28]
-;; ldr q31, #0x1e0
+;; ldr q31, #0x200
;; stur q31, [x28, #0x10]
;; bl #0
-;; 124: add x28, x28, #0x20
-;; mov sp, x28
+;; 144: mov x28, sp
;; ldur x9, [x28, #8]
;; add x28, x28, #0x10
;; mov sp, x28
@@ -103,45 +103,46 @@
;; ldr x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 148: udf #0xc11f
-;; 14c: udf #0
-;; 150: udf #0
-;; 154: udf #0
-;; 158: udf #0
-;; 15c: udf #0
-;; 160: udf #1
-;; 164: udf #0
-;; 168: udf #1
+;; 164: udf #0xc11f
+;; 168: udf #0
;; 16c: udf #0
-;; 170: udf #2
+;; 170: udf #0
;; 174: udf #0
-;; 178: udf #2
+;; 178: udf #0
;; 17c: udf #0
-;; 180: udf #3
+;; 180: udf #1
;; 184: udf #0
-;; 188: udf #3
+;; 188: udf #1
;; 18c: udf #0
-;; 190: udf #4
+;; 190: udf #2
;; 194: udf #0
-;; 198: udf #4
+;; 198: udf #2
;; 19c: udf #0
-;; 1a0: udf #5
+;; 1a0: udf #3
;; 1a4: udf #0
-;; 1a8: udf #5
+;; 1a8: udf #3
;; 1ac: udf #0
-;; 1b0: udf #6
+;; 1b0: udf #4
;; 1b4: udf #0
-;; 1b8: udf #6
+;; 1b8: udf #4
;; 1bc: udf #0
-;; 1c0: udf #7
+;; 1c0: udf #5
;; 1c4: udf #0
-;; 1c8: udf #7
+;; 1c8: udf #5
;; 1cc: udf #0
-;; 1d0: udf #8
+;; 1d0: udf #6
;; 1d4: udf #0
-;; 1d8: udf #8
+;; 1d8: udf #6
;; 1dc: udf #0
-;; 1e0: udf #9
+;; 1e0: udf #7
;; 1e4: udf #0
-;; 1e8: udf #9
+;; 1e8: udf #7
;; 1ec: udf #0
+;; 1f0: udf #8
+;; 1f4: udf #0
+;; 1f8: udf #8
+;; 1fc: udf #0
+;; 200: udf #9
+;; 204: udf #0
+;; 208: udf #9
+;; 20c: udf #0
diff --git a/tests/disas/winch/x64/atomic/notify/notify.wat b/tests/disas/winch/x64/atomic/notify/notify.wat
index 98d426f598..27a48a85d7 100644
--- a/tests/disas/winch/x64/atomic/notify/notify.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify.wat
@@ -12,7 +12,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x7d
+;; ja 0x76
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -27,11 +27,10 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x183
-;; addq $4, %rsp
-;; addq $0xc, %rsp
+;; callq 0x17c
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 7d: ud2
+;; 76: ud2
diff --git a/tests/disas/winch/x64/atomic/notify/notify_offset.wat b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
index 19e54fba98..0ba240722b 100644
--- a/tests/disas/winch/x64/atomic/notify/notify_offset.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
@@ -12,7 +12,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x8a
+;; ja 0x83
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -20,7 +20,7 @@
;; movl $0xa, %eax
;; movl $0, %ecx
;; addq $8, %rcx
-;; jb 0x8c
+;; jb 0x85
;; 46: pushq %rcx
;; subq $4, %rsp
;; movl %eax, (%rsp)
@@ -29,12 +29,11 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x192
-;; addq $4, %rsp
-;; addq $0xc, %rsp
+;; callq 0x18b
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 8a: ud2
-;; 8c: ud2
+;; 83: ud2
+;; 85: ud2
diff --git a/tests/disas/winch/x64/atomic/wait/wait32.wat b/tests/disas/winch/x64/atomic/wait/wait32.wat
index 3142b7dfc0..d41a4ddd98 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32.wat
@@ -12,7 +12,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x8a
+;; ja 0x83
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -30,11 +30,10 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x190
-;; addq $0xc, %rsp
-;; addq $0x14, %rsp
+;; callq 0x189
+;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 8a: ud2
+;; 83: ud2
diff --git a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
index 97c31b9c88..4bc16177c4 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
@@ -15,7 +15,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x97
+;; ja 0x90
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -24,7 +24,7 @@
;; movl $0, %ecx
;; movl $4, %edx
;; addq $8, %rdx
-;; jb 0x99
+;; jb 0x92
;; 4d: pushq %rdx
;; subq $4, %rsp
;; movl %ecx, (%rsp)
@@ -35,12 +35,11 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x19f
-;; addq $0xc, %rsp
-;; addq $0x14, %rsp
+;; callq 0x198
+;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 97: ud2
-;; 99: ud2
+;; 90: ud2
+;; 92: ud2
diff --git a/tests/disas/winch/x64/atomic/wait/wait64.wat b/tests/disas/winch/x64/atomic/wait/wait64.wat
index f9d488d6d5..deb252e073 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64.wat
@@ -12,7 +12,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x82
+;; ja 0x7b
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -29,11 +29,10 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x188
-;; addq $8, %rsp
-;; addq $0x18, %rsp
+;; callq 0x181
+;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 82: ud2
+;; 7b: ud2
diff --git a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
index 584073f39c..322268cf1b 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
@@ -15,7 +15,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x8f
+;; ja 0x88
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -24,7 +24,7 @@
;; movl $0, %ecx
;; movl $4, %edx
;; addq $8, %rdx
-;; jb 0x91
+;; jb 0x8a
;; 4d: pushq %rdx
;; pushq %rcx
;; pushq %rax
@@ -34,12 +34,11 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x197
-;; addq $8, %rsp
-;; addq $0x18, %rsp
+;; callq 0x190
+;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 8f: ud2
-;; 91: ud2
+;; 88: ud2
+;; 8a: ud2
diff --git a/tests/disas/winch/x64/br_if/as_call_first.wat b/tests/disas/winch/x64/br_if/as_call_first.wat
index 6eca24863b..049bca913c 100644
--- a/tests/disas/winch/x64/br_if/as_call_first.wat
+++ b/tests/disas/winch/x64/br_if/as_call_first.wat
@@ -38,7 +38,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0xd8
+;; ja 0xd1
;; 6c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -46,7 +46,7 @@
;; movl $1, %ecx
;; movl $0xc, %eax
;; testl %ecx, %ecx
-;; jne 0xcf
+;; jne 0xc8
;; 91: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $0xc, %rsp
@@ -56,10 +56,9 @@
;; movl $2, %ecx
;; movl $3, %r8d
;; callq 0
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; d8: ud2
+;; d1: ud2
diff --git a/tests/disas/winch/x64/br_if/as_call_last.wat b/tests/disas/winch/x64/br_if/as_call_last.wat
index 42294ab378..bf9befd856 100644
--- a/tests/disas/winch/x64/br_if/as_call_last.wat
+++ b/tests/disas/winch/x64/br_if/as_call_last.wat
@@ -38,7 +38,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0xd8
+;; ja 0xd1
;; 6c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -46,7 +46,7 @@
;; movl $1, %ecx
;; movl $0xe, %eax
;; testl %ecx, %ecx
-;; jne 0xcf
+;; jne 0xc8
;; 91: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $0xc, %rsp
@@ -56,10 +56,9 @@
;; movl $2, %ecx
;; movl 0xc(%rsp), %r8d
;; callq 0
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; d8: ud2
+;; d1: ud2
diff --git a/tests/disas/winch/x64/br_if/as_call_mid.wat b/tests/disas/winch/x64/br_if/as_call_mid.wat
index d2c4cbffcd..4fe6b42808 100644
--- a/tests/disas/winch/x64/br_if/as_call_mid.wat
+++ b/tests/disas/winch/x64/br_if/as_call_mid.wat
@@ -38,7 +38,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0xd8
+;; ja 0xd1
;; 6c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
@@ -46,7 +46,7 @@
;; movl $1, %ecx
;; movl $0xd, %eax
;; testl %ecx, %ecx
-;; jne 0xcf
+;; jne 0xc8
;; 91: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $0xc, %rsp
@@ -56,10 +56,9 @@
;; movl 0xc(%rsp), %ecx
;; movl $3, %r8d
;; callq 0
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; d8: ud2
+;; d1: ud2
diff --git a/tests/disas/winch/x64/call/params.wat b/tests/disas/winch/x64/call/params.wat
index 87dc038ac5..145b0edf04 100644
--- a/tests/disas/winch/x64/call/params.wat
+++ b/tests/disas/winch/x64/call/params.wat
@@ -43,7 +43,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x60, %r11
;; cmpq %rsp, %r11
-;; ja 0x182
+;; ja 0x174
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -73,9 +73,8 @@
;; movl %r11d, 0x18(%rsp)
;; movl $8, %r11d
;; movl %r11d, 0x20(%rsp)
-;; callq 0x190
-;; addq $0x3c, %rsp
-;; addq $4, %rsp
+;; callq 0x180
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; subq $4, %rsp
;; movl %eax, (%rsp)
@@ -110,14 +109,13 @@
;; movl %r11d, 0x18(%rsp)
;; movl $8, %r11d
;; movl %r11d, 0x20(%rsp)
-;; callq 0x190
-;; addq $0x38, %rsp
-;; addq $8, %rsp
+;; callq 0x180
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 182: ud2
+;; 174: ud2
;;
;; wasm[0]::function[1]::add:
;; pushq %rbp
@@ -126,8 +124,8 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x20a
-;; 1ac: movq %rdi, %r14
+;; ja 0x205
+;; 19c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
;; movq %rsi, 0x10(%rsp)
@@ -153,7 +151,9 @@
;; movl 0x30(%rbp), %eax
;; addl %eax, %ecx
;; movl %ecx, %eax
-;; addq $0x20, %rsp
-;; popq %rbp
+;; movq 0x28(%rsp), %r11
+;; movq %r11, 0x58(%rsp)
+;; movq 0x20(%rsp), %rbp
+;; addq $0x58, %rsp
;; retq
-;; 20a: ud2
+;; 205: ud2
diff --git a/tests/disas/winch/x64/call/recursive.wat b/tests/disas/winch/x64/call/recursive.wat
index 4eb4b2e067..24ff3443f6 100644
--- a/tests/disas/winch/x64/call/recursive.wat
+++ b/tests/disas/winch/x64/call/recursive.wat
@@ -30,7 +30,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0xee
+;; ja 0xe0
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -43,7 +43,7 @@
;; testl %eax, %eax
;; je 0x54
;; 4b: movl 0xc(%rsp), %eax
-;; jmp 0xe5
+;; jmp 0xd7
;; 54: movl 0xc(%rsp), %eax
;; subl $1, %eax
;; subq $4, %rsp
@@ -53,8 +53,7 @@
;; movq %r14, %rsi
;; movl 0xc(%rsp), %edx
;; callq 0
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; movl 0xc(%rsp), %ecx
;; subl $2, %ecx
@@ -67,8 +66,7 @@
;; movq %r14, %rsi
;; movl 8(%rsp), %edx
;; callq 0
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -77,4 +75,4 @@
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; ee: ud2
+;; e0: ud2
diff --git a/tests/disas/winch/x64/call_indirect/call_indirect.wat b/tests/disas/winch/x64/call_indirect/call_indirect.wat
index 17fa2ad96a..7da1cdbb25 100644
--- a/tests/disas/winch/x64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/x64/call_indirect/call_indirect.wat
@@ -37,7 +37,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x22b
+;; ja 0x20f
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -50,7 +50,7 @@
;; testl %eax, %eax
;; je 0x55
;; 4b: movl $1, %eax
-;; jmp 0x222
+;; jmp 0x206
;; 55: movl 0xc(%rsp), %eax
;; subl $2, %eax
;; subq $4, %rsp
@@ -59,7 +59,7 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x22d
+;; jae 0x211
;; 7d: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
@@ -69,27 +69,26 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0xde
+;; jne 0xd7
;; a4: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x34b
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; callq 0x32f
+;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
-;; jmp 0xe4
-;; de: andq $0xfffffffffffffffe, %rax
+;; jmp 0xdd
+;; d7: andq $0xfffffffffffffffe, %rax
;; testq %rax, %rax
-;; je 0x22f
-;; ed: movq 0x28(%r14), %r11
+;; je 0x213
+;; e6: movq 0x28(%r14), %r11
;; movl (%r11), %ecx
;; movl 0x10(%rax), %edx
;; cmpl %edx, %ecx
-;; jne 0x231
-;; ff: pushq %rax
+;; jne 0x215
+;; f8: pushq %rax
;; popq %rcx
;; movq 0x18(%rcx), %r8
;; movq 8(%rcx), %rbx
@@ -98,8 +97,7 @@
;; movq %r14, %rsi
;; movl 0xc(%rsp), %edx
;; callq *%rbx
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; movl 0xc(%rsp), %ecx
;; subl $1, %ecx
@@ -111,8 +109,8 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x233
-;; 162: movq %rcx, %r11
+;; jae 0x217
+;; 154: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
;; movq %rdx, %rsi
@@ -121,27 +119,26 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x1c3
-;; 189: subq $4, %rsp
+;; jne 0x1ae
+;; 17b: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $4, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 4(%rsp), %edx
-;; callq 0x34b
-;; addq $4, %rsp
-;; addq $4, %rsp
+;; callq 0x32f
+;; addq $8, %rsp
;; movq 0x20(%rsp), %r14
-;; jmp 0x1c9
-;; 1c3: andq $0xfffffffffffffffe, %rax
+;; jmp 0x1b4
+;; 1ae: andq $0xfffffffffffffffe, %rax
;; testq %rax, %rax
-;; je 0x235
-;; 1d2: movq 0x28(%r14), %r11
+;; je 0x219
+;; 1bd: movq 0x28(%r14), %r11
;; movl (%r11), %ecx
;; movl 0x10(%rax), %edx
;; cmpl %edx, %ecx
-;; jne 0x237
-;; 1e4: pushq %rax
+;; jne 0x21b
+;; 1cf: pushq %rax
;; popq %rcx
;; movq 0x18(%rcx), %r8
;; movq 8(%rcx), %rbx
@@ -150,8 +147,7 @@
;; movq %r14, %rsi
;; movl 8(%rsp), %edx
;; callq *%rbx
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -160,10 +156,10 @@
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 22b: ud2
-;; 22d: ud2
-;; 22f: ud2
-;; 231: ud2
-;; 233: ud2
-;; 235: ud2
-;; 237: ud2
+;; 20f: ud2
+;; 211: ud2
+;; 213: ud2
+;; 215: ud2
+;; 217: ud2
+;; 219: ud2
+;; 21b: ud2
diff --git a/tests/disas/winch/x64/call_indirect/local_arg.wat b/tests/disas/winch/x64/call_indirect/local_arg.wat
index 10858f12f6..1033373365 100644
--- a/tests/disas/winch/x64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/x64/call_indirect/local_arg.wat
@@ -42,7 +42,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x157
+;; ja 0x149
;; 5c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -55,7 +55,7 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x159
+;; jae 0x14b
;; 9e: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
@@ -65,40 +65,38 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0xff
+;; jne 0xf8
;; c5: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x34a
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; callq 0x33c
+;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
-;; jmp 0x105
-;; ff: andq $0xfffffffffffffffe, %rax
+;; jmp 0xfe
+;; f8: andq $0xfffffffffffffffe, %rax
;; testq %rax, %rax
-;; je 0x15b
-;; 10e: movq 0x28(%r14), %r11
+;; je 0x14d
+;; 107: movq 0x28(%r14), %r11
;; movl (%r11), %ecx
;; movl 0x10(%rax), %edx
;; cmpl %edx, %ecx
-;; jne 0x15d
-;; 120: movq 0x18(%rax), %rbx
+;; jne 0x14f
+;; 119: movq 0x18(%rax), %rbx
;; movq 8(%rax), %rcx
;; subq $0xc, %rsp
;; movq %rbx, %rdi
;; movq %r14, %rsi
;; movl 0xc(%rsp), %edx
;; callq *%rcx
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 157: ud2
-;; 159: ud2
-;; 15b: ud2
-;; 15d: ud2
+;; 149: ud2
+;; 14b: ud2
+;; 14d: ud2
+;; 14f: ud2
diff --git a/tests/disas/winch/x64/callee_pop/cleanup.wat b/tests/disas/winch/x64/callee_pop/cleanup.wat
new file mode 100644
index 0000000000..b272ee421a
--- /dev/null
+++ b/tests/disas/winch/x64/callee_pop/cleanup.wat
@@ -0,0 +1,349 @@
+;;! target = "x86_64"
+;;! test = "winch"
+
+;; Nonconstant arguments force spills. Keep a value live below the arguments,
+;; then consume both register results and a separate stack-result area.
+(module
+ (type $single (func (param i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) (result i64)))
+ (func $single (type $single) local.get 0 local.get 9 i64.add)
+ (func $multi (param i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) (result i64 i64 f64)
+ local.get 0 local.get 9 f64.const 3.5)
+ (table funcref (elem $single))
+ (func (export "direct") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ call $single i64.add)
+ (func (export "indirect") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ i32.const 0 call_indirect (type $single) i64.add)
+ (func (export "stack_results") (param i64) (result i64)
+ local.get 0
+ local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0 local.get 0
+ call $multi i64.trunc_f64_s i64.add i64.add i64.add))
+;; wasm[0]::function[0]::single:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x30, %r11
+;; cmpq %rsp, %r11
+;; ja 0x66
+;; 1c: movq %rdi, %r14
+;; subq $0x30, %rsp
+;; movq %rdi, 0x28(%rsp)
+;; movq %rsi, 0x20(%rsp)
+;; movq %rdx, 0x18(%rsp)
+;; movq %rcx, 0x10(%rsp)
+;; movq %r8, 8(%rsp)
+;; movq %r9, (%rsp)
+;; movq 0x38(%rbp), %rax
+;; movq 0x18(%rsp), %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; movq 0x38(%rsp), %r11
+;; movq %r11, 0x68(%rsp)
+;; movq 0x30(%rsp), %rbp
+;; addq $0x68, %rsp
+;; retq
+;; 66: ud2
+;;
+;; wasm[0]::function[1]::multi:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rsi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x50, %r11
+;; cmpq %rsp, %r11
+;; ja 0xed
+;; 8c: movq %rsi, %r14
+;; subq $0x40, %rsp
+;; movq %rsi, 0x38(%rsp)
+;; movq %rdx, 0x30(%rsp)
+;; movq %rcx, 0x28(%rsp)
+;; movq %r8, 0x20(%rsp)
+;; movq %r9, 0x18(%rsp)
+;; movq %rdi, 8(%rsp)
+;; movsd 0x34(%rip), %xmm0
+;; movq 0x28(%rsp), %r11
+;; pushq %r11
+;; movq 0x40(%rbp), %r11
+;; pushq %r11
+;; movq 0x18(%rsp), %rax
+;; popq %r11
+;; movq %r11, (%rax)
+;; popq %r11
+;; movq %r11, 8(%rax)
+;; movq 8(%rbp), %r11
+;; movq %r11, 0x48(%rbp)
+;; leaq 0x48(%rbp), %r11
+;; movq (%rbp), %rbp
+;; movq %r11, %rsp
+;; retq
+;; ed: ud2
+;; ef: addb %al, (%rax)
+;; f1: addb %al, (%rax)
+;; f3: addb %al, (%rax)
+;; f5: addb %cl, (%rax, %rax, 2)
+;;
+;; wasm[0]::function[2]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0xb0, %r11
+;; cmpq %rsp, %r11
+;; ja 0x202
+;; 11c: movq %rdi, %r14
+;; subq $0x20, %rsp
+;; movq %rdi, 0x18(%rsp)
+;; movq %rsi, 0x10(%rsp)
+;; movq %rdx, 8(%rsp)
+;; movq 8(%rsp), %r11
+;; pushq %r11
+;; movq 0x10(%rsp), %r11
+;; pushq %r11
+;; movq 0x18(%rsp), %r11
+;; pushq %r11
+;; movq 0x20(%rsp), %r11
+;; pushq %r11
+;; movq 0x28(%rsp), %r11
+;; pushq %r11
+;; movq 0x30(%rsp), %r11
+;; pushq %r11
+;; movq 0x38(%rsp), %r11
+;; pushq %r11
+;; movq 0x40(%rsp), %r11
+;; pushq %r11
+;; movq 0x48(%rsp), %r11
+;; pushq %r11
+;; movq 0x50(%rsp), %r11
+;; pushq %r11
+;; movq 0x58(%rsp), %r11
+;; pushq %r11
+;; subq $0x38, %rsp
+;; movq %r14, %rdi
+;; movq %r14, %rsi
+;; movq 0x80(%rsp), %rdx
+;; movq 0x78(%rsp), %rcx
+;; movq 0x70(%rsp), %r8
+;; movq 0x68(%rsp), %r9
+;; movq 0x60(%rsp), %r11
+;; movq %r11, (%rsp)
+;; movq 0x58(%rsp), %r11
+;; movq %r11, 8(%rsp)
+;; movq 0x50(%rsp), %r11
+;; movq %r11, 0x10(%rsp)
+;; movq 0x48(%rsp), %r11
+;; movq %r11, 0x18(%rsp)
+;; movq 0x40(%rsp), %r11
+;; movq %r11, 0x20(%rsp)
+;; movq 0x38(%rsp), %r11
+;; movq %r11, 0x28(%rsp)
+;; callq 0
+;; addq $0x58, %rsp
+;; movq 0x20(%rsp), %r14
+;; popq %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; addq $0x20, %rsp
+;; popq %rbp
+;; retq
+;; 202: ud2
+;;
+;; wasm[0]::function[3]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0xb0, %r11
+;; cmpq %rsp, %r11
+;; ja 0x3aa
+;; 22c: movq %rdi, %r14
+;; subq $0x20, %rsp
+;; movq %rdi, 0x18(%rsp)
+;; movq %rsi, 0x10(%rsp)
+;; movq %rdx, 8(%rsp)
+;; movq 8(%rsp), %r11
+;; pushq %r11
+;; movq 0x10(%rsp), %r11
+;; pushq %r11
+;; movq 0x18(%rsp), %r11
+;; pushq %r11
+;; movq 0x20(%rsp), %r11
+;; pushq %r11
+;; movq 0x28(%rsp), %r11
+;; pushq %r11
+;; movq 0x30(%rsp), %r11
+;; pushq %r11
+;; movq 0x38(%rsp), %r11
+;; pushq %r11
+;; movq 0x40(%rsp), %r11
+;; pushq %r11
+;; movq 0x48(%rsp), %r11
+;; pushq %r11
+;; movq 0x50(%rsp), %r11
+;; pushq %r11
+;; movq 0x58(%rsp), %r11
+;; pushq %r11
+;; movl $0, %ecx
+;; movq %r14, %rdx
+;; movq 0x38(%rdx), %rbx
+;; cmpq %rbx, %rcx
+;; jae 0x3ac
+;; 2a7: movq %rcx, %r11
+;; imulq $8, %r11, %r11
+;; movq 0x30(%rdx), %rdx
+;; movq %rdx, %rsi
+;; addq %r11, %rdx
+;; cmpq %rbx, %rcx
+;; cmovaeq %rsi, %rdx
+;; movq (%rdx), %rax
+;; testq %rax, %rax
+;; jne 0x301
+;; 2ce: subq $4, %rsp
+;; movl %ecx, (%rsp)
+;; subq $4, %rsp
+;; movq %r14, %rdi
+;; movl $0, %esi
+;; movl 4(%rsp), %edx
+;; callq 0xa7a
+;; addq $8, %rsp
+;; movq 0x70(%rsp), %r14
+;; jmp 0x307
+;; 301: andq $0xfffffffffffffffe, %rax
+;; testq %rax, %rax
+;; je 0x3ae
+;; 310: movq 0x28(%r14), %r11
+;; movl (%r11), %ecx
+;; movl 0x10(%rax), %edx
+;; cmpl %edx, %ecx
+;; jne 0x3b0
+;; 322: pushq %rax
+;; popq %rbx
+;; movq 0x18(%rbx), %r12
+;; movq 8(%rbx), %r10
+;; subq $0x38, %rsp
+;; movq %r12, %rdi
+;; movq %r14, %rsi
+;; movq 0x80(%rsp), %rdx
+;; movq 0x78(%rsp), %rcx
+;; movq 0x70(%rsp), %r8
+;; movq 0x68(%rsp), %r9
+;; movq 0x60(%rsp), %r11
+;; movq %r11, (%rsp)
+;; movq 0x58(%rsp), %r11
+;; movq %r11, 8(%rsp)
+;; movq 0x50(%rsp), %r11
+;; movq %r11, 0x10(%rsp)
+;; movq 0x48(%rsp), %r11
+;; movq %r11, 0x18(%rsp)
+;; movq 0x40(%rsp), %r11
+;; movq %r11, 0x20(%rsp)
+;; movq 0x38(%rsp), %r11
+;; movq %r11, 0x28(%rsp)
+;; callq *%r10
+;; addq $0x58, %rsp
+;; movq 0x20(%rsp), %r14
+;; popq %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; addq $0x20, %rsp
+;; popq %rbp
+;; retq
+;; 3aa: ud2
+;; 3ac: ud2
+;; 3ae: ud2
+;; 3b0: ud2
+;;
+;; wasm[0]::function[4]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0xd0, %r11
+;; cmpq %rsp, %r11
+;; ja 0x544
+;; 3dc: movq %rdi, %r14
+;; subq $0x20, %rsp
+;; movq %rdi, 0x18(%rsp)
+;; movq %rsi, 0x10(%rsp)
+;; movq %rdx, 8(%rsp)
+;; movq 8(%rsp), %r11
+;; pushq %r11
+;; movq 0x10(%rsp), %r11
+;; pushq %r11
+;; movq 0x18(%rsp), %r11
+;; pushq %r11
+;; movq 0x20(%rsp), %r11
+;; pushq %r11
+;; movq 0x28(%rsp), %r11
+;; pushq %r11
+;; movq 0x30(%rsp), %r11
+;; pushq %r11
+;; movq 0x38(%rsp), %r11
+;; pushq %r11
+;; movq 0x40(%rsp), %r11
+;; pushq %r11
+;; movq 0x48(%rsp), %r11
+;; pushq %r11
+;; movq 0x50(%rsp), %r11
+;; pushq %r11
+;; movq 0x58(%rsp), %r11
+;; pushq %r11
+;; subq $0x10, %rsp
+;; subq $0x48, %rsp
+;; movq %r14, %rsi
+;; movq %r14, %rdx
+;; movq 0xa0(%rsp), %rcx
+;; movq 0x98(%rsp), %r8
+;; movq 0x90(%rsp), %r9
+;; movq 0x88(%rsp), %r11
+;; movq %r11, (%rsp)
+;; movq 0x80(%rsp), %r11
+;; movq %r11, 8(%rsp)
+;; movq 0x78(%rsp), %r11
+;; movq %r11, 0x10(%rsp)
+;; movq 0x70(%rsp), %r11
+;; movq %r11, 0x18(%rsp)
+;; movq 0x68(%rsp), %r11
+;; movq %r11, 0x20(%rsp)
+;; movq 0x60(%rsp), %r11
+;; movq %r11, 0x28(%rsp)
+;; movq 0x58(%rsp), %r11
+;; movq %r11, 0x30(%rsp)
+;; leaq 0x48(%rsp), %rdi
+;; callq 0x70
+;; addq $8, %rsp
+;; movq 8(%rsp), %r11
+;; movq %r11, 0x58(%rsp)
+;; movq (%rsp), %r11
+;; movq %r11, 0x50(%rsp)
+;; addq $0x50, %rsp
+;; movq 0x30(%rsp), %r14
+;; cvttsd2si %xmm0, %rax
+;; cmpq $1, %rax
+;; jno 0x52c
+;; 4f8: ucomisd %xmm0, %xmm0
+;; jp 0x546
+;; 502: movabsq $14114281232179134464, %r11
+;; movq %r11, %xmm15
+;; ucomisd %xmm15, %xmm0
+;; jb 0x548
+;; 51c: xorpd %xmm15, %xmm15
+;; ucomisd %xmm0, %xmm15
+;; jb 0x54a
+;; 52c: popq %rcx
+;; addq %rax, %rcx
+;; popq %rax
+;; addq %rcx, %rax
+;; popq %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; addq $0x20, %rsp
+;; popq %rbp
+;; retq
+;; 544: ud2
+;; 546: ud2
+;; 548: ud2
+;; 54a: ud2
diff --git a/tests/disas/winch/x64/callee_pop/epilogues.wat b/tests/disas/winch/x64/callee_pop/epilogues.wat
new file mode 100644
index 0000000000..f8285abceb
--- /dev/null
+++ b/tests/disas/winch/x64/callee_pop/epilogues.wat
@@ -0,0 +1,97 @@
+;;! target = "x86_64"
+;;! test = "winch"
+
+;; Pin the ordinary return, compact SP-relative return, and FP-relative
+;; fallback. The last two differ by one local at the compact-frame boundary.
+(module
+ (func (export "no_stack_args") (result i64)
+ i64.const 42)
+ (func (export "compact") (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)
+ (local i64 i64 i64 i64)
+ local.get 0 local.get 7 i64.add)
+ (func (export "fallback") (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)
+ (local i64 i64 i64 i64 i64)
+ local.get 0 local.get 7 i64.add))
+;; wasm[0]::function[0]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x10, %r11
+;; cmpq %rsp, %r11
+;; ja 0x3d
+;; 1c: movq %rdi, %r14
+;; subq $0x10, %rsp
+;; movq %rdi, 8(%rsp)
+;; movq %rsi, (%rsp)
+;; movl $0x2a, %eax
+;; addq $0x10, %rsp
+;; popq %rbp
+;; retq
+;; 3d: ud2
+;;
+;; wasm[0]::function[1]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x50, %r11
+;; cmpq %rsp, %r11
+;; ja 0xbd
+;; 5c: movq %rdi, %r14
+;; subq $0x50, %rsp
+;; movq %rdi, 0x48(%rsp)
+;; movq %rsi, 0x40(%rsp)
+;; movq %rdx, 0x38(%rsp)
+;; movq %rcx, 0x30(%rsp)
+;; movq %r8, 0x28(%rsp)
+;; movq %r9, 0x20(%rsp)
+;; xorq %r11, %r11
+;; movq %r11, 0x18(%rsp)
+;; movq %r11, 0x10(%rsp)
+;; movq %r11, 8(%rsp)
+;; movq %r11, (%rsp)
+;; movq 0x28(%rbp), %rax
+;; movq 0x38(%rsp), %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; movq 0x58(%rsp), %r11
+;; movq %r11, 0x78(%rsp)
+;; movq 0x50(%rsp), %rbp
+;; addq $0x78, %rsp
+;; retq
+;; bd: ud2
+;;
+;; wasm[0]::function[2]:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; movq 8(%rdi), %r11
+;; movq 0x18(%r11), %r11
+;; addq $0x60, %r11
+;; cmpq %rsp, %r11
+;; ja 0x143
+;; dc: movq %rdi, %r14
+;; subq $0x60, %rsp
+;; movq %rdi, 0x58(%rsp)
+;; movq %rsi, 0x50(%rsp)
+;; movq %rdx, 0x48(%rsp)
+;; movq %rcx, 0x40(%rsp)
+;; movq %r8, 0x38(%rsp)
+;; movq %r9, 0x30(%rsp)
+;; xorq %r11, %r11
+;; movq %r11, 0x28(%rsp)
+;; movq %r11, 0x20(%rsp)
+;; movq %r11, 0x18(%rsp)
+;; movq %r11, 0x10(%rsp)
+;; movq %r11, 8(%rsp)
+;; movq 0x28(%rbp), %rax
+;; movq 0x48(%rsp), %rcx
+;; addq %rax, %rcx
+;; movq %rcx, %rax
+;; movq 8(%rbp), %r11
+;; movq %r11, 0x28(%rbp)
+;; leaq 0x28(%rbp), %r11
+;; movq (%rbp), %rbp
+;; movq %r11, %rsp
+;; retq
+;; 143: ud2
diff --git a/tests/disas/winch/x64/exceptions/catch-ref-drc.wat b/tests/disas/winch/x64/exceptions/catch-ref-drc.wat
index b47d554362..56e16919e1 100644
--- a/tests/disas/winch/x64/exceptions/catch-ref-drc.wat
+++ b/tests/disas/winch/x64/exceptions/catch-ref-drc.wat
@@ -20,15 +20,15 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x26b
+;; ja 0x25d
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x3d7
+;; callq 0x3c9
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x13a
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x12c
;; movq 8(%rsp), %r14
;; movq 0x28(%r14), %rcx
;; movl 0xc(%rcx), %ecx
@@ -42,11 +42,10 @@
;; movl 8(%rsp), %edx
;; movl $0x28, %ecx
;; movl $8, %r8d
-;; callq 0x388
+;; callq 0x37a
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x13a
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x12c
+;; addq $0xc, %rsp
;; movq 0xc(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -59,19 +58,19 @@
;; movl $0, 0x1c(%rdx)
;; movl $0, %ecx
;; testl %ecx, %ecx
-;; je 0x107
-;; c0: movl %ecx, %r11d
+;; je 0x100
+;; b9: movl %ecx, %r11d
;; andl $1, %r11d
;; testl %r11d, %r11d
-;; jne 0x107
-;; d3: movq 8(%r14), %rbx
+;; jne 0x100
+;; cc: movq 8(%r14), %rbx
;; movq 0x28(%rbx), %rsi
;; movq 0x20(%rbx), %rbx
;; movq %rcx, %r11
;; addq $0x10, %r11
;; cmpq %rsi, %r11
-;; ja 0x26d
-;; f2: movq %rbx, %rdi
+;; ja 0x25f
+;; eb: movq %rbx, %rdi
;; addq %rcx, %rdi
;; movq 8(%rdi), %r8
;; addq $1, %r8
@@ -82,11 +81,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x404
+;; callq 0x3f6
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x13a
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x12c
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -97,8 +95,8 @@
;; movq %rax, %r11
;; addq $0x28, %r11
;; cmpq %rdx, %r11
-;; ja 0x26f
-;; 168: movq %rcx, %rdx
+;; ja 0x261
+;; 15a: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x20(%rdx), %ecx
;; pushq %rdx
@@ -108,25 +106,25 @@
;; movl %ecx, (%rsp)
;; movl (%rsp), %eax
;; testl %eax, %eax
-;; je 0x23a
-;; 191: movl %eax, %r11d
+;; je 0x22c
+;; 183: movl %eax, %r11d
;; andl $1, %r11d
;; testl %r11d, %r11d
-;; jne 0x23a
-;; 1a4: movq 8(%r14), %rcx
+;; jne 0x22c
+;; 196: movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
;; movq 0x20(%rcx), %rcx
;; movq %rax, %r11
;; addq $0x14, %r11
;; cmpq %rdx, %r11
-;; ja 0x271
-;; 1c3: movq %rcx, %rdx
+;; ja 0x263
+;; 1b5: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl (%rdx), %r11d
;; andl $2, %r11d
;; testl %r11d, %r11d
-;; jne 0x23a
-;; 1dc: movq 0x20(%r14), %rbx
+;; jne 0x22c
+;; 1ce: movq 0x20(%r14), %rbx
;; movl (%rbx), %r11d
;; movl %r11d, 0x10(%rdx)
;; movl (%rdx), %r11d
@@ -142,11 +140,11 @@
;; movl 8(%rbx), %r11d
;; addl %r11d, %r11d
;; cmpl %r11d, %esi
-;; jb 0x23a
-;; 221: cmpl $0x400, %esi
-;; jb 0x23a
-;; 22d: movq %r14, %rdi
-;; callq 0x44b
+;; jb 0x22c
+;; 213: cmpl $0x400, %esi
+;; jb 0x22c
+;; 21f: movq %r14, %rdi
+;; callq 0x43d
;; movq 0x18(%rsp), %r14
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[0, 4]
;; movl (%rsp), %eax
@@ -161,7 +159,7 @@
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 26b: ud2
-;; 26d: ud2
-;; 26f: ud2
-;; 271: ud2
+;; 25d: ud2
+;; 25f: ud2
+;; 261: ud2
+;; 263: ud2
diff --git a/tests/disas/winch/x64/exceptions/catch-ref-null.wat b/tests/disas/winch/x64/exceptions/catch-ref-null.wat
index 62d6831f3c..a28c960df9 100644
--- a/tests/disas/winch/x64/exceptions/catch-ref-null.wat
+++ b/tests/disas/winch/x64/exceptions/catch-ref-null.wat
@@ -19,24 +19,24 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x234
+;; ja 0x21f
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x413
+;; callq 0x3fe
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x185
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x177
;; movq 8(%rsp), %r14
;; movq 0x20(%r14), %r11
;; movl (%r11), %ecx
;; addl $7, %ecx
-;; jb 0x236
+;; jb 0x221
;; 4f: andl $0xfffffff8, %ecx
;; movl %ecx, %edx
;; addl $0x18, %edx
-;; jb 0x238
+;; jb 0x223
;; 63: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $4, %rsp
@@ -54,9 +54,9 @@
;; pushq %rax
;; movq %r14, %rdi
;; movq (%rsp), %rsi
-;; callq 0x354
+;; callq 0x33f
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x185
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x177
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movl (%rsp), %eax
@@ -86,11 +86,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x39b
+;; callq 0x386
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x185
-;; addq $0xc, %rsp
-;; addq $8, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x177
+;; addq $0x14, %rsp
;; movq 0x14(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -101,11 +100,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x440
+;; callq 0x42b
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x185
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x177
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -116,8 +114,8 @@
;; movq %rax, %r11
;; addq $0x18, %r11
;; cmpq %rdx, %r11
-;; ja 0x23a
-;; 1b3: movq %rcx, %rdx
+;; ja 0x225
+;; 1a5: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x10(%rdx), %ecx
;; pushq %rdx
@@ -128,7 +126,7 @@
;; movq %r14, %rdi
;; movl (%rsp), %esi
;; movl $0xffffffff, %edx
-;; callq 0x3c8
+;; callq 0x3b3
;; addq $4, %rsp
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[4]
;; movq 0x14(%rsp), %r14
@@ -142,15 +140,14 @@
;; subq $4, %rsp
;; movq %r14, %rdi
;; movl 4(%rsp), %esi
-;; callq 0x440
-;; addq $4, %rsp
+;; callq 0x42b
+;; addq $8, %rsp
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[4]
-;; addq $4, %rsp
;; movq 0x10(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 234: ud2
-;; 236: ud2
-;; 238: ud2
-;; 23a: ud2
+;; 21f: ud2
+;; 221: ud2
+;; 223: ud2
+;; 225: ud2
diff --git a/tests/disas/winch/x64/exceptions/catch-ref.wat b/tests/disas/winch/x64/exceptions/catch-ref.wat
index 5399845b49..78456c188d 100644
--- a/tests/disas/winch/x64/exceptions/catch-ref.wat
+++ b/tests/disas/winch/x64/exceptions/catch-ref.wat
@@ -19,15 +19,15 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x1d8
+;; ja 0x1bc
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x3bb
+;; callq 0x39f
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x129
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x114
;; movq 8(%rsp), %r14
;; movq 0x28(%r14), %rcx
;; movl 0xc(%rcx), %ecx
@@ -41,11 +41,10 @@
;; movl 8(%rsp), %edx
;; movl $0x20, %ecx
;; movl $0x10, %r8d
-;; callq 0x2f4
+;; callq 0x2d8
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x129
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x114
+;; addq $0xc, %rsp
;; movq 0xc(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -64,11 +63,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x343
+;; callq 0x327
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x129
-;; addq $0xc, %rsp
-;; addq $8, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x28], handler=0x114
+;; addq $0x14, %rsp
;; movq 0x14(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -79,11 +77,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x3e8
+;; callq 0x3cc
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x129
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x114
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -94,8 +91,8 @@
;; movq %rax, %r11
;; addq $0x20, %r11
;; cmpq %rdx, %r11
-;; ja 0x1da
-;; 157: movq %rcx, %rdx
+;; ja 0x1be
+;; 142: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x18(%rdx), %ecx
;; pushq %rdx
@@ -106,7 +103,7 @@
;; movq %r14, %rdi
;; movl (%rsp), %esi
;; movl $0xffffffff, %edx
-;; callq 0x370
+;; callq 0x354
;; addq $4, %rsp
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[4]
;; movq 0x14(%rsp), %r14
@@ -120,13 +117,12 @@
;; subq $4, %rsp
;; movq %r14, %rdi
;; movl 4(%rsp), %esi
-;; callq 0x3e8
-;; addq $4, %rsp
+;; callq 0x3cc
+;; addq $8, %rsp
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[4]
-;; addq $4, %rsp
;; movq 0x10(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 1d8: ud2
-;; 1da: ud2
+;; 1bc: ud2
+;; 1be: ud2
diff --git a/tests/disas/winch/x64/exceptions/landing-pads.wat b/tests/disas/winch/x64/exceptions/landing-pads.wat
index b28f59360c..0f35a8705e 100644
--- a/tests/disas/winch/x64/exceptions/landing-pads.wat
+++ b/tests/disas/winch/x64/exceptions/landing-pads.wat
@@ -22,7 +22,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0xdf
+;; ja 0xd8
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -39,17 +39,16 @@
;; movl 0xc(%rsp), %edx
;; callq *%rax
;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ├─╼ exception handler: tag=0, context at [SP+0x28], handler=0x8b
-;; ╰─╼ exception handler: default handler, context at [SP+0x28], handler=0x77
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ├─╼ exception handler: tag=0, context at [SP+0x28], handler=0x84
+;; ╰─╼ exception handler: default handler, context at [SP+0x28], handler=0x70
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
-;; jmp 0xc7
-;; 77: movq %rbp, %rsp
+;; jmp 0xc0
+;; 70: movq %rbp, %rsp
;; subq $0x20, %rsp
;; movq 0x18(%rsp), %r14
-;; jmp 0xd1
-;; 8b: movq %rbp, %rsp
+;; jmp 0xca
+;; 84: movq %rbp, %rsp
;; subq $0x20, %rsp
;; movq 0x18(%rsp), %r14
;; movq 8(%r14), %rcx
@@ -58,16 +57,16 @@
;; movq %rax, %r11
;; addq $0x18, %r11
;; cmpq %rdx, %r11
-;; ja 0xe1
-;; b9: movq %rcx, %rdx
+;; ja 0xda
+;; b2: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x10(%rdx), %eax
-;; jmp 0xd6
-;; c7: movl $0, %eax
-;; jmp 0xd6
-;; d1: movl $0xffffffff, %eax
+;; jmp 0xcf
+;; c0: movl $0, %eax
+;; jmp 0xcf
+;; ca: movl $0xffffffff, %eax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; df: ud2
-;; e1: ud2
+;; d8: ud2
+;; da: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-drc.wat b/tests/disas/winch/x64/exceptions/throw-drc.wat
index 0aa8836563..94a40eb8f9 100644
--- a/tests/disas/winch/x64/exceptions/throw-drc.wat
+++ b/tests/disas/winch/x64/exceptions/throw-drc.wat
@@ -17,15 +17,15 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x12a
+;; ja 0x11c
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x231
+;; callq 0x223
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xdc
;; movq 8(%rsp), %r14
;; movq 0x28(%r14), %rcx
;; movl 8(%rcx), %ecx
@@ -39,11 +39,10 @@
;; movl 8(%rsp), %edx
;; movl $0x28, %ecx
;; movl $8, %r8d
-;; callq 0x1e2
+;; callq 0x1d4
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xdc
+;; addq $0xc, %rsp
;; movq 0xc(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -60,11 +59,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x25e
+;; callq 0x250
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xdc
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -75,12 +73,12 @@
;; movq %rax, %r11
;; addq $0x28, %r11
;; cmpq %rdx, %r11
-;; ja 0x12c
-;; 118: movq %rcx, %rdx
+;; ja 0x11e
+;; 10a: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x20(%rdx), %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 12a: ud2
-;; 12c: ud2
+;; 11c: ud2
+;; 11e: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-externref-drc.wat b/tests/disas/winch/x64/exceptions/throw-externref-drc.wat
index 05becddcfc..8cfc9c178f 100644
--- a/tests/disas/winch/x64/exceptions/throw-externref-drc.wat
+++ b/tests/disas/winch/x64/exceptions/throw-externref-drc.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x16b
+;; ja 0x15d
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -25,7 +25,7 @@
;; movl %r11d, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
-;; callq 0x21b
+;; callq 0x20d
;; addq $0xc, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12, 28]
;; movq 0x1c(%rsp), %r14
@@ -41,10 +41,9 @@
;; movl 4(%rsp), %edx
;; movl $0x28, %ecx
;; movl $8, %r8d
-;; callq 0x1cc
-;; addq $4, %rsp
+;; callq 0x1be
+;; addq $8, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12, 28]
-;; addq $4, %rsp
;; movq 0x20(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -58,19 +57,19 @@
;; movl (%rsp), %ecx
;; addq $4, %rsp
;; testl %ecx, %ecx
-;; je 0x12f
-;; e8: movl %ecx, %r11d
+;; je 0x128
+;; e1: movl %ecx, %r11d
;; andl $1, %r11d
;; testl %r11d, %r11d
-;; jne 0x12f
-;; fb: movq 8(%r14), %rbx
+;; jne 0x128
+;; f4: movq 8(%r14), %rbx
;; movq 0x28(%rbx), %rsi
;; movq 0x20(%rbx), %rbx
;; movq %rcx, %r11
;; addq $0x10, %r11
;; cmpq %rsi, %r11
-;; ja 0x16d
-;; 11a: movq %rbx, %rdi
+;; ja 0x15f
+;; 113: movq %rbx, %rdi
;; addq %rcx, %rdi
;; movq 8(%rdi), %r8
;; addq $1, %r8
@@ -81,13 +80,12 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x248
-;; addq $0xc, %rsp
+;; callq 0x23a
+;; addq $0x10, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]
-;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 16b: ud2
-;; 16d: ud2
+;; 15d: ud2
+;; 15f: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-externref-null.wat b/tests/disas/winch/x64/exceptions/throw-externref-null.wat
index e22be7197e..7755d2a0d8 100644
--- a/tests/disas/winch/x64/exceptions/throw-externref-null.wat
+++ b/tests/disas/winch/x64/exceptions/throw-externref-null.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0x189
+;; ja 0x17b
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -25,18 +25,18 @@
;; movl %r11d, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
-;; callq 0x233
+;; callq 0x225
;; addq $0xc, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12, 28]
;; movq 0x1c(%rsp), %r14
;; movq 0x20(%r14), %r11
;; movl (%r11), %ecx
;; addl $7, %ecx
-;; jb 0x18b
+;; jb 0x17d
;; 72: andl $0xfffffff8, %ecx
;; movl %ecx, %edx
;; addl $0x18, %edx
-;; jb 0x18d
+;; jb 0x17f
;; 86: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $4, %rsp
@@ -49,16 +49,15 @@
;; movq 0x28(%rcx), %rdx
;; movq 0x20(%rcx), %rcx
;; cmpq %rdx, %rax
-;; jbe 0xee
+;; jbe 0xe7
;; c3: subq %rdx, %rax
;; pushq %rax
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x1ec
-;; addq $0xc, %rsp
+;; callq 0x1de
+;; addq $0x14, %rsp
;; ╰─╼ stack_map: frame_size=64, frame_offsets=[28, 44]
-;; addq $8, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %eax
;; addq $4, %rsp
@@ -87,14 +86,13 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x260
-;; addq $0xc, %rsp
+;; callq 0x252
+;; addq $0x10, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]
-;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 189: ud2
-;; 18b: ud2
-;; 18d: ud2
+;; 17b: ud2
+;; 17d: ud2
+;; 17f: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-externref.wat b/tests/disas/winch/x64/exceptions/throw-externref.wat
index 2fc54687e4..8a3a25d65e 100644
--- a/tests/disas/winch/x64/exceptions/throw-externref.wat
+++ b/tests/disas/winch/x64/exceptions/throw-externref.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x11e
+;; ja 0x110
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -25,7 +25,7 @@
;; movl %r11d, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
-;; callq 0x1cc
+;; callq 0x1be
;; addq $0xc, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12, 28]
;; movq 0x1c(%rsp), %r14
@@ -41,10 +41,9 @@
;; movl 4(%rsp), %edx
;; movl $0x20, %ecx
;; movl $0x10, %r8d
-;; callq 0x17d
-;; addq $4, %rsp
+;; callq 0x16f
+;; addq $8, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12, 28]
-;; addq $4, %rsp
;; movq 0x20(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -63,12 +62,11 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x1f9
-;; addq $0xc, %rsp
+;; callq 0x1eb
+;; addq $0x10, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]
-;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 11e: ud2
+;; 110: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-funcref-drc.wat b/tests/disas/winch/x64/exceptions/throw-funcref-drc.wat
index ef9b4680eb..790f716b9f 100644
--- a/tests/disas/winch/x64/exceptions/throw-funcref-drc.wat
+++ b/tests/disas/winch/x64/exceptions/throw-funcref-drc.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0x13a
+;; ja 0x12c
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -24,7 +24,7 @@
;; pushq %r11
;; subq $8, %rsp
;; movq %r14, %rdi
-;; callq 0x216
+;; callq 0x208
;; addq $8, %rsp
;; movq 0x20(%rsp), %r14
;; movq 0x28(%r14), %rcx
@@ -38,7 +38,7 @@
;; movl (%rsp), %edx
;; movl $0x28, %ecx
;; movl $8, %r8d
-;; callq 0x19a
+;; callq 0x18c
;; addq $4, %rsp
;; movq 0x24(%rsp), %r14
;; movq 8(%r14), %rcx
@@ -58,9 +58,8 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x1e9
-;; addq $0xc, %rsp
-;; addq $8, %rsp
+;; callq 0x1db
+;; addq $0x14, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -71,11 +70,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x243
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x235
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 13a: ud2
+;; 12c: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-funcref-null.wat b/tests/disas/winch/x64/exceptions/throw-funcref-null.wat
index 5b46e0245b..aebe1d5349 100644
--- a/tests/disas/winch/x64/exceptions/throw-funcref-null.wat
+++ b/tests/disas/winch/x64/exceptions/throw-funcref-null.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0x1b4
+;; ja 0x19f
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -24,17 +24,17 @@
;; pushq %r11
;; subq $8, %rsp
;; movq %r14, %rdi
-;; callq 0x28c
+;; callq 0x277
;; addq $8, %rsp
;; movq 0x20(%rsp), %r14
;; movq 0x20(%r14), %r11
;; movl (%r11), %ecx
;; addl $7, %ecx
-;; jb 0x1b6
+;; jb 0x1a1
;; 6a: andl $0xfffffff8, %ecx
;; movl %ecx, %edx
;; addl $0x18, %edx
-;; jb 0x1b8
+;; jb 0x1a3
;; 7e: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $4, %rsp
@@ -47,15 +47,14 @@
;; movq 0x28(%rcx), %rdx
;; movq 0x20(%rcx), %rcx
;; cmpq %rdx, %rax
-;; jbe 0xe6
+;; jbe 0xdf
;; bb: subq %rdx, %rax
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movq 8(%rsp), %rsi
-;; callq 0x218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x203
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; movl (%rsp), %eax
;; addq $4, %rsp
@@ -84,9 +83,8 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x25f
-;; addq $0xc, %rsp
-;; addq $8, %rsp
+;; callq 0x24a
+;; addq $0x14, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -97,13 +95,12 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x2b9
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x2a4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 1b4: ud2
-;; 1b6: ud2
-;; 1b8: ud2
+;; 19f: ud2
+;; 1a1: ud2
+;; 1a3: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-funcref.wat b/tests/disas/winch/x64/exceptions/throw-funcref.wat
index 2a13beef05..c539a278e0 100644
--- a/tests/disas/winch/x64/exceptions/throw-funcref.wat
+++ b/tests/disas/winch/x64/exceptions/throw-funcref.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0x13a
+;; ja 0x12c
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -24,7 +24,7 @@
;; pushq %r11
;; subq $8, %rsp
;; movq %r14, %rdi
-;; callq 0x216
+;; callq 0x208
;; addq $8, %rsp
;; movq 0x20(%rsp), %r14
;; movq 0x28(%r14), %rcx
@@ -38,7 +38,7 @@
;; movl (%rsp), %edx
;; movl $0x20, %ecx
;; movl $0x10, %r8d
-;; callq 0x19a
+;; callq 0x18c
;; addq $4, %rsp
;; movq 0x24(%rsp), %r14
;; movq 8(%r14), %rcx
@@ -58,9 +58,8 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movq 0xc(%rsp), %rsi
-;; callq 0x1e9
-;; addq $0xc, %rsp
-;; addq $8, %rsp
+;; callq 0x1db
+;; addq $0x14, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %ecx
;; addq $4, %rsp
@@ -71,11 +70,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x243
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x235
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 13a: ud2
+;; 12c: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw-null.wat b/tests/disas/winch/x64/exceptions/throw-null.wat
index 7498d39859..a19fd56935 100644
--- a/tests/disas/winch/x64/exceptions/throw-null.wat
+++ b/tests/disas/winch/x64/exceptions/throw-null.wat
@@ -17,24 +17,24 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x186
+;; ja 0x17f
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x289
+;; callq 0x282
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x146
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x13f
;; movq 8(%rsp), %r14
;; movq 0x20(%r14), %r11
;; movl (%r11), %ecx
;; addl $7, %ecx
-;; jb 0x188
+;; jb 0x181
;; 4f: andl $0xfffffff8, %ecx
;; movl %ecx, %edx
;; addl $0x18, %edx
-;; jb 0x18a
+;; jb 0x183
;; 63: subq $4, %rsp
;; movl %eax, (%rsp)
;; subq $4, %rsp
@@ -52,9 +52,9 @@
;; pushq %rax
;; movq %r14, %rdi
;; movq (%rsp), %rsi
-;; callq 0x242
+;; callq 0x23b
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x13f
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movl (%rsp), %eax
@@ -82,11 +82,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x2b6
+;; callq 0x2af
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x13f
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -97,14 +96,14 @@
;; movq %rax, %r11
;; addq $0x18, %r11
;; cmpq %rdx, %r11
-;; ja 0x18c
-;; 174: movq %rcx, %rdx
+;; ja 0x185
+;; 16d: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x10(%rdx), %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 186: ud2
-;; 188: ud2
-;; 18a: ud2
-;; 18c: ud2
+;; 17f: ud2
+;; 181: ud2
+;; 183: ud2
+;; 185: ud2
diff --git a/tests/disas/winch/x64/exceptions/throw.wat b/tests/disas/winch/x64/exceptions/throw.wat
index 38a0582083..a19c93f969 100644
--- a/tests/disas/winch/x64/exceptions/throw.wat
+++ b/tests/disas/winch/x64/exceptions/throw.wat
@@ -17,15 +17,15 @@
;; movq 0x18(%r11), %r11
;; addq $0x20, %r11
;; cmpq %rsp, %r11
-;; ja 0x12a
+;; ja 0x11c
;; 1c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
-;; callq 0x231
+;; callq 0x223
;; ├─╼ exception frame offset: SP = FP - 0x10
-;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea
+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xdc
;; movq 8(%rsp), %r14
;; movq 0x28(%r14), %rcx
;; movl 8(%rcx), %ecx
@@ -39,11 +39,10 @@
;; movl 8(%rsp), %edx
;; movl $0x20, %ecx
;; movl $0x10, %r8d
-;; callq 0x1e2
+;; callq 0x1d4
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xdc
+;; addq $0xc, %rsp
;; movq 0xc(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq 0x28(%rcx), %rdx
@@ -60,11 +59,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x25e
+;; callq 0x250
;; ├─╼ exception frame offset: SP = FP - 0x20
-;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xdc
+;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; movq %rbp, %rsp
;; subq $0x10, %rsp
@@ -75,12 +73,12 @@
;; movq %rax, %r11
;; addq $0x20, %r11
;; cmpq %rdx, %r11
-;; ja 0x12c
-;; 118: movq %rcx, %rdx
+;; ja 0x11e
+;; 10a: movq %rcx, %rdx
;; addq %rax, %rdx
;; movl 0x18(%rdx), %eax
;; addq $0x10, %rsp
;; popq %rbp
;; retq
-;; 12a: ud2
-;; 12c: ud2
+;; 11c: ud2
+;; 11e: ud2
diff --git a/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat b/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat
index 64604cb13f..c29e172635 100644
--- a/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat
+++ b/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movss 0xc(%rsp), %xmm0
-;; callq 0xdb
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f32_floor/f32_floor_param.wat b/tests/disas/winch/x64/f32_floor/f32_floor_param.wat
index c860da414f..640d1d7e95 100644
--- a/tests/disas/winch/x64/f32_floor/f32_floor_param.wat
+++ b/tests/disas/winch/x64/f32_floor/f32_floor_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movss 0xc(%rsp), %xmm0
-;; callq 0xdb
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat b/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat
index 1099f240c0..c00ebabecd 100644
--- a/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat
+++ b/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movss 0xc(%rsp), %xmm0
-;; callq 0xdb
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat b/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat
index 04ea2d3771..1c2cb40eea 100644
--- a/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat
+++ b/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movss 0xc(%rsp), %xmm0
-;; callq 0xdb
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat b/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat
index d0823c70d6..f07d388bd5 100644
--- a/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat
+++ b/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $8, %rsp
;; movq %r14, %rdi
;; movsd 8(%rsp), %xmm0
-;; callq 0xdb
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f64_floor/f64_floor_param.wat b/tests/disas/winch/x64/f64_floor/f64_floor_param.wat
index 87ada7f89f..558412dbe1 100644
--- a/tests/disas/winch/x64/f64_floor/f64_floor_param.wat
+++ b/tests/disas/winch/x64/f64_floor/f64_floor_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $8, %rsp
;; movq %r14, %rdi
;; movsd 8(%rsp), %xmm0
-;; callq 0xdb
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat b/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat
index 64d7814165..069e12fef0 100644
--- a/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat
+++ b/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $8, %rsp
;; movq %r14, %rdi
;; movsd 8(%rsp), %xmm0
-;; callq 0xdb
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat b/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat
index 507039062c..fdf25f82c9 100644
--- a/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat
+++ b/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x7b
+;; ja 0x74
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -26,11 +26,10 @@
;; subq $8, %rsp
;; movq %r14, %rdi
;; movsd 8(%rsp), %xmm0
-;; callq 0xdb
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0xd4
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 7b: ud2
+;; 74: ud2
diff --git a/tests/disas/winch/x64/global-set-externref-drc.wat b/tests/disas/winch/x64/global-set-externref-drc.wat
index d272b62c6c..d4ede43e2e 100644
--- a/tests/disas/winch/x64/global-set-externref-drc.wat
+++ b/tests/disas/winch/x64/global-set-externref-drc.wat
@@ -13,7 +13,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x130
+;; ja 0x129
;; 1c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -37,7 +37,7 @@
;; 79: movq %rax, %r11
;; addq $0x10, %r11
;; cmpq %rdx, %r11
-;; ja 0x132
+;; ja 0x12b
;; 8c: movq %rcx, %rsi
;; addq %rax, %rsi
;; movq 8(%rsi), %rdi
@@ -45,15 +45,15 @@
;; movq %rdi, 8(%rsi)
;; movl %eax, 0x30(%r14)
;; testl %ebx, %ebx
-;; je 0x127
+;; je 0x120
;; ad: movl %ebx, %r11d
;; andl $1, %r11d
;; testl %r11d, %r11d
-;; jne 0x127
+;; jne 0x120
;; c0: movq %rbx, %r11
;; addq $0x10, %r11
;; cmpq %rdx, %r11
-;; ja 0x134
+;; ja 0x12d
;; d3: movq %rcx, %rsi
;; addq %rbx, %rsi
;; movq 8(%rsi), %rdi
@@ -61,20 +61,19 @@
;; cmpq $0, %rdi
;; je 0xf7
;; ee: movq %rdi, 8(%rsi)
-;; jmp 0x127
+;; jmp 0x120
;; f7: subq $4, %rsp
;; movl %ebx, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
-;; callq 0x193
-;; addq $0xc, %rsp
+;; callq 0x18c
+;; addq $0x10, %rsp
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28]
-;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 130: ud2
-;; 132: ud2
-;; 134: ud2
+;; 129: ud2
+;; 12b: ud2
+;; 12d: ud2
diff --git a/tests/disas/winch/x64/load/grow_load.wat b/tests/disas/winch/x64/load/grow_load.wat
index 500e3caa24..f4af848ed1 100644
--- a/tests/disas/winch/x64/load/grow_load.wat
+++ b/tests/disas/winch/x64/load/grow_load.wat
@@ -34,7 +34,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x70, %r11
;; cmpq %rsp, %r11
-;; ja 0x137
+;; ja 0x130
;; 1c: movq %rsi, %r14
;; subq $0x60, %rsp
;; movq %rsi, 0x58(%rsp)
@@ -65,9 +65,8 @@
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
;; movl $0, %edx
-;; callq 0x30a
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x302
+;; addq $0x10, %rsp
;; movq 0x58(%rsp), %r14
;; movl %eax, %eax
;; movq 0x38(%r14), %rcx
@@ -75,11 +74,11 @@
;; addq %rax, %rcx
;; addq $0x23024, %rcx
;; movsbq (%rcx), %rax
-;; movss 0x62(%rip), %xmm0
+;; movss 0x61(%rip), %xmm0
;; subq $0xc, %rsp
-;; movsd 0x5a(%rip), %xmm15
+;; movsd 0x59(%rip), %xmm15
;; movsd %xmm15, (%rsp)
-;; movss 0x43(%rip), %xmm15
+;; movss 0x42(%rip), %xmm15
;; movss %xmm15, 8(%rsp)
;; movq 0x14(%rsp), %rax
;; movsd (%rsp), %xmm15
@@ -91,15 +90,15 @@
;; addq $0x60, %rsp
;; popq %rbp
;; retq
-;; 137: ud2
-;; 139: addb %al, (%rax)
-;; 13b: addb %al, (%rax)
-;; 13d: addb %al, (%rax)
-;; 13f: addb %al, (%rax)
-;; 141: addb %al, (%rax)
-;; 143: addb %al, (%rax)
-;; 145: addb %al, (%rax)
-;; 147: addb %al, (%rax)
-;; 149: addb %al, (%rax)
-;; 14b: addb %al, (%rax)
-;; 14d: addb %al, (%rax)
+;; 130: ud2
+;; 132: addb %al, (%rax)
+;; 134: addb %al, (%rax)
+;; 136: addb %al, (%rax)
+;; 138: addb %al, (%rax)
+;; 13a: addb %al, (%rax)
+;; 13c: addb %al, (%rax)
+;; 13e: addb %al, (%rax)
+;; 140: addb %al, (%rax)
+;; 142: addb %al, (%rax)
+;; 144: addb %al, (%rax)
+;; 146: addb %al, (%rax)
diff --git a/tests/disas/winch/x64/table/fill.wat b/tests/disas/winch/x64/table/fill.wat
index b971e7a536..b50b63c5ed 100644
--- a/tests/disas/winch/x64/table/fill.wat
+++ b/tests/disas/winch/x64/table/fill.wat
@@ -78,7 +78,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0x22a
+;; ja 0x223
;; dc: movq %rdi, %r14
;; subq $0x30, %rsp
;; movq %rdi, 0x28(%rsp)
@@ -96,7 +96,7 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x22c
+;; jae 0x225
;; 138: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
@@ -106,19 +106,18 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x199
+;; jne 0x192
;; 15f: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x575
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x56e
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x19f
-;; 199: andq $0xfffffffffffffffe, %rax
+;; jmp 0x198
+;; 192: andq $0xfffffffffffffffe, %rax
;; movq %rax, 0xc(%rsp)
;; movl 0x14(%rsp), %eax
;; movq 0xc(%rsp), %rcx
@@ -127,18 +126,18 @@
;; movq 0x48(%r11), %rbx
;; movl %edx, %esi
;; addl %eax, %esi
-;; jb 0x22e
-;; 1c2: cmpl %ebx, %esi
-;; ja 0x230
-;; 1ca: cmpq $0, %rax
-;; je 0x221
-;; 1d4: movq %rdx, %rbx
+;; jb 0x227
+;; 1bb: cmpl %ebx, %esi
+;; ja 0x229
+;; 1c3: cmpq $0, %rax
+;; je 0x21a
+;; 1cd: movq %rdx, %rbx
;; movq %rcx, %rsi
;; movq %r14, %rdi
;; movq 0x48(%rdi), %r8
;; cmpq %r8, %rbx
-;; jae 0x232
-;; 1ea: movq %rbx, %r11
+;; jae 0x22b
+;; 1e3: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0x40(%rdi), %rdi
;; movq %rdi, %r9
@@ -149,12 +148,12 @@
;; movq %rsi, (%rdi)
;; addq $1, %rdx
;; subq $1, %rax
-;; jmp 0x1ca
-;; 221: addq $0x30, %rsp
+;; jmp 0x1c3
+;; 21a: addq $0x30, %rsp
;; popq %rbp
;; retq
-;; 22a: ud2
-;; 22c: ud2
-;; 22e: ud2
-;; 230: ud2
-;; 232: ud2
+;; 223: ud2
+;; 225: ud2
+;; 227: ud2
+;; 229: ud2
+;; 22b: ud2
diff --git a/tests/disas/winch/x64/table/get.wat b/tests/disas/winch/x64/table/get.wat
index 20c466bc5e..203726ad30 100644
--- a/tests/disas/winch/x64/table/get.wat
+++ b/tests/disas/winch/x64/table/get.wat
@@ -34,7 +34,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x10e
+;; ja 0x107
;; 5c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -48,7 +48,7 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x110
+;; jae 0x109
;; 9e: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
@@ -58,21 +58,20 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0xff
+;; jne 0xf8
;; c5: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x311
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x30a
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
-;; jmp 0x105
-;; ff: andq $0xfffffffffffffffe, %rax
+;; jmp 0xfe
+;; f8: andq $0xfffffffffffffffe, %rax
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 10e: ud2
-;; 110: ud2
+;; 107: ud2
+;; 109: ud2
diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat
index f50465b2e8..3d7ee2e76b 100644
--- a/tests/disas/winch/x64/table/init_copy_drop.wat
+++ b/tests/disas/winch/x64/table/init_copy_drop.wat
@@ -131,20 +131,20 @@
;; movq 0x18(%r11), %r11
;; addq $0x40, %r11
;; cmpq %rsp, %r11
-;; ja 0xa31
+;; ja 0xa0e
;; 15c: movq %rdi, %r14
;; subq $0x10, %rsp
;; movq %rdi, 8(%rsp)
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x1153
+;; callq 0x111c
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x117e
+;; callq 0x1147
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -156,14 +156,14 @@
;; movl %ebx, %ebx
;; movl %esi, %r8d
;; addl %ebx, %r8d
-;; jb 0xa33
+;; jb 0xa10
;; 1ca: cmpl %edx, %r8d
-;; ja 0xa35
+;; ja 0xa12
;; 1d3: movl %edi, %r8d
;; addl %ebx, %r8d
-;; jb 0xa37
+;; jb 0xa14
;; 1df: cmpl %ecx, %r8d
-;; ja 0xa39
+;; ja 0xa16
;; 1e8: movl %esi, %esi
;; imulq $0x10, %rsi, %rsi
;; addq %rsi, %rax
@@ -175,7 +175,7 @@
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %r8
;; cmpq %r8, %rdx
-;; jae 0xa3b
+;; jae 0xa18
;; 21c: movq %rdx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
@@ -190,17 +190,17 @@
;; jmp 0x1f4
;; 256: movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x11a9
+;; callq 0x1172
;; movq 8(%rsp), %r14
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x1153
+;; callq 0x111c
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x117e
+;; callq 0x1147
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -212,14 +212,14 @@
;; movl %ebx, %ebx
;; movl %esi, %r8d
;; addl %ebx, %r8d
-;; jb 0xa3d
+;; jb 0xa1a
;; 2c3: cmpl %edx, %r8d
-;; ja 0xa3f
+;; ja 0xa1c
;; 2cc: movl %edi, %r8d
;; addl %ebx, %r8d
-;; jb 0xa41
+;; jb 0xa1e
;; 2d8: cmpl %ecx, %r8d
-;; ja 0xa43
+;; ja 0xa20
;; 2e1: movl %esi, %esi
;; imulq $0x10, %rsi, %rsi
;; addq %rsi, %rax
@@ -231,7 +231,7 @@
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %r8
;; cmpq %r8, %rdx
-;; jae 0xa45
+;; jae 0xa22
;; 315: movq %rdx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
@@ -246,7 +246,7 @@
;; jmp 0x2ed
;; 34f: movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x11a9
+;; callq 0x1172
;; movq 8(%rsp), %r14
;; movl $5, %eax
;; movl $0xf, %ecx
@@ -258,16 +258,16 @@
;; movq 0xd8(%r11), %rbx
;; movq %rcx, %rsi
;; addq %rax, %rsi
-;; jb 0xa47
+;; jb 0xa24
;; 38c: cmpq %rbx, %rsi
-;; ja 0xa49
+;; ja 0xa26
;; 395: movq %r14, %r11
;; movq 0xd8(%r11), %rbx
;; movq %rdx, %rsi
;; addq %rax, %rsi
-;; jb 0xa4b
+;; jb 0xa28
;; 3ab: cmpq %rbx, %rsi
-;; ja 0xa4d
+;; ja 0xa2a
;; 3b4: cmpq %rcx, %rdx
;; jbe 0x3dd
;; 3bd: movq $18446744073709551615, %rbx
@@ -278,7 +278,7 @@
;; jmp 0x3e2
;; 3dd: movl $1, %ebx
;; cmpq $0, %rax
-;; je 0x4bc
+;; je 0x4b5
;; 3ec: movq %rcx, %rsi
;; pushq %rbx
;; pushq %rax
@@ -289,7 +289,7 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xa4f
+;; jae 0xa2c
;; 408: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
@@ -299,26 +299,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x464
+;; jne 0x45d
;; 432: pushq %rcx
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x46a
-;; 464: andq $0xfffffffffffffffe, %rax
+;; jmp 0x463
+;; 45d: andq $0xfffffffffffffffe, %rax
;; popq %rcx
;; popq %rdx
;; movq %rdx, %rbx
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %rdi
;; cmpq %rdi, %rbx
-;; jae 0xa51
-;; 482: movq %rbx, %r11
+;; jae 0xa2e
+;; 47b: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
;; movq %rsi, %r8
@@ -333,7 +332,7 @@
;; addq %rbx, %rcx
;; subq $1, %rax
;; jmp 0x3e2
-;; 4bc: movl $1, %eax
+;; 4b5: movl $1, %eax
;; movl $0x1d, %ecx
;; movl $0x15, %edx
;; movl %eax, %eax
@@ -343,28 +342,28 @@
;; movq 0xd8(%r11), %rbx
;; movq %rcx, %rsi
;; addq %rax, %rsi
-;; jb 0xa53
-;; 4e7: cmpq %rbx, %rsi
-;; ja 0xa55
-;; 4f0: movq %r14, %r11
+;; jb 0xa30
+;; 4e0: cmpq %rbx, %rsi
+;; ja 0xa32
+;; 4e9: movq %r14, %r11
;; movq 0xd8(%r11), %rbx
;; movq %rdx, %rsi
;; addq %rax, %rsi
-;; jb 0xa57
-;; 506: cmpq %rbx, %rsi
-;; ja 0xa59
-;; 50f: cmpq %rcx, %rdx
-;; jbe 0x538
-;; 518: movq $18446744073709551615, %rbx
+;; jb 0xa34
+;; 4ff: cmpq %rbx, %rsi
+;; ja 0xa36
+;; 508: cmpq %rcx, %rdx
+;; jbe 0x531
+;; 511: movq $18446744073709551615, %rbx
;; addq %rax, %rcx
;; subq $1, %rcx
;; addq %rax, %rdx
;; subq $1, %rdx
-;; jmp 0x53d
-;; 538: movl $1, %ebx
+;; jmp 0x536
+;; 531: movl $1, %ebx
;; cmpq $0, %rax
-;; je 0x617
-;; 547: movq %rcx, %rsi
+;; je 0x609
+;; 540: movq %rcx, %rsi
;; pushq %rbx
;; pushq %rax
;; pushq %rdx
@@ -374,8 +373,8 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xa5b
-;; 563: movq %rcx, %r11
+;; jae 0xa38
+;; 55c: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
;; movq %rdx, %rsi
@@ -384,26 +383,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x5bf
-;; 58d: pushq %rcx
+;; jne 0x5b1
+;; 586: pushq %rcx
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x5c5
-;; 5bf: andq $0xfffffffffffffffe, %rax
+;; jmp 0x5b7
+;; 5b1: andq $0xfffffffffffffffe, %rax
;; popq %rcx
;; popq %rdx
;; movq %rdx, %rbx
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %rdi
;; cmpq %rdi, %rbx
-;; jae 0xa5d
-;; 5dd: movq %rbx, %r11
+;; jae 0xa3a
+;; 5cf: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
;; movq %rsi, %r8
@@ -417,8 +415,8 @@
;; addq %rbx, %rdx
;; addq %rbx, %rcx
;; subq $1, %rax
-;; jmp 0x53d
-;; 617: movl $1, %eax
+;; jmp 0x536
+;; 609: movl $1, %eax
;; movl $0xa, %ecx
;; movl $0x18, %edx
;; movl %eax, %eax
@@ -428,28 +426,28 @@
;; movq 0xd8(%r11), %rbx
;; movq %rcx, %rsi
;; addq %rax, %rsi
-;; jb 0xa5f
-;; 642: cmpq %rbx, %rsi
-;; ja 0xa61
-;; 64b: movq %r14, %r11
+;; jb 0xa3c
+;; 634: cmpq %rbx, %rsi
+;; ja 0xa3e
+;; 63d: movq %r14, %r11
;; movq 0xd8(%r11), %rbx
;; movq %rdx, %rsi
;; addq %rax, %rsi
-;; jb 0xa63
-;; 661: cmpq %rbx, %rsi
-;; ja 0xa65
-;; 66a: cmpq %rcx, %rdx
-;; jbe 0x693
-;; 673: movq $18446744073709551615, %rbx
+;; jb 0xa40
+;; 653: cmpq %rbx, %rsi
+;; ja 0xa42
+;; 65c: cmpq %rcx, %rdx
+;; jbe 0x685
+;; 665: movq $18446744073709551615, %rbx
;; addq %rax, %rcx
;; subq $1, %rcx
;; addq %rax, %rdx
;; subq $1, %rdx
-;; jmp 0x698
-;; 693: movl $1, %ebx
+;; jmp 0x68a
+;; 685: movl $1, %ebx
;; cmpq $0, %rax
-;; je 0x772
-;; 6a2: movq %rcx, %rsi
+;; je 0x75d
+;; 694: movq %rcx, %rsi
;; pushq %rbx
;; pushq %rax
;; pushq %rdx
@@ -459,8 +457,8 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xa67
-;; 6be: movq %rcx, %r11
+;; jae 0xa44
+;; 6b0: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
;; movq %rdx, %rsi
@@ -469,26 +467,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x71a
-;; 6e8: pushq %rcx
+;; jne 0x705
+;; 6da: pushq %rcx
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x720
-;; 71a: andq $0xfffffffffffffffe, %rax
+;; jmp 0x70b
+;; 705: andq $0xfffffffffffffffe, %rax
;; popq %rcx
;; popq %rdx
;; movq %rdx, %rbx
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %rdi
;; cmpq %rdi, %rbx
-;; jae 0xa69
-;; 738: movq %rbx, %r11
+;; jae 0xa46
+;; 723: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
;; movq %rsi, %r8
@@ -502,8 +499,8 @@
;; addq %rbx, %rdx
;; addq %rbx, %rcx
;; subq $1, %rax
-;; jmp 0x698
-;; 772: movl $4, %eax
+;; jmp 0x68a
+;; 75d: movl $4, %eax
;; movl $0xb, %ecx
;; movl $0xd, %edx
;; movl %eax, %eax
@@ -513,28 +510,28 @@
;; movq 0xd8(%r11), %rbx
;; movq %rcx, %rsi
;; addq %rax, %rsi
-;; jb 0xa6b
-;; 79d: cmpq %rbx, %rsi
-;; ja 0xa6d
-;; 7a6: movq %r14, %r11
+;; jb 0xa48
+;; 788: cmpq %rbx, %rsi
+;; ja 0xa4a
+;; 791: movq %r14, %r11
;; movq 0xd8(%r11), %rbx
;; movq %rdx, %rsi
;; addq %rax, %rsi
-;; jb 0xa6f
-;; 7bc: cmpq %rbx, %rsi
-;; ja 0xa71
-;; 7c5: cmpq %rcx, %rdx
-;; jbe 0x7ee
-;; 7ce: movq $18446744073709551615, %rbx
+;; jb 0xa4c
+;; 7a7: cmpq %rbx, %rsi
+;; ja 0xa4e
+;; 7b0: cmpq %rcx, %rdx
+;; jbe 0x7d9
+;; 7b9: movq $18446744073709551615, %rbx
;; addq %rax, %rcx
;; subq $1, %rcx
;; addq %rax, %rdx
;; subq $1, %rdx
-;; jmp 0x7f3
-;; 7ee: movl $1, %ebx
+;; jmp 0x7de
+;; 7d9: movl $1, %ebx
;; cmpq $0, %rax
-;; je 0x8cd
-;; 7fd: movq %rcx, %rsi
+;; je 0x8b1
+;; 7e8: movq %rcx, %rsi
;; pushq %rbx
;; pushq %rax
;; pushq %rdx
@@ -544,8 +541,8 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xa73
-;; 819: movq %rcx, %r11
+;; jae 0xa50
+;; 804: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
;; movq %rdx, %rsi
@@ -554,26 +551,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x875
-;; 843: pushq %rcx
+;; jne 0x859
+;; 82e: pushq %rcx
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x87b
-;; 875: andq $0xfffffffffffffffe, %rax
+;; jmp 0x85f
+;; 859: andq $0xfffffffffffffffe, %rax
;; popq %rcx
;; popq %rdx
;; movq %rdx, %rbx
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %rdi
;; cmpq %rdi, %rbx
-;; jae 0xa75
-;; 893: movq %rbx, %r11
+;; jae 0xa52
+;; 877: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
;; movq %rsi, %r8
@@ -587,8 +583,8 @@
;; addq %rbx, %rdx
;; addq %rbx, %rcx
;; subq $1, %rax
-;; jmp 0x7f3
-;; 8cd: movl $5, %eax
+;; jmp 0x7de
+;; 8b1: movl $5, %eax
;; movl $0x14, %ecx
;; movl $0x13, %edx
;; movl %eax, %eax
@@ -598,28 +594,28 @@
;; movq 0xd8(%r11), %rbx
;; movq %rcx, %rsi
;; addq %rax, %rsi
-;; jb 0xa77
-;; 8f8: cmpq %rbx, %rsi
-;; ja 0xa79
-;; 901: movq %r14, %r11
+;; jb 0xa54
+;; 8dc: cmpq %rbx, %rsi
+;; ja 0xa56
+;; 8e5: movq %r14, %r11
;; movq 0xd8(%r11), %rbx
;; movq %rdx, %rsi
;; addq %rax, %rsi
-;; jb 0xa7b
-;; 917: cmpq %rbx, %rsi
-;; ja 0xa7d
-;; 920: cmpq %rcx, %rdx
-;; jbe 0x949
-;; 929: movq $18446744073709551615, %rbx
+;; jb 0xa58
+;; 8fb: cmpq %rbx, %rsi
+;; ja 0xa5a
+;; 904: cmpq %rcx, %rdx
+;; jbe 0x92d
+;; 90d: movq $18446744073709551615, %rbx
;; addq %rax, %rcx
;; subq $1, %rcx
;; addq %rax, %rdx
;; subq $1, %rdx
-;; jmp 0x94e
-;; 949: movl $1, %ebx
+;; jmp 0x932
+;; 92d: movl $1, %ebx
;; cmpq $0, %rax
-;; je 0xa28
-;; 958: movq %rcx, %rsi
+;; je 0xa05
+;; 93c: movq %rcx, %rsi
;; pushq %rbx
;; pushq %rax
;; pushq %rdx
@@ -629,8 +625,8 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xa7f
-;; 974: movq %rcx, %r11
+;; jae 0xa5c
+;; 958: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
;; movq %rdx, %rsi
@@ -639,26 +635,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x9d0
-;; 99e: pushq %rcx
+;; jne 0x9ad
+;; 982: pushq %rcx
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1218
-;; addq $8, %rsp
-;; addq $8, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
-;; jmp 0x9d6
-;; 9d0: andq $0xfffffffffffffffe, %rax
+;; jmp 0x9b3
+;; 9ad: andq $0xfffffffffffffffe, %rax
;; popq %rcx
;; popq %rdx
;; movq %rdx, %rbx
;; movq %r14, %rsi
;; movq 0xd8(%rsi), %rdi
;; cmpq %rdi, %rbx
-;; jae 0xa81
-;; 9ee: movq %rbx, %r11
+;; jae 0xa5e
+;; 9cb: movq %rbx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rsi), %rsi
;; movq %rsi, %r8
@@ -672,51 +667,51 @@
;; addq %rbx, %rdx
;; addq %rbx, %rcx
;; subq $1, %rax
-;; jmp 0x94e
-;; a28: addq $0x10, %rsp
+;; jmp 0x932
+;; a05: addq $0x10, %rsp
;; popq %rbp
;; retq
-;; a31: ud2
-;; a33: ud2
-;; a35: ud2
-;; a37: ud2
-;; a39: ud2
-;; a3b: ud2
-;; a3d: ud2
-;; a3f: ud2
-;; a41: ud2
-;; a43: ud2
-;; a45: ud2
-;; a47: ud2
-;; a49: ud2
-;; a4b: ud2
-;; a4d: ud2
-;; a4f: ud2
-;; a51: ud2
-;; a53: ud2
-;; a55: ud2
-;; a57: ud2
-;; a59: ud2
-;; a5b: ud2
-;; a5d: ud2
-;; a5f: ud2
-;; a61: ud2
-;; a63: ud2
-;; a65: ud2
-;; a67: ud2
-;; a69: ud2
-;; a6b: ud2
-;; a6d: ud2
-;; a6f: ud2
-;; a71: ud2
-;; a73: ud2
-;; a75: ud2
-;; a77: ud2
-;; a79: ud2
-;; a7b: ud2
-;; a7d: ud2
-;; a7f: ud2
-;; a81: ud2
+;; a0e: ud2
+;; a10: ud2
+;; a12: ud2
+;; a14: ud2
+;; a16: ud2
+;; a18: ud2
+;; a1a: ud2
+;; a1c: ud2
+;; a1e: ud2
+;; a20: ud2
+;; a22: ud2
+;; a24: ud2
+;; a26: ud2
+;; a28: ud2
+;; a2a: ud2
+;; a2c: ud2
+;; a2e: ud2
+;; a30: ud2
+;; a32: ud2
+;; a34: ud2
+;; a36: ud2
+;; a38: ud2
+;; a3a: ud2
+;; a3c: ud2
+;; a3e: ud2
+;; a40: ud2
+;; a42: ud2
+;; a44: ud2
+;; a46: ud2
+;; a48: ud2
+;; a4a: ud2
+;; a4c: ud2
+;; a4e: ud2
+;; a50: ud2
+;; a52: ud2
+;; a54: ud2
+;; a56: ud2
+;; a58: ud2
+;; a5a: ud2
+;; a5c: ud2
+;; a5e: ud2
;;
;; wasm[0]::function[11]:
;; pushq %rbp
@@ -725,8 +720,8 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0xb96
-;; aac: movq %rdi, %r14
+;; ja 0xb5f
+;; a7c: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
;; movq %rsi, 0x10(%rsp)
@@ -739,8 +734,8 @@
;; movq %r14, %rdx
;; movq 0xd8(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0xb98
-;; af1: movq %rcx, %r11
+;; jae 0xb61
+;; ac1: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0xd0(%rdx), %rdx
;; movq %rdx, %rsi
@@ -749,27 +744,26 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0xb55
-;; b1b: subq $4, %rsp
+;; jne 0xb1e
+;; aeb: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $0xc, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x1218
-;; addq $0xc, %rsp
-;; addq $4, %rsp
+;; callq 0x11e1
+;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
-;; jmp 0xb5b
-;; b55: andq $0xfffffffffffffffe, %rax
+;; jmp 0xb24
+;; b1e: andq $0xfffffffffffffffe, %rax
;; testq %rax, %rax
-;; je 0xb9a
-;; b64: movq 0x28(%r14), %r11
+;; je 0xb63
+;; b2d: movq 0x28(%r14), %r11
;; movl (%r11), %ecx
;; movl 0x10(%rax), %edx
;; cmpl %edx, %ecx
-;; jne 0xb9c
-;; b76: pushq %rax
+;; jne 0xb65
+;; b3f: pushq %rax
;; popq %rcx
;; movq 0x18(%rcx), %rbx
;; movq 8(%rcx), %rdx
@@ -780,7 +774,7 @@
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; b96: ud2
-;; b98: ud2
-;; b9a: ud2
-;; b9c: ud2
+;; b5f: ud2
+;; b61: ud2
+;; b63: ud2
+;; b65: ud2
diff --git a/tests/disas/winch/x64/table/set.wat b/tests/disas/winch/x64/table/set.wat
index 562ef0da46..3d1d6199ff 100644
--- a/tests/disas/winch/x64/table/set.wat
+++ b/tests/disas/winch/x64/table/set.wat
@@ -74,7 +74,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x1f0
+;; ja 0x1e9
;; ec: movq %rdi, %r14
;; subq $0x20, %rsp
;; movq %rdi, 0x18(%rsp)
@@ -92,7 +92,7 @@
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x1f2
+;; jae 0x1eb
;; 142: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
@@ -102,26 +102,25 @@
;; cmovaeq %rsi, %rdx
;; movq (%rdx), %rax
;; testq %rax, %rax
-;; jne 0x1a3
+;; jne 0x19c
;; 169: subq $4, %rsp
;; movl %ecx, (%rsp)
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x4f9
-;; addq $8, %rsp
-;; addq $4, %rsp
+;; callq 0x4f2
+;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
-;; jmp 0x1a9
-;; 1a3: andq $0xfffffffffffffffe, %rax
+;; jmp 0x1a2
+;; 19c: andq $0xfffffffffffffffe, %rax
;; movl (%rsp), %ecx
;; addq $4, %rsp
;; movq %r14, %rdx
;; movq 0x38(%rdx), %rbx
;; cmpq %rbx, %rcx
-;; jae 0x1f4
-;; 1c3: movq %rcx, %r11
+;; jae 0x1ed
+;; 1bc: movq %rcx, %r11
;; imulq $8, %r11, %r11
;; movq 0x30(%rdx), %rdx
;; movq %rdx, %rsi
@@ -133,6 +132,6 @@
;; addq $0x20, %rsp
;; popq %rbp
;; retq
-;; 1f0: ud2
-;; 1f2: ud2
-;; 1f4: ud2
+;; 1e9: ud2
+;; 1eb: ud2
+;; 1ed: ud2
diff --git a/tests/disas/winch/x64/v128_mixed_sig.wat b/tests/disas/winch/x64/v128_mixed_sig.wat
index e389d8b7aa..f8c0451c93 100644
--- a/tests/disas/winch/x64/v128_mixed_sig.wat
+++ b/tests/disas/winch/x64/v128_mixed_sig.wat
@@ -14,7 +14,7 @@
;; movq 0x18(%r11), %r11
;; addq $0x30, %r11
;; cmpq %rsp, %r11
-;; ja 0x6d
+;; ja 0x78
;; 1c: movq %rdi, %r14
;; subq $0x30, %rsp
;; movq %rdi, 0x28(%rsp)
@@ -28,7 +28,9 @@
;; movss %xmm6, 4(%rsp)
;; movss %xmm7, (%rsp)
;; movdqu 0x20(%rbp), %xmm0
-;; addq $0x30, %rsp
-;; popq %rbp
+;; movq 0x38(%rsp), %r11
+;; movq %r11, 0x58(%rsp)
+;; movq 0x30(%rsp), %rbp
+;; addq $0x58, %rsp
;; retq
-;; 6d: ud2
+;; 78: ud2
diff --git a/tests/misc_testsuite/component-model/tail-calls.wast b/tests/misc_testsuite/component-model/tail-calls.wast
new file mode 100644
index 0000000000..90b9afa994
--- /dev/null
+++ b/tests/misc_testsuite/component-model/tail-calls.wast
@@ -0,0 +1,82 @@
+;;! tail_call = true
+
+;; Tail-call through canon lower, then resize the argument area in the provider.
+(component
+ (component $provider
+ (core module $m
+ (func $final
+ (param i32 i32 i32 i32 i32 i32 i32 i32 i32 i32 i32 i32 i32)
+ (result i32)
+ local.get 0)
+ (func (export "f")
+ (param i32 i32 i32 i32 i32 i32 i32 i32 i32)
+ (result i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ local.get 5
+ local.get 6
+ local.get 7
+ local.get 8
+ i32.const 10
+ i32.const 11
+ i32.const 12
+ i32.const 13
+ return_call $final))
+ (core instance $i (instantiate $m))
+ (func (export "f")
+ (param "a" u32)
+ (param "b" u32)
+ (param "c" u32)
+ (param "d" u32)
+ (param "e" u32)
+ (param "f" u32)
+ (param "g" u32)
+ (param "h" u32)
+ (param "i" u32)
+ (result u32)
+ (canon lift (core func $i "f"))))
+
+ (instance $provider-instance (instantiate $provider))
+
+ (component $consumer
+ (import "f" (func $f
+ (param "a" u32)
+ (param "b" u32)
+ (param "c" u32)
+ (param "d" u32)
+ (param "e" u32)
+ (param "f" u32)
+ (param "g" u32)
+ (param "h" u32)
+ (param "i" u32)
+ (result u32)))
+ (core func $lowered-f (canon lower (func $f)))
+ (core module $m
+ (import "" "f" (func $f
+ (param i32 i32 i32 i32 i32 i32 i32 i32 i32)
+ (result i32)))
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ i32.const 5
+ i32.const 6
+ i32.const 7
+ i32.const 8
+ i32.const 9
+ return_call $f))
+ (core instance $i (instantiate $m
+ (with "" (instance
+ (export "f" (func $lowered-f))))))
+ (func (export "run") (param "x" u32) (result u32)
+ (canon lift (core func $i "run"))))
+
+ (instance $consumer-instance (instantiate $consumer
+ (with "f" (func $provider-instance "f"))))
+ (export "run" (func $consumer-instance "run")))
+
+(assert_return (invoke "run" (u32.const 42)) (u32.const 42))
diff --git a/tests/misc_testsuite/tail-call/argument-area-resize.wast b/tests/misc_testsuite/tail-call/argument-area-resize.wast
new file mode 100644
index 0000000000..516c86c859
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/argument-area-resize.wast
@@ -0,0 +1,171 @@
+;;! tail_call = true
+;;! reference_types = true
+
+;; Direct tail calls grow and shrink the incoming stack-argument area.
+(module
+ (func $small (param i32 i32 i32 i32 i32) (result i32)
+ local.get 0
+ i32.eqz
+ if (result i32)
+ local.get 1
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ i32.const 5
+ i32.const 6
+ i32.const 7
+ i32.const 8
+ return_call $large
+ end)
+
+ (func $large (param i32 i32 i32 i32 i32 i32 i32 i32 i32) (result i32)
+ local.get 0
+ i32.eqz
+ if (result i32)
+ local.get 1
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ return_call $small
+ end)
+
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ i32.const 42
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ call $small))
+
+(assert_return (invoke "run" (i32.const 100000)) (i32.const 42))
+
+;; The same resizing through indirect tail calls.
+(module
+ (type $small-ty (func (param i32 i32 i32 i32 i32) (result i32)))
+ (type $large-ty
+ (func (param i32 i32 i32 i32 i32 i32 i32 i32 i32) (result i32)))
+
+ (table funcref (elem $small $large))
+
+ (func $small (type $small-ty)
+ local.get 0
+ i32.eqz
+ if (result i32)
+ local.get 1
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ i32.const 5
+ i32.const 6
+ i32.const 7
+ i32.const 8
+ i32.const 1
+ return_call_indirect (type $large-ty)
+ end)
+
+ (func $large (type $large-ty)
+ local.get 0
+ i32.eqz
+ if (result i32)
+ local.get 1
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ i32.const 0
+ return_call_indirect (type $small-ty)
+ end)
+
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ i32.const 42
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ call $small))
+
+(assert_return (invoke "run" (i32.const 100000)) (i32.const 42))
+
+;; Forward mixed register and stack results through direct and indirect tails.
+(module
+ (type $small-ty
+ (func (param i32 i32 i64 i32 i64) (result i32 i64 i32 i64)))
+ (type $large-ty
+ (func
+ (param i32 i32 i64 i32 i64 i32 i32 i32 i32)
+ (result i32 i64 i32 i64)))
+
+ (table funcref (elem $small $large))
+
+ (func $small (type $small-ty)
+ local.get 0
+ i32.eqz
+ if (result i32 i64 i32 i64)
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ i32.const 5
+ i32.const 6
+ i32.const 7
+ i32.const 8
+ i32.const 1
+ return_call_indirect (type $large-ty)
+ end)
+
+ (func $large (type $large-ty)
+ local.get 0
+ i32.eqz
+ if (result i32 i64 i32 i64)
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ else
+ local.get 0
+ i32.const 1
+ i32.sub
+ local.get 1
+ local.get 2
+ local.get 3
+ local.get 4
+ return_call $small
+ end)
+
+ (func (export "run") (param i32) (result i32 i64 i32 i64)
+ local.get 0
+ i32.const 42
+ i64.const 43
+ i32.const 44
+ i64.const 45
+ return_call $small))
+
+(assert_return (invoke "run" (i32.const 100000))
+ (i32.const 42) (i64.const 43) (i32.const 44) (i64.const 45))
diff --git a/tests/misc_testsuite/tail-call/bounded-recursion.wast b/tests/misc_testsuite/tail-call/bounded-recursion.wast
new file mode 100644
index 0000000000..c440332d0f
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/bounded-recursion.wast
@@ -0,0 +1,29 @@
+;;! tail_call = true
+
+(module
+ (func $ordinary (export "ordinary") (param $count i64) (result i64)
+ local.get $count
+ i64.eqz
+ if (result i64)
+ i64.const 42
+ else
+ local.get $count
+ i64.const 1
+ i64.sub
+ call $ordinary
+ end)
+
+ (func $tail (export "tail") (param $count i64) (result i64)
+ local.get $count
+ i64.eqz
+ if (result i64)
+ i64.const 42
+ else
+ local.get $count
+ i64.const 1
+ i64.sub
+ return_call $tail
+ end))
+
+(assert_exhaustion (invoke "ordinary" (i64.const 1000000)) "call stack exhausted")
+(assert_return (invoke "tail" (i64.const 1000000)) (i64.const 42))
diff --git a/tests/misc_testsuite/tail-call/imports.wast b/tests/misc_testsuite/tail-call/imports.wast
new file mode 100644
index 0000000000..7b12086b10
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/imports.wast
@@ -0,0 +1,18 @@
+;;! tail_call = true
+
+(module
+ (func (export "callee") (param i32 i32 i32 i32 i32) (result i32)
+ local.get 0))
+(register "m")
+
+(module
+ (import "m" "callee" (func $callee (param i32 i32 i32 i32 i32) (result i32)))
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ i32.const 2
+ i32.const 3
+ i32.const 4
+ i32.const 5
+ return_call $callee))
+
+(assert_return (invoke "run" (i32.const 42)) (i32.const 42))
diff --git a/tests/misc_testsuite/tail-call/indirect-traps.wast b/tests/misc_testsuite/tail-call/indirect-traps.wast
new file mode 100644
index 0000000000..28ccb5a7f7
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/indirect-traps.wast
@@ -0,0 +1,20 @@
+;;! tail_call = true
+;;! reference_types = true
+
+(module
+ (type $expected (func (result i32)))
+ (type $wrong (func (result i64)))
+
+ (func $good (type $expected) i32.const 7)
+ (func $bad (type $wrong) i64.const 8)
+
+ (table 3 funcref)
+ (elem (i32.const 0) $good $bad)
+
+ (func (export "run") (param i32) (result i32)
+ local.get 0
+ return_call_indirect (type $expected)))
+
+(assert_return (invoke "run" (i32.const 0)) (i32.const 7))
+(assert_trap (invoke "run" (i32.const 1)) "indirect call type mismatch")
+(assert_trap (invoke "run" (i32.const 2)) "uninitialized element")
diff --git a/tests/misc_testsuite/tail-call/reachable-joins.wast b/tests/misc_testsuite/tail-call/reachable-joins.wast
new file mode 100644
index 0000000000..d2afbefc82
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/reachable-joins.wast
@@ -0,0 +1,150 @@
+;;! tail_call = true
+;;! reference_types = true
+
+(module
+ (func (export "leaf") (result i32) i32.const 42))
+(register "provider")
+
+;; Each taken branch exits, but compilation continues at the join. There are
+;; enough branches to exhaust the registers if temporary callee registers leak.
+(module
+ (type $t (func (result i32)))
+ (import "provider" "leaf" (func $leaf (type $t)))
+ (table funcref (elem $leaf))
+ (func (export "direct") (param i32) (result i32)
+ local.get 0 i32.const 0 i32.eq if return_call $leaf end
+ local.get 0 i32.const 1 i32.eq if return_call $leaf end
+ local.get 0 i32.const 2 i32.eq if return_call $leaf end
+ local.get 0 i32.const 3 i32.eq if return_call $leaf end
+ local.get 0 i32.const 4 i32.eq if return_call $leaf end
+ local.get 0 i32.const 5 i32.eq if return_call $leaf end
+ local.get 0 i32.const 6 i32.eq if return_call $leaf end
+ local.get 0 i32.const 7 i32.eq if return_call $leaf end
+ local.get 0 i32.const 8 i32.eq if return_call $leaf end
+ local.get 0 i32.const 9 i32.eq if return_call $leaf end
+ local.get 0 i32.const 10 i32.eq if return_call $leaf end
+ local.get 0 i32.const 11 i32.eq if return_call $leaf end
+ local.get 0 i32.const 12 i32.eq if return_call $leaf end
+ local.get 0 i32.const 13 i32.eq if return_call $leaf end
+ local.get 0 i32.const 14 i32.eq if return_call $leaf end
+ local.get 0 i32.const 15 i32.eq if return_call $leaf end
+ local.get 0 i32.const 16 i32.eq if return_call $leaf end
+ local.get 0 i32.const 17 i32.eq if return_call $leaf end
+ local.get 0 i32.const 18 i32.eq if return_call $leaf end
+ local.get 0 i32.const 19 i32.eq if return_call $leaf end
+ local.get 0 i32.const 20 i32.eq if return_call $leaf end
+ local.get 0 i32.const 21 i32.eq if return_call $leaf end
+ local.get 0 i32.const 22 i32.eq if return_call $leaf end
+ local.get 0 i32.const 23 i32.eq if return_call $leaf end
+ local.get 0 i32.const 24 i32.eq if return_call $leaf end
+ local.get 0 i32.const 25 i32.eq if return_call $leaf end
+ local.get 0 i32.const 26 i32.eq if return_call $leaf end
+ local.get 0 i32.const 27 i32.eq if return_call $leaf end
+ local.get 0 i32.const 28 i32.eq if return_call $leaf end
+ local.get 0 i32.const 29 i32.eq if return_call $leaf end
+ local.get 0 i32.const 30 i32.eq if return_call $leaf end
+ local.get 0 i32.const 31 i32.eq if return_call $leaf end
+ i32.const -1)
+ (func (export "indirect") (param i32) (result i32)
+ local.get 0 i32.const 0 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 1 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 2 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 3 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 4 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 5 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 6 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 7 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 8 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 9 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 10 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 11 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 12 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 13 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 14 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 15 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 16 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 17 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 18 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 19 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 20 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 21 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 22 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 23 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 24 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 25 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 26 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 27 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 28 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 29 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 30 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ local.get 0 i32.const 31 i32.eq if i32.const 0 return_call_indirect (type $t) end
+ i32.const -1)
+)
+
+(assert_return (invoke "direct" (i32.const 0)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 1)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 2)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 3)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 4)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 5)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 6)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 7)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 8)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 9)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 10)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 11)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 12)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 13)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 14)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 15)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 16)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 17)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 18)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 19)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 20)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 21)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 22)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 23)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 24)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 25)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 26)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 27)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 28)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 29)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 30)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 31)) (i32.const 42))
+(assert_return (invoke "direct" (i32.const 32)) (i32.const -1))
+
+(assert_return (invoke "indirect" (i32.const 0)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 1)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 2)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 3)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 4)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 5)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 6)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 7)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 8)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 9)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 10)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 11)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 12)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 13)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 14)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 15)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 16)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 17)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 18)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 19)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 20)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 21)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 22)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 23)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 24)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 25)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 26)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 27)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 28)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 29)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 30)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 31)) (i32.const 42))
+(assert_return (invoke "indirect" (i32.const 32)) (i32.const -1))
diff --git a/tests/misc_testsuite/tail-call/returns.wast b/tests/misc_testsuite/tail-call/returns.wast
new file mode 100644
index 0000000000..1b9ec12127
--- /dev/null
+++ b/tests/misc_testsuite/tail-call/returns.wast
@@ -0,0 +1,50 @@
+;; Ordinary returns with no stack arguments and with adjacent local-frame sizes.
+;; On x86-64 the last two functions straddle Winch's compact epilogue boundary.
+
+(module
+ (func (export "no_stack_args") (result i64)
+ i64.const 42)
+ (func (export "compact") (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)
+ (local i64 i64 i64 i64)
+ local.get 0 local.get 7 i64.add)
+ (func (export "fallback") (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)
+ (local i64 i64 i64 i64 i64)
+ local.get 0 local.get 7 i64.add))
+
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "compact" (i64.const -17) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const -9))
+ (i64.const -26))
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "compact" (i64.const 0) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const 8))
+ (i64.const 8))
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "compact" (i64.const 42) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const 50))
+ (i64.const 92))
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "fallback" (i64.const -17) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const -9))
+ (i64.const -26))
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "fallback" (i64.const 0) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const 8))
+ (i64.const 8))
+(assert_return (invoke "no_stack_args") (i64.const 42))
+
+(assert_return
+ (invoke "fallback" (i64.const 42) (i64.const 2) (i64.const 3) (i64.const 4)
+ (i64.const 5) (i64.const 6) (i64.const 7) (i64.const 50))
+ (i64.const 92))
+(assert_return (invoke "no_stack_args") (i64.const 42))
diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs
index 5a0d148446..0e99b7ff56 100644
--- a/winch/codegen/src/codegen/call.rs
+++ b/winch/codegen/src/codegen/call.rs
@@ -60,7 +60,7 @@
use crate::{
FuncEnv, Result,
- abi::{ABIOperand, ABISig, RetArea, vmctx},
+ abi::{self, ABI, ABIOperand, ABISig, RetArea, vmctx},
codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext, CodeGenError, Emission},
ensure,
masm::{
@@ -72,6 +72,46 @@ use crate::{
};
use wasmtime_environ::{DefinedFuncIndex, FuncIndex, PtrSize, VMOffsets};
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub(crate) enum TailCallFrameKind {
+ SameSize,
+ EmptyCallee,
+ Resize,
+}
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct TailCallPlan {
+ pub(crate) callee_args_size: u32,
+ pub(crate) args_delta: i64,
+ pub(crate) callee_args_from_fp: i64,
+ pub(crate) frame_kind: TailCallFrameKind,
+}
+
+impl TailCallPlan {
+ pub(crate) fn new<A: abi::ABI>(caller_args_size: u32, callee_args_size: u32) -> Self {
+ let alignment = u32::from(A::call_stack_align());
+ let caller_args_size = abi::align_to(caller_args_size, alignment);
+ let callee_args_size = abi::align_to(callee_args_size, alignment);
+ let args_delta = i64::from(caller_args_size) - i64::from(callee_args_size);
+ let callee_args_from_fp = i64::from(A::arg_base_offset()) + args_delta;
+
+ let frame_kind = if caller_args_size == callee_args_size {
+ TailCallFrameKind::SameSize
+ } else if callee_args_size == 0 {
+ TailCallFrameKind::EmptyCallee
+ } else {
+ TailCallFrameKind::Resize
+ };
+
+ Self {
+ callee_args_size,
+ args_delta,
+ callee_args_from_fp,
+ frame_kind,
+ }
+ }
+}
+
/// All the information needed to emit a function call.
#[derive(Copy, Clone)]
pub(crate) struct FnCall {}
@@ -133,6 +173,85 @@ impl FnCall {
)
}
+ /// Emit a tail call which replaces the current frame.
+ ///
+ /// Stack arguments are first staged below the current frame, then the macro
+ /// assembler slides the caller's return address and argument block into the
+ /// callee's incoming layout before jumping. The final callee pops its own
+ /// incoming argument area when returning from the tail-call chain.
+ pub fn emit_return<M: MacroAssembler>(
+ env: &mut FuncEnv<M::Ptr>,
+ masm: &mut M,
+ context: &mut CodeGenContext<Emission>,
+ caller_stack_args_size: u32,
+ callee: Callee,
+ ) -> Result<()> {
+ let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm)?;
+ let sig = env.callee_sig::<M::ABI>(&callee)?;
+ let ret_area = if sig.has_stack_results() {
+ let slot = context
+ .frame
+ .results_base_slot
+ .ok_or_else(CodeGenError::results_area_expected)?;
+ Some(RetArea::slot(slot))
+ } else {
+ None
+ };
+
+ context.spill(masm)?;
+
+ let plan = TailCallPlan::new::<M::ABI>(caller_stack_args_size, sig.params_stack_size());
+
+ // Stage the callee arguments first.
+ masm.reserve_stack(plan.callee_args_size)?;
+ Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;
+
+ match plan.frame_kind {
+ TailCallFrameKind::SameSize => {
+ Self::move_tail_call_args(masm, plan, 0)?;
+ masm.finish_tail_call_same_size()?;
+ }
+ TailCallFrameKind::EmptyCallee => {
+ masm.finish_tail_call_empty(plan)?;
+ }
+ TailCallFrameKind::Resize => {
+ masm.with_tail_call_resize(plan, |masm, staged_args_offset| {
+ Self::move_tail_call_args(masm, plan, staged_args_offset)
+ })?;
+ }
+ }
+
+ masm.reset_stack_pointer(SPOffset::from_u32(0))?;
+ masm.tail_jump(kind);
+ // There is no returning-call cleanup, but compilation can continue at
+ // another reachable branch. Release the lowering temporaries there too.
+ Self::free_callee_registers(&callee_context, &kind, context);
+ Ok(())
+ }
+
+ /// Move staged stack arguments into the tail callee's incoming argument
+ /// area. The destination is above the source, so copy from high addresses
+ /// to low addresses to permit overlap.
+ fn move_tail_call_args<M: MacroAssembler>(
+ masm: &mut M,
+ plan: TailCallPlan,
+ staged_args_offset: u32,
+ ) -> Result<()> {
+ let word_bytes = u32::from(<M::ABI as abi::ABI>::word_bytes());
+
+ masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ let mut offset = plan.callee_args_size;
+ while offset > 0 {
+ offset -= word_bytes;
+ let src = masm.address_at_sp(SPOffset::from_u32(staged_args_offset + offset))?;
+ let dst = masm.address_at_fp(plan.callee_args_from_fp + i64::from(offset))?;
+ masm.load_ptr(src, scratch.writable())?;
+ masm.store_ptr(scratch.inner(), dst)?;
+ }
+ wasmtime_environ::error::Ok(())
+ })
+ }
+
/// Calculates the return area for the callee, if any.
fn make_ret_area<M: MacroAssembler>(
callee_sig: &ABISig,
@@ -364,38 +483,51 @@ impl FnCall {
if sig.has_stack_results() {
let operand = sig.params.unwrap_results_area_operand();
- let base = ret_area.unwrap().unwrap_sp();
- let addr = masm.address_from_sp(base)?;
-
- match operand {
- &ABIOperand::Reg { ty, reg, .. } => {
- masm.compute_addr(addr, writable!(reg), ty.try_into()?)?;
+ match ret_area.unwrap() {
+ RetArea::SP(base) => {
+ let addr = masm.address_from_sp(*base)?;
+ match operand {
+ &ABIOperand::Reg { ty, reg, .. } => {
+ masm.compute_addr(addr, writable!(reg), ty.try_into()?)?;
+ }
+ &ABIOperand::Stack { ty, offset, .. } => {
+ let slot = masm.address_at_sp(SPOffset::from_u32(offset))?;
+ // Don't rely on `ABI::scratch_for` as we always use
+ // an int register as the return pointer.
+ masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.compute_addr(addr, scratch.writable(), ty.try_into()?)?;
+ masm.store(scratch.inner().into(), slot, ty.try_into()?)
+ })?;
+ }
+ }
}
- &ABIOperand::Stack { ty, offset, .. } => {
- let slot = masm.address_at_sp(SPOffset::from_u32(offset))?;
- // Don't rely on `ABI::scratch_for` as we always use
- // an int register as the return pointer.
- masm.with_scratch::<IntScratch, _>(|masm, scratch| {
- masm.compute_addr(addr, scratch.writable(), ty.try_into()?)?;
- masm.store(scratch.inner().into(), slot, ty.try_into()?)
- })?;
+ RetArea::Slot(source) => {
+ let source = masm.local_address(source)?;
+ match operand {
+ ABIOperand::Reg { reg, .. } => {
+ masm.load_ptr(source, writable!(*reg))?;
+ }
+ ABIOperand::Stack { offset, .. } => {
+ let destination = masm.address_at_sp(SPOffset::from_u32(*offset))?;
+ masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load_ptr(source, scratch.writable())?;
+ masm.store_ptr(scratch.inner(), destination)
+ })?;
+ }
+ }
}
+ RetArea::Uninit => crate::bail!(CodeGenError::results_area_expected()),
}
}
Ok(())
}
- /// Cleanup stack space, handle multiple results, and free registers after
- /// emitting the call.
- fn cleanup<M: MacroAssembler>(
- sig: &ABISig,
+ /// Release the temporary registers used to lower an ordinary or tail call.
+ fn free_callee_registers(
callee_context: &ContextArgs,
callee_kind: &CalleeKind,
- reserved_space: u32,
- ret_area: Option<RetArea>,
- masm: &mut M,
context: &mut CodeGenContext<Emission>,
- ) -> Result<()> {
+ ) {
// Free any registers holding any function references.
match callee_kind {
CalleeKind::Indirect(r) => context.free_reg(*r),
@@ -409,9 +541,39 @@ impl FnCall {
_ => {}
}
}
- // Deallocate the reserved space for stack arguments and for alignment,
- // which was allocated last.
- masm.free_stack(reserved_space)?;
+ }
+
+ /// Cleanup stack space, handle multiple results, and free registers after
+ /// emitting the call.
+ fn cleanup<M: MacroAssembler>(
+ sig: &ABISig,
+ callee_context: &ContextArgs,
+ callee_kind: &CalleeKind,
+ reserved_space: u32,
+ ret_area: Option<RetArea>,
+ masm: &mut M,
+ context: &mut CodeGenContext<Emission>,
+ ) -> Result<()> {
+ Self::free_callee_registers(callee_context, callee_kind, context);
+
+ // Default-ABI callees pop their aligned stack-argument area. Update the
+ // abstract stack depth for that pop and deallocate only the remaining
+ // call-alignment space. Other calling conventions remain caller-pop.
+ let callee_pop_size = if sig.call_conv.is_default() {
+ abi::align_to(
+ sig.params_stack_size(),
+ u32::from(M::ABI::call_stack_align()),
+ )
+ } else {
+ 0
+ };
+ // Without stack results, reclaim alignment space and consumed argument
+ // spills together. Stack results need the intermediate SP for the
+ // result-area move below, so retain their existing cleanup order.
+ let combine_cleanup = ret_area.is_none();
+ if !combine_cleanup {
+ masm.restore_stack_after_call(reserved_space, callee_pop_size)?;
+ }
ensure!(
sig.params.len_without_retptr() >= callee_context.len(),
@@ -453,7 +615,16 @@ impl FnCall {
};
// Free the bytes consumed by the call.
- masm.free_stack(stack_consumed)?;
+ if combine_cleanup {
+ masm.restore_stack_after_call(
+ reserved_space
+ .checked_add(stack_consumed)
+ .ok_or_else(CodeGenError::invalid_sp_offset)?,
+ callee_pop_size,
+ )?;
+ } else {
+ masm.free_stack(stack_consumed)?;
+ }
let mut calculated_ret_area = None;
diff --git a/winch/codegen/src/codegen/error.rs b/winch/codegen/src/codegen/error.rs
index 2b91cf7e07..8c887d49f9 100644
--- a/winch/codegen/src/codegen/error.rs
+++ b/winch/codegen/src/codegen/error.rs
@@ -82,6 +82,9 @@ pub(crate) enum InternalError {
/// Expected memory location to be addressed via the stack pointer.
#[error("Expected stack pointer addressing")]
SPAddressingExpected,
+ /// Expected a fully resolved results area.
+ #[error("Results area expected")]
+ ResultsAreaExpected,
/// Stack pointer offset is illegal.
#[error("Invalid stack pointer offset")]
InvalidSPOffset,
@@ -128,6 +131,10 @@ impl CodeGenError {
Self::Internal(InternalError::SPAddressingExpected)
}
+ pub(crate) const fn results_area_expected() -> Self {
+ Self::Internal(InternalError::ResultsAreaExpected)
+ }
+
pub(crate) const fn invalid_sp_offset() -> Self {
Self::Internal(InternalError::InvalidSPOffset)
}
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 1fbd393c49..c59499da8f 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -611,8 +611,16 @@ where
self.context.stack.len() == 0,
CodeGenError::unexpected_value_in_value_stack()
);
- self.masm.free_stack(self.context.frame.locals_size)?;
- self.masm.epilogue()?;
+ let stack_args_size = if self.sig.call_conv.is_default() {
+ crate::abi::align_to(
+ self.sig.params_stack_size(),
+ u32::from(M::ABI::call_stack_align()),
+ )
+ } else {
+ 0
+ };
+ self.masm
+ .epilogue(self.context.frame.locals_size, stack_args_size)?;
self.masm.end_source_loc()?;
Ok(())
}
diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs
index 6253bb0c19..0b215d3869 100644
--- a/winch/codegen/src/isa/aarch64/asm.rs
+++ b/winch/codegen/src/isa/aarch64/asm.rs
@@ -27,7 +27,7 @@ use cranelift_codegen::{
FPULeftShiftImm, FPUOp1, FPUOp2,
FPUOpRI::{self, UShr32, UShr64},
FPUOpRIMod, FPURightShiftImm, FpuRoundMode, Imm12, ImmLogic, ImmShift, Inst, IntToFpuOp,
- PairAMode, ScalarSize, VecALUOp, VecLanesOp, VecMisc2, VectorSize,
+ PairAMode, ReturnCallInfo, ScalarSize, VecALUOp, VecLanesOp, VecMisc2, VectorSize,
emit::{EmitInfo, EmitState},
},
settings,
@@ -1493,6 +1493,36 @@ impl Assembler {
})
}
+ /// Emit a direct tail jump to a named target.
+ pub fn tail_jump_with_name(&mut self, name: UserExternalNameRef) {
+ // Winch has already replaced the frame, so the generic Cranelift
+ // return-call sequence must see a zero-sized default frame here.
+ self.emit(Inst::ReturnCall {
+ info: Box::new(ReturnCallInfo {
+ dest: ExternalName::user(name),
+ uses: Default::default(),
+ new_stack_arg_size: 0,
+ key: None,
+ sign_return_address_all: false,
+ }),
+ })
+ }
+
+ /// Emit an indirect tail jump to the address in `callee`.
+ pub fn tail_jump_with_reg(&mut self, callee: Reg) {
+ // Winch has already replaced the frame, so the generic Cranelift
+ // return-call sequence must see a zero-sized default frame here.
+ self.emit(Inst::ReturnCallInd {
+ info: Box::new(ReturnCallInfo {
+ dest: callee.into(),
+ uses: Default::default(),
+ new_stack_arg_size: 0,
+ key: None,
+ sign_return_address_all: false,
+ }),
+ })
+ }
+
/// Load the min value for an integer of size out_size, as a floating-point
/// of size `in-size`, into register `rd`.
fn min_fp_value(
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 7d5ac568d2..5868651288 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -9,7 +9,9 @@ use crate::{
Result,
abi::{self, align_to, calculate_frame_adjustment, local::LocalSlot, vmctx},
bail,
- codegen::{CodeGenContext, CodeGenError, Emission, FuncEnv, ptr_type_from_ptr_size},
+ codegen::{
+ CodeGenContext, CodeGenError, Emission, FuncEnv, TailCallPlan, ptr_type_from_ptr_size,
+ },
format_err,
isa::{
CallingConvention,
@@ -219,7 +221,7 @@ impl Masm for MacroAssembler {
})
}
- fn frame_restore(&mut self) -> Result<()> {
+ fn frame_restore(&mut self, stack_args_size: u32) -> Result<()> {
debug_assert_eq!(self.sp_offset, 0);
// Sync the real stack pointer with the value of the shadow stack
@@ -250,6 +252,15 @@ impl Masm for MacroAssembler {
let addr = Address::post_indexed_from_sp_for_pair(offset);
self.asm.ldp(fp, lr, addr.to_pair_addressing_mode());
+
+ if stack_args_size > 0 {
+ self.add_ir(
+ writable!(regs::sp()),
+ regs::sp(),
+ I::I64(stack_args_size.into()),
+ OperandSize::S64,
+ )?;
+ }
self.asm.ret();
Ok(())
}
@@ -316,6 +327,21 @@ impl Masm for MacroAssembler {
Ok(())
}
+ fn restore_stack_after_call(&mut self, reserved_size: u32, callee_pop_size: u32) -> Result<()> {
+ let caller_pop_size = reserved_size
+ .checked_sub(callee_pop_size)
+ .ok_or_else(|| CodeGenError::invalid_sp_offset())?;
+ if callee_pop_size > 0 {
+ self.decrement_sp(callee_pop_size);
+ // The architectural SP reflects the callee's pop, while the
+ // callee-saved shadow SP does not. Synchronize them before freeing
+ // the caller-owned alignment space.
+ self.move_sp_to_shadow_sp();
+ }
+ self.free_stack(caller_pop_size)?;
+ Ok(())
+ }
+
fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()> {
self.sp_offset = offset.as_u32();
Ok(())
@@ -466,6 +492,138 @@ impl Masm for MacroAssembler {
Ok(total_stack)
}
+ fn finish_tail_call_same_size(&mut self) -> Result<()> {
+ self.load_ptr(
+ Address::offset(regs::fp(), -i64::from(SHADOW_STACK_POINTER_SLOT_SIZE)),
+ writable!(regs::shadow_sp()),
+ )?;
+
+ let zero = Imm12::maybe_from_u64(0).unwrap();
+ self.asm
+ .add_ir(zero, regs::fp(), writable!(regs::sp()), OperandSize::S64);
+ let offset = SImm7Scaled::maybe_from_i64(16, types::I64)
+ .expect("Frame pointer offset 16 is valid for pair addressing");
+ let addr = Address::post_indexed_from_sp_for_pair(offset);
+ self.asm
+ .ldp(regs::fp(), regs::lr(), addr.to_pair_addressing_mode());
+
+ Ok(())
+ }
+
+ fn finish_tail_call_empty(&mut self, plan: TailCallPlan) -> Result<()> {
+ let word_bytes = u32::from(<Self::ABI as ABI>::word_bytes());
+
+ // A zero-sized callee has no argument move that could overwrite frame
+ // state. Restore it directly and position SP at the end of the old
+ // incoming argument area.
+ self.load_ptr(
+ Address::offset(regs::fp(), -i64::from(SHADOW_STACK_POINTER_SLOT_SIZE)),
+ writable!(regs::shadow_sp()),
+ )?;
+ self.load_ptr(
+ Address::offset(regs::fp(), i64::from(word_bytes)),
+ writable!(regs::lr()),
+ )?;
+
+ let entry_sp_offset = plan.callee_args_from_fp.unsigned_abs();
+ if let Some(imm) = Imm12::maybe_from_u64(entry_sp_offset) {
+ assert!(plan.callee_args_from_fp >= 0);
+ self.asm
+ .add_ir(imm, regs::fp(), writable!(regs::sp()), OperandSize::S64);
+ } else {
+ self.with_scratch::<IntScratch, _>(|masm, work| {
+ masm.asm
+ .mov_ir(work.writable(), I::I64(entry_sp_offset), OperandSize::S64);
+ masm.asm.add_rrr(
+ regs::fp(),
+ work.inner(),
+ writable!(regs::sp()),
+ OperandSize::S64,
+ );
+ });
+ }
+
+ self.load_ptr(Address::offset(regs::fp(), 0), writable!(regs::fp()))
+ }
+
+ fn with_tail_call_resize(
+ &mut self,
+ plan: TailCallPlan,
+ move_args: impl FnOnce(&mut Self, u32) -> Result<()>,
+ ) -> Result<()> {
+ let word_bytes = u32::from(<Self::ABI as ABI>::word_bytes());
+
+ // Save FP and the caller's shadow SP below the staged arguments. Both
+ // integer scratch registers must be available during the argument move:
+ // one holds the value and the other materializes large address offsets.
+ let scratch_size = align_to(
+ 2 * word_bytes,
+ u32::from(<Self::ABI as ABI>::call_stack_align()),
+ );
+ self.reserve_stack(scratch_size)?;
+
+ self.with_scratch::<IntScratch, _>(|masm, work| {
+ // The argument move can overwrite all three original frame slots.
+ masm.load_ptr(Address::offset(regs::fp(), 0), work.writable())?;
+ masm.store_ptr(work.inner(), Address::from_shadow_sp(0))?;
+ masm.load_ptr(
+ Address::offset(regs::fp(), -i64::from(SHADOW_STACK_POINTER_SLOT_SIZE)),
+ work.writable(),
+ )?;
+ masm.store_ptr(work.inner(), Address::from_shadow_sp(i64::from(word_bytes)))
+ })?;
+ self.load_ptr(
+ Address::offset(regs::fp(), i64::from(word_bytes)),
+ writable!(regs::lr()),
+ )?;
+
+ move_args(self, scratch_size)?;
+
+ self.with_scratch::<IntScratch, _>(|masm, work| {
+ // Calculate the callee's entry SP while the current FP is available.
+ let magnitude = plan.callee_args_from_fp.unsigned_abs();
+ if let Some(imm) = Imm12::maybe_from_u64(magnitude) {
+ if plan.callee_args_from_fp >= 0 {
+ masm.asm
+ .add_ir(imm, regs::fp(), work.writable(), OperandSize::S64);
+ } else {
+ masm.asm
+ .sub_ir(imm, regs::fp(), work.writable(), OperandSize::S64);
+ }
+ } else {
+ masm.asm
+ .mov_ir(work.writable(), I::I64(magnitude), OperandSize::S64);
+ if plan.callee_args_from_fp >= 0 {
+ masm.asm
+ .add_rrr(work.inner(), regs::fp(), work.writable(), OperandSize::S64);
+ } else {
+ masm.asm
+ .sub_rrr(work.inner(), regs::fp(), work.writable(), OperandSize::S64);
+ }
+ }
+
+ // Finish all old-frame reads before advancing SP. The tail callee
+ // will save FP, LR, and x28 into its replacement frame.
+ masm.load_ptr(Address::from_shadow_sp(0), writable!(regs::fp()))?;
+ masm.load_ptr(
+ Address::from_shadow_sp(i64::from(word_bytes)),
+ writable!(regs::shadow_sp()),
+ )?;
+
+ let zero = Imm12::maybe_from_u64(0).unwrap();
+ masm.asm
+ .add_ir(zero, work.inner(), writable!(regs::sp()), OperandSize::S64);
+ wasmtime_environ::error::Ok(())
+ })
+ }
+
+ fn tail_jump(&mut self, callee: CalleeKind) {
+ match callee {
+ CalleeKind::Indirect(reg) => self.asm.tail_jump_with_reg(reg),
+ CalleeKind::Direct(name) => self.asm.tail_jump_with_name(name),
+ }
+ }
+
fn load(&mut self, src: Address, dst: WritableReg, size: OperandSize) -> Result<()> {
src.to_addressing_mode(self, size, |masm, mem| {
Ok(masm.asm.uload(mem, dst, size, TRUSTED_FLAGS))
@@ -1199,6 +1357,10 @@ impl Masm for MacroAssembler {
Ok(Address::offset(reg, offset as i64))
}
+ fn address_at_fp(&self, offset: i64) -> Result<Self::Address> {
+ Ok(Address::offset(regs::fp(), offset))
+ }
+
fn cmp_with_set(
&mut self,
dst: WritableReg,
diff --git a/winch/codegen/src/isa/mod.rs b/winch/codegen/src/isa/mod.rs
index 4ed9204c2d..fc1ead7fac 100644
--- a/winch/codegen/src/isa/mod.rs
+++ b/winch/codegen/src/isa/mod.rs
@@ -94,8 +94,9 @@ pub enum CallingConvention {
/// See [cranelift_codegen::isa::CallConv::AppleAarch64]
AppleAarch64,
/// The default calling convention for Winch. It largely follows SystemV
- /// for parameter and result handling. This calling convention is part of
- /// Winch's default ABI `crate::abi::ABI`.
+ /// for parameter and result handling, except that callees pop their stack
+ /// arguments. This calling convention is part of Winch's default ABI
+ /// `crate::abi::ABI`.
Default,
}
diff --git a/winch/codegen/src/isa/x64/address.rs b/winch/codegen/src/isa/x64/address.rs
index eae32e2831..98056f689a 100644
--- a/winch/codegen/src/isa/x64/address.rs
+++ b/winch/codegen/src/isa/x64/address.rs
@@ -25,6 +25,14 @@ impl Address {
Self::Offset { base, offset }
}
+ /// Create an address with a signed offset.
+ pub fn offset_i32(base: Reg, offset: i32) -> Self {
+ Self::Offset {
+ base,
+ offset: offset as u32,
+ }
+ }
+
/// Create an address for a constant.
pub fn constant(data: VCodeConstant) -> Self {
Self::Const(data)
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 094c0f1d41..7183c336d0 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -12,6 +12,7 @@ use crate::{
use cranelift_codegen::{
CallInfo, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState,
MachLabel, PatchRegion, Writable,
+ binemit::Reloc,
ir::{ExternalName, MemFlagsData, SourceLoc, TrapCode, Type, UserExternalNameRef, types},
isa::{
unwind::UnwindInst,
@@ -379,9 +380,13 @@ impl Assembler {
self.emit(Inst::External { inst });
}
- /// Return instruction.
- pub fn ret(&mut self) {
- let inst = asm::inst::retq_zo::new().into();
+ /// Return and pop the given number of stack-argument bytes.
+ pub fn ret(&mut self, stack_args_size: u16) {
+ let inst = if stack_args_size == 0 {
+ asm::inst::retq_zo::new().into()
+ } else {
+ asm::inst::retq_i::new(stack_args_size).into()
+ };
self.emit(Inst::External { inst });
}
@@ -1202,6 +1207,12 @@ impl Assembler {
self.emit(Inst::External { inst });
}
+ /// Add a sign-extended 8-bit immediate to a 64-bit register.
+ pub fn add_ir8(&mut self, imm: i8, dst: WritableReg) {
+ let inst = asm::inst::addq_mi_sxb::new(pair_gpr(dst), imm).into();
+ self.emit(Inst::External { inst });
+ }
+
/// Add register and register.
pub fn add_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) {
let dst = pair_gpr(dst);
@@ -1619,6 +1630,29 @@ impl Assembler {
});
}
+ /// Emit a tail jump to an unknown location through a register.
+ pub fn tail_jump_with_reg(&mut self, callee: Reg) {
+ let callee = Gpr::unwrap_new(callee.into());
+ let inst = asm::inst::jmpq_m::new(callee).into();
+ self.emit(Inst::External { inst });
+ self.buffer.add_call_site();
+ }
+
+ /// Emit a tail jump to a locally defined function through an index.
+ pub fn tail_jump_with_name(&mut self, name: UserExternalNameRef) {
+ let inst = asm::inst::jmp_d32::new(0).into();
+ self.emit(Inst::External { inst });
+
+ let offset = self.buffer.cur_offset();
+ self.buffer.add_reloc_at_offset(
+ offset - 4,
+ Reloc::X86CallPCRel4,
+ &ExternalName::user(name),
+ -4,
+ );
+ self.buffer.add_call_site();
+ }
+
/// Emits a conditional jump to the given label.
pub fn jmp_if(&mut self, cc: impl Into<CC>, taken: MachLabel) {
self.emit(Inst::WinchJmpIf {
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index f708ea7bad..8b2684142e 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -5,6 +5,7 @@ use super::{
asm::{Assembler, PatchableAddToReg, VcmpKind, VcvtKind, VroundMode},
regs::{self, rbp, rsp, scratch_fpr_bitset, scratch_gpr_bitset},
};
+use crate::codegen::TailCallPlan;
use crate::masm::{
DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm as I, IntCmpKind,
IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind, OperandSize, RegImm,
@@ -217,6 +218,15 @@ impl Masm for MacroAssembler {
Ok(())
}
+ fn restore_stack_after_call(&mut self, reserved_size: u32, callee_pop_size: u32) -> Result<()> {
+ let caller_pop_size = reserved_size
+ .checked_sub(callee_pop_size)
+ .ok_or_else(|| CodeGenError::invalid_sp_offset())?;
+ self.decrement_sp(callee_pop_size);
+ self.free_stack(caller_pop_size)?;
+ Ok(())
+ }
+
fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()> {
self.sp_offset = offset.as_u32();
@@ -356,6 +366,103 @@ impl Masm for MacroAssembler {
Ok(total_stack)
}
+ fn finish_tail_call_same_size(&mut self) -> Result<()> {
+ self.asm.mov_rr(rbp(), writable!(rsp()), OperandSize::S64);
+ self.asm.pop_r(writable!(rbp()));
+ Ok(())
+ }
+
+ fn finish_tail_call_empty(&mut self, plan: TailCallPlan) -> Result<()> {
+ let word_bytes = u32::from(<Self::ABI as ABI>::word_bytes());
+ let return_address_offset = i32::try_from(i64::from(word_bytes) + plan.args_delta)?;
+
+ // A zero-sized callee has no arguments to copy, and moving the return
+ // address cannot overwrite the current frame pointer. Use one register
+ // to slide the return address and then restore the frame pointer.
+ self.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load_ptr(
+ Address::offset_i32(rbp(), i32::from(<Self::ABI as ABI>::word_bytes())),
+ scratch.writable(),
+ )?;
+ masm.store_ptr(
+ scratch.inner(),
+ Address::offset_i32(rbp(), return_address_offset),
+ )?;
+
+ masm.load_ptr(Address::offset(rbp(), 0), scratch.writable())?;
+ masm.asm.lea(
+ &Address::offset_i32(rbp(), return_address_offset),
+ writable!(rsp()),
+ OperandSize::S64,
+ );
+ masm.asm
+ .mov_rr(scratch.inner(), writable!(rbp()), OperandSize::S64);
+
+ wasmtime_environ::error::Ok(())
+ })
+ }
+
+ fn with_tail_call_resize(
+ &mut self,
+ plan: TailCallPlan,
+ move_args: impl FnOnce(&mut Self, u32) -> Result<()>,
+ ) -> Result<()> {
+ let word_bytes = u32::from(<Self::ABI as ABI>::word_bytes());
+ let return_address_offset = i32::try_from(i64::from(word_bytes) + plan.args_delta)?;
+
+ // Preserve the return address and caller frame pointer in two
+ // scratch slots below the staged arguments. The argument move
+ // can overwrite both of their original frame slots.
+ let scratch_size = align_to(
+ word_bytes * 2,
+ u32::from(<Self::ABI as ABI>::call_stack_align()),
+ );
+ self.reserve_stack(scratch_size)?;
+
+ // Save the frame state in memory so that the scratch register is
+ // available to the generic argument-moving callback.
+ self.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.load_ptr(
+ Address::offset_i32(rbp(), i32::from(<Self::ABI as ABI>::word_bytes())),
+ scratch.writable(),
+ )?;
+ masm.store_ptr(scratch.inner(), Address::offset(rsp(), 0))?;
+ masm.load_ptr(Address::offset(rbp(), 0), scratch.writable())?;
+ masm.store_ptr(scratch.inner(), Address::offset(rsp(), word_bytes))?;
+ wasmtime_environ::error::Ok(())
+ })?;
+
+ move_args(self, scratch_size)?;
+
+ self.with_scratch::<IntScratch, _>(|masm, scratch| {
+ // Slide the original return address so that the end of the
+ // incoming argument area remains anchored across a chain
+ // of tail calls, then discard this frame.
+ masm.load_ptr(Address::offset(rsp(), 0), scratch.writable())?;
+ masm.store_ptr(
+ scratch.inner(),
+ Address::offset_i32(rbp(), return_address_offset),
+ )?;
+ masm.load_ptr(Address::offset(rsp(), word_bytes), scratch.writable())?;
+ masm.asm.lea(
+ &Address::offset_i32(rbp(), return_address_offset),
+ writable!(rsp()),
+ OperandSize::S64,
+ );
+ masm.asm
+ .mov_rr(scratch.inner(), writable!(rbp()), OperandSize::S64);
+
+ wasmtime_environ::error::Ok(())
+ })
+ }
+
+ fn tail_jump(&mut self, callee: CalleeKind) {
+ match callee {
+ CalleeKind::Indirect(reg) => self.asm.tail_jump_with_reg(reg),
+ CalleeKind::Direct(name) => self.asm.tail_jump_with_name(name),
+ }
+ }
+
fn load_ptr(&mut self, src: Self::Address, dst: WritableReg) -> Result<()> {
self.load(src, dst, self.ptr_size)
}
@@ -948,10 +1055,51 @@ impl Masm for MacroAssembler {
Ok(())
}
- fn frame_restore(&mut self) -> Result<()> {
+ fn epilogue(&mut self, locals_size: u32, stack_args_size: u32) -> Result<()> {
+ if stack_args_size == 0 {
+ self.free_stack(locals_size)?;
+ return self.frame_restore(0);
+ }
+
+ // At this boundary SP + locals_size == FP. Move the return address up
+ // by the argument-area size, then discard the frame and finish with a
+ // plain RET instead of an immediate-pop RET.
+ assert_eq!(self.sp_offset, locals_size);
+ let destination = stack_args_size.checked_add(8).unwrap();
+ let sp_destination = locals_size.checked_add(destination).unwrap();
+ self.with_scratch::<IntScratch, _>(|masm, scratch| {
+ // Use SP-relative addressing when all displacements and the ADD
+ // fit signed 8-bit encodings; use FP-relative addressing otherwise.
+ if let Ok(destination) = i8::try_from(sp_destination) {
+ let return_slot = locals_size.checked_add(8).unwrap();
+ masm.load_ptr(Address::offset(rsp(), return_slot), scratch.writable())?;
+ masm.store_ptr(scratch.inner(), Address::offset(rsp(), sp_destination))?;
+ masm.load_ptr(Address::offset(rsp(), locals_size), writable!(rbp()))?;
+ masm.asm.add_ir8(destination, writable!(rsp()));
+ } else {
+ masm.load_ptr(Address::offset(rbp(), 8), scratch.writable())?;
+ masm.store_ptr(scratch.inner(), Address::offset(rbp(), destination))?;
+ masm.asm.lea(
+ &Address::offset(rbp(), destination),
+ scratch.writable(),
+ OperandSize::S64,
+ );
+ masm.load_ptr(Address::offset(rbp(), 0), writable!(rbp()))?;
+ masm.asm
+ .mov_rr(scratch.inner(), writable!(rsp()), OperandSize::S64);
+ }
+ // Both paths finish reading the old frame before advancing SP.
+ masm.sp_offset = 0;
+ wasmtime_environ::error::Ok(())
+ })?;
+ self.asm.ret(0);
+ Ok(())
+ }
+
+ fn frame_restore(&mut self, stack_args_size: u32) -> Result<()> {
debug_assert_eq!(self.sp_offset, 0);
self.asm.pop_r(writable!(rbp()));
- self.asm.ret();
+ self.asm.ret(u16::try_from(stack_args_size)?);
Ok(())
}
@@ -967,6 +1115,10 @@ impl Masm for MacroAssembler {
Ok(Address::offset(reg, offset))
}
+ fn address_at_fp(&self, offset: i64) -> Result<Self::Address> {
+ Ok(Address::offset_i32(rbp(), i32::try_from(offset)?))
+ }
+
fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) -> Result<()> {
match src2 {
RegImm::Imm(imm) => {
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 603bf86dc4..01cb207196 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1,6 +1,6 @@
use crate::Result;
use crate::abi::{self, LocalSlot, align_to};
-use crate::codegen::{CodeGenContext, Emission, FuncEnv};
+use crate::codegen::{CodeGenContext, Emission, FuncEnv, TailCallPlan};
use crate::isa::{
CallingConvention,
reg::{Reg, RegClass, WritableReg, writable},
@@ -1410,14 +1410,15 @@ pub(crate) trait MacroAssembler {
fn frame_setup(&mut self) -> Result<()>;
/// Generate the frame restore sequence.
- fn frame_restore(&mut self) -> Result<()>;
+ fn frame_restore(&mut self, stack_args_size: u32) -> Result<()>;
/// Emit a stack check.
fn check_stack(&mut self, vmctx: Reg) -> Result<()>;
/// Emit the function epilogue.
- fn epilogue(&mut self) -> Result<()> {
- self.frame_restore()
+ fn epilogue(&mut self, locals_size: u32, stack_args_size: u32) -> Result<()> {
+ self.free_stack(locals_size)?;
+ self.frame_restore(stack_args_size)
}
/// Reserve stack space.
@@ -1426,6 +1427,14 @@ pub(crate) trait MacroAssembler {
/// Free stack space.
fn free_stack(&mut self, bytes: u32) -> Result<()>;
+ /// Reclaim stack space after the callee returns.
+ ///
+ /// `callee_pop_size` bytes have already been removed by a default-ABI
+ /// callee. The implementation must update its abstract stack accounting
+ /// for those bytes and physically free only the remaining caller-owned
+ /// space, including alignment padding and any consumed argument spills.
+ fn restore_stack_after_call(&mut self, reserved_size: u32, callee_pop_size: u32) -> Result<()>;
+
/// Reset the stack pointer to the given offset;
///
/// Used to reset the stack pointer to a given offset
@@ -1457,6 +1466,9 @@ pub(crate) trait MacroAssembler {
/// of the given register.
fn address_at_reg(&self, reg: Reg, offset: u32) -> Result<Self::Address>;
+ /// Construct an address relative to the frame pointer.
+ fn address_at_fp(&self, offset: i64) -> Result<Self::Address>;
+
/// Emit a function call to either a local or external function.
fn call(
&mut self,
@@ -1469,6 +1481,28 @@ pub(crate) trait MacroAssembler {
finalize: impl FnMut(&mut Self, &mut CodeGenContext<Emission>) -> Result<()>,
) -> Result<u32>;
+ /// Finish replacing a frame when the caller and tail callee have equally
+ /// sized incoming stack-argument areas.
+ fn finish_tail_call_same_size(&mut self) -> Result<()>;
+
+ /// Finish replacing a frame for a tail callee with no stack arguments.
+ fn finish_tail_call_empty(&mut self, plan: TailCallPlan) -> Result<()>;
+
+ /// Preserve the frame state that a resized argument move may overwrite,
+ /// invoke `move_args`, and then finish replacing the current frame.
+ ///
+ /// `move_args` receives the distance from the current stack pointer to the
+ /// staged argument block. A callback is used because an implementation may
+ /// need to keep scratch registers live across the argument move.
+ fn with_tail_call_resize(
+ &mut self,
+ plan: TailCallPlan,
+ move_args: impl FnOnce(&mut Self, u32) -> Result<()>,
+ ) -> Result<()>;
+
+ /// Jump to a tail callee without adding a return address.
+ fn tail_jump(&mut self, callee: CalleeKind);
+
/// Record a GC stack map at the current code offset, which must be the
/// return address of the call emitted immediately before. Each offset is
/// the distance from the stack pointer at the call site to a slot holding
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 3cd65dee47..80d009c82b 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -191,6 +191,7 @@ macro_rules! def_unsupported {
(emit LocalGet $($rest:tt)*) => {};
(emit LocalSet $($rest:tt)*) => {};
(emit Call $($rest:tt)*) => {};
+ (emit ReturnCall $($rest:tt)*) => {};
(emit End $($rest:tt)*) => {};
(emit Nop $($rest:tt)*) => {};
(emit If $($rest:tt)*) => {};
@@ -215,6 +216,7 @@ macro_rules! def_unsupported {
(emit Drop $($rest:tt)*) => {};
(emit BrTable $($rest:tt)*) => {};
(emit CallIndirect $($rest:tt)*) => {};
+ (emit ReturnCallIndirect $($rest:tt)*) => {};
(emit TableInit $($rest:tt)*) => {};
(emit TableCopy $($rest:tt)*) => {};
(emit TableGet $($rest:tt)*) => {};
@@ -1695,6 +1697,19 @@ where
Ok(())
}
+ fn visit_return_call(&mut self, index: u32) -> Self::Output {
+ let callee = self.env.callee_from_index(FuncIndex::from_u32(index));
+ FnCall::emit_return::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ self.sig.params_stack_size(),
+ callee,
+ )?;
+ self.context.reachable = false;
+ Ok(())
+ }
+
fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output {
// Spill now because `emit_table_get` and the `FnCall::emit`
// invocations will both trigger spills since they both call functions.
@@ -1727,6 +1742,38 @@ where
Ok(())
}
+ fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output {
+ // As with a normal indirect call, spill before table lookup because
+ // lazy funcref initialization can itself make a call.
+ self.context.spill(self.masm)?;
+
+ let type_index = TypeIndex::from_u32(type_index);
+ let table_index = TableIndex::from_u32(table_index);
+
+ self.emit_table_get(table_index)?;
+
+ let funcref_ptr = self
+ .context
+ .stack
+ .peek()
+ .map(|v| v.unwrap_reg())
+ .ok_or_else(|| CodeGenError::missing_values_in_stack())?;
+ self.masm
+ .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?;
+ self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?;
+
+ let callee = self.env.funcref(type_index);
+ FnCall::emit_return::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ self.sig.params_stack_size(),
+ callee,
+ )?;
+ self.context.reachable = false;
+ Ok(())
+ }
+
fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output {
self.emit_table_init(ElemIndex::from_u32(elem), TableIndex::from_u32(table))
}
@@ -1952,6 +1999,12 @@ where
&mut self.context,
)?);
+ // Unreachable loops still need a control frame, but must not emit
+ // runtime-limit checks against a potentially unwound stack frame.
+ if !self.context.reachable {
+ return Ok(());
+ }
+
self.maybe_emit_epoch_check()?;
self.maybe_emit_fuel_check()
}