Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Rewrite `wasmtime_fiber_init` in Rust (#11860)

Alex Crichton65879713410a
diff --git a/crates/fiber/src/stackswitch/aarch64.rs b/crates/fiber/src/stackswitch/aarch64.rs
index 3711e83920..0d2428516a 100644
--- a/crates/fiber/src/stackswitch/aarch64.rs
+++ b/crates/fiber/src/stackswitch/aarch64.rs
@@ -25,14 +25,10 @@ cfg_if::cfg_if! {
macro_rules! paci1716 { () => ("pacib1716\n"); }
macro_rules! pacisp { () => ("pacibsp\n"); }
macro_rules! autisp { () => ("autibsp\n"); }
- macro_rules! sym_adrp { ($s:tt) => (concat!($s, "@PAGE")); }
- macro_rules! sym_add { ($s:tt) => (concat!($s, "@PAGEOFF")); }
} else {
macro_rules! paci1716 { () => ("pacia1716\n"); }
macro_rules! pacisp { () => ("paciasp\n"); }
macro_rules! autisp { () => ("autiasp\n"); }
- macro_rules! sym_adrp { ($s:tt) => (concat!($s, "")); }
- macro_rules! sym_add { ($s:tt) => (concat!(":lo12:", $s)); }
}
}
@@ -48,15 +44,15 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x
// Save all callee-saved registers on the stack since we're
// assuming they're clobbered as a result of the stack switch.
stp x29, x30, [sp, -16]!
- stp x20, x19, [sp, -16]!
- stp x22, x21, [sp, -16]!
- stp x24, x23, [sp, -16]!
- stp x26, x25, [sp, -16]!
- stp x28, x27, [sp, -16]!
- stp d9, d8, [sp, -16]!
- stp d11, d10, [sp, -16]!
- stp d13, d12, [sp, -16]!
- stp d15, d14, [sp, -16]!
+ stp x27, x28, [sp, -16]!
+ stp x25, x26, [sp, -16]!
+ stp x23, x24, [sp, -16]!
+ stp x21, x22, [sp, -16]!
+ stp x19, x20, [sp, -16]!
+ stp d14, d15, [sp, -16]!
+ stp d12, d13, [sp, -16]!
+ stp d10, d11, [sp, -16]!
+ stp d8, d9, [sp, -16]!
// Load our previously saved stack pointer to resume to, and save
// off our current stack pointer on where to come back to
@@ -68,15 +64,16 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x
// Switch to the new stack and restore all our callee-saved
// registers after the switch and return to our new stack.
mov sp, x8
- ldp d15, d14, [sp], 16
- ldp d13, d12, [sp], 16
- ldp d11, d10, [sp], 16
- ldp d9, d8, [sp], 16
- ldp x28, x27, [sp], 16
- ldp x26, x25, [sp], 16
- ldp x24, x23, [sp], 16
- ldp x22, x21, [sp], 16
- ldp x20, x19, [sp], 16
+ ldp d8, d9, [sp], 16
+ ldp d10, d11, [sp], 16
+ ldp d12, d13, [sp], 16
+ ldp d14, d15, [sp], 16
+
+ ldp x19, x20, [sp], 16
+ ldp x21, x22, [sp], 16
+ ldp x23, x24, [sp], 16
+ ldp x25, x26, [sp], 16
+ ldp x27, x28, [sp], 16
ldp x29, x30, [sp], 16
",
autisp!(),
@@ -88,54 +85,85 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x
));
}
-// We set up the newly initialized fiber, so that it resumes execution
-// from wasmtime_fiber_start(). As a result, we need a signed address
-// of this function, so there are 2 requirements:
-// * The fiber stack pointer value that is used by the signing operation
-// must match the value when the pointer is authenticated inside
-// wasmtime_fiber_switch(), otherwise the latter would fault
-// * We would like to use an instruction that is executed as a no-op by
-// processors that do not support PAuth, so that the code is
-// backward-compatible and there is no duplication; `PACIA1716` is a
-// suitable one, which has the following operand register
-// conventions:
-// * X17 contains the pointer value to sign
-// * X16 contains the modifier value
-//
-// TODO: Use the PACGA instruction to authenticate the saved register
-// state, which avoids creating signed pointers to
-// wasmtime_fiber_start(), and provides wider coverage.
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
- top_of_stack: *mut u8, // x0
- entry_point: extern "C" fn(*mut u8, *mut u8), // x1
- entry_arg0: *mut u8, // x2
+pub(crate) unsafe fn wasmtime_fiber_init(
+ top_of_stack: *mut u8,
+ entry_point: extern "C" fn(*mut u8, *mut u8),
+ entry_arg0: *mut u8, // x2
) {
- naked_asm!(
- concat!(
- "
- .cfi_startproc
- hint #34 // bti c
- sub x16, x0, #16
- adrp x17, ", sym_adrp!("{fiber}"), "
- add x17, x17, ", sym_add!("{fiber}"), "
- ",
- paci1716!(),
- "
- str x17, [x16, -0x8] // x17 => lr
- str x0, [x16, -0x18] // x0 => x19
- stp x2, x1, [x0, -0x38] // x1 => x20, x2 => x21
-
- // `wasmtime_fiber_switch` has an 0xa0 byte stack, and we add 0x10 more for
- // the original reserved 16 bytes.
- add x8, x0, -0xb0
- str x8, [x0, -0x10]
- ret
- .cfi_endproc
- ",
- ),
- fiber = sym wasmtime_fiber_start,
- );
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ d8: u64,
+ d9: u64,
+ d10: u64,
+ d11: u64,
+ d12: u64,
+ d13: u64,
+ d14: u64,
+ d15: u64,
+
+ x19: *mut u8,
+ x20: *mut u8,
+ x21: *mut u8,
+ x22: *mut u8,
+ x23: *mut u8,
+ x24: *mut u8,
+ x25: *mut u8,
+ x26: *mut u8,
+ x27: *mut u8,
+ x28: *mut u8,
+
+ fp: *mut u8,
+ lr: *mut u8,
+
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
+
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ x19: top_of_stack,
+ x20: entry_point as *mut u8,
+ x21: entry_arg0,
+
+ // We set up the newly initialized fiber, so that it resumes
+ // execution from wasmtime_fiber_start(). As a result, we need a
+ // signed address of this function because `wasmtime_fiber_switch`
+ // ends with a `auti{a,b}sp` instruction. There are 2 requirements:
+ // * We would like to use an instruction that is executed as a no-op
+ // by processors that do not support PAuth, so that the code is
+ // backward-compatible and there is no duplication; `PACIA1716` is
+ // a suitable one.
+ // * The fiber stack pointer value that is used by the signing
+ // operation must match the value when the pointer is
+ // authenticated inside wasmtime_fiber_switch(), which is 16 bytes
+ // below the `top_of_stack` which will be `sp` at the time of the
+ // `auti{a,b}sp`.
+ //
+ // TODO: Use the PACGA instruction to authenticate the saved register
+ // state, which avoids creating signed pointers to
+ // wasmtime_fiber_start(), and provides wider coverage.
+ lr: paci1716(wasmtime_fiber_start as *mut u8, top_of_stack.sub(16)),
+
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
+}
+
+/// Signs `r17` with the value in `r16` using either `paci{a,b}1716` depending
+/// on the platform.
+fn paci1716(mut r17: *mut u8, r16: *mut u8) -> *mut u8 {
+ unsafe {
+ core::arch::asm!(
+ paci1716!(),
+ inout("x17") r17,
+ in("x16") r16,
+ );
+ r17
+ }
}
// See the x86_64 file for more commentary on what these CFI directives are
@@ -152,18 +180,19 @@ unsafe extern "C" fn wasmtime_fiber_start() -> ! {
0x6f, /* DW_OP_reg31(%sp) */ \
0x06, /* DW_OP_deref */ \
0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */
- .cfi_rel_offset x29, -0x10
.cfi_rel_offset x30, -0x08
+ .cfi_rel_offset x29, -0x10
.cfi_window_save
- .cfi_rel_offset x19, -0x18
- .cfi_rel_offset x20, -0x20
- .cfi_rel_offset x21, -0x28
- .cfi_rel_offset x22, -0x30
- .cfi_rel_offset x23, -0x38
- .cfi_rel_offset x24, -0x40
- .cfi_rel_offset x25, -0x48
- .cfi_rel_offset x26, -0x50
- .cfi_rel_offset x27, -0x58
+ .cfi_rel_offset x28, -0x18
+ .cfi_rel_offset x27, -0x20
+ .cfi_rel_offset x26, -0x28
+ .cfi_rel_offset x25, -0x30
+ .cfi_rel_offset x24, -0x38
+ .cfi_rel_offset x23, -0x40
+ .cfi_rel_offset x22, -0x48
+ .cfi_rel_offset x21, -0x50
+ .cfi_rel_offset x20, -0x58
+ .cfi_rel_offset x19, -0x60
// Load our two arguments from the stack, where x1 is our start
// procedure and x0 is its first argument. This also blows away the
diff --git a/crates/fiber/src/stackswitch/arm.rs b/crates/fiber/src/stackswitch/arm.rs
index 44f88b4252..f02ff757d4 100644
--- a/crates/fiber/src/stackswitch/arm.rs
+++ b/crates/fiber/src/stackswitch/arm.rs
@@ -29,26 +29,40 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* r
);
}
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
- top_of_stack: *mut u8, // r0
- entry_point: extern "C" fn(*mut u8, *mut u8), // r1
- entry_arg0: *mut u8, // r2
+pub(crate) unsafe fn wasmtime_fiber_init(
+ top_of_stack: *mut u8,
+ entry_point: extern "C" fn(*mut u8, *mut u8),
+ entry_arg0: *mut u8,
) {
- naked_asm!(
- "
- adr r3, {start}
- str r3, [r0, #-0x0c] // => lr
- str r0, [r0, #-0x10] // => r11
- str r1, [r0, #-0x14] // => r10
- str r2, [r0, #-0x18] // => r9
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ r4: *mut u8,
+ r5: *mut u8,
+ r6: *mut u8,
+ r7: *mut u8,
+ r8: *mut u8,
+ r9: *mut u8,
+ r10: *mut u8,
+ r11: *mut u8,
+ lr: *mut u8,
- add r3, r0, #-0x2c
- str r3, [r0, #-0x08]
- bx lr
- ",
- start = sym wasmtime_fiber_start,
- );
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
+
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ r9: entry_arg0,
+ r10: entry_point as *mut u8,
+ r11: top_of_stack,
+ lr: wasmtime_fiber_start as *mut u8,
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
}
#[unsafe(naked)]
diff --git a/crates/fiber/src/stackswitch/riscv64.rs b/crates/fiber/src/stackswitch/riscv64.rs
index 4267dfc149..4883004fa5 100644
--- a/crates/fiber/src/stackswitch/riscv64.rs
+++ b/crates/fiber/src/stackswitch/riscv64.rs
@@ -84,27 +84,49 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* a
);
}
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
- top_of_stack: *mut u8, // a0
- entry_point: extern "C" fn(*mut u8, *mut u8), // a1
- entry_arg0: *mut u8, // a2
+pub(crate) unsafe fn wasmtime_fiber_init(
+ top_of_stack: *mut u8,
+ entry_point: extern "C" fn(*mut u8, *mut u8),
+ entry_arg0: *mut u8,
) {
- naked_asm!(
- "
- lla t0,{}
- sd t0,-0x18(a0) // ra,first should be wasmtime_fiber_start.
- sd a0,-0x20(a0) // fp pointer.
- sd a1,-0x28(a0) // entry_point will load to s1.
- sd a2,-0x30(a0) // entry_arg0 will load to s2.
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ align_to_16_byte_size: u64,
- //
- addi t0,a0,-0xe0
- sd t0,-0x10(a0)
- ret
- ",
- sym wasmtime_fiber_start,
- );
+ fs: [f64; 12],
+
+ s11: *mut u8,
+ s10: *mut u8,
+ s9: *mut u8,
+ s8: *mut u8,
+ s7: *mut u8,
+ s6: *mut u8,
+ s5: *mut u8,
+ s4: *mut u8,
+ s3: *mut u8,
+ s2: *mut u8,
+ s1: *mut u8,
+ fp: *mut u8,
+
+ ra: *mut u8,
+
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
+
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ s1: entry_point as *mut u8,
+ s2: entry_arg0,
+ fp: top_of_stack,
+ ra: wasmtime_fiber_start as *mut u8,
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
}
#[unsafe(naked)]
diff --git a/crates/fiber/src/stackswitch/s390x.rs b/crates/fiber/src/stackswitch/s390x.rs
index b284ee2b7b..15cb543b16 100644
--- a/crates/fiber/src/stackswitch/s390x.rs
+++ b/crates/fiber/src/stackswitch/s390x.rs
@@ -45,29 +45,65 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x
);
}
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
- top_of_stack: *mut u8, // x0
- entry_point: extern "C" fn(*mut u8, *mut u8), // x1
- entry_arg0: *mut u8, // x2
+pub(crate) unsafe fn wasmtime_fiber_init(
+ top_of_stack: *mut u8,
+ entry_point: extern "C" fn(*mut u8, *mut u8),
+ entry_arg0: *mut u8,
) {
- naked_asm!(
- "
- larl %r1, {}
- stg %r1, -48(%r2) // wasmtime_fiber_start - restored into %r14
- stg %r2, -112(%r2) // top_of_stack - restored into %r6
- stg %r3, -104(%r2) // entry_point - restored into %r7
- stg %r4, -96(%r2) // entry_arg0 - restored into %r8
- aghi %r2, -160 // 160 bytes register save area
- stg %r2, 120(%r2) // bottom of register save area - restored into %r15
-
- // `wasmtime_fiber_switch` has a 64 byte stack.
- aghi %r2, -64
- stg %r2, 208(%r2)
- br %r14
- ",
- sym wasmtime_fiber_start,
- );
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ f8: *mut u8,
+ f9: *mut u8,
+ f10: *mut u8,
+ f11: *mut u8,
+ f12: *mut u8,
+ f13: *mut u8,
+ f14: *mut u8,
+ f15: *mut u8,
+
+ back_chain: *mut u8,
+ compiler_reserved: *mut u8,
+
+ r2: *mut u8,
+ r3: *mut u8,
+ r4: *mut u8,
+ r5: *mut u8,
+
+ r6: *mut u8,
+ r7: *mut u8,
+ r8: *mut u8,
+ r9: *mut u8,
+ r10: *mut u8,
+ r11: *mut u8,
+ r12: *mut u8,
+ r13: *mut u8,
+ r14: *mut u8,
+ r15: *mut u8,
+
+ f0: *mut u8,
+ f2: *mut u8,
+ f4: *mut u8,
+ f6: *mut u8,
+
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
+
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ r15: (&raw mut (*initial_stack).back_chain).cast(),
+ r14: wasmtime_fiber_start as *mut u8,
+ r6: top_of_stack,
+ r7: entry_point as *mut u8,
+ r8: entry_arg0,
+
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
}
#[unsafe(naked)]
@@ -82,7 +118,7 @@ unsafe extern "C" fn wasmtime_fiber_start() -> ! {
// match the frame layout in `wasmtime_fiber_switch`.
.cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \
7, /* the byte length of this expression */ \
- 0x7f, 0x90, 0x1, /* DW_OP_breg15 0x90 */ \
+ 0x7f, 0xa0, 0x1, /* DW_OP_breg15 0x90 */ \
0x06, /* DW_OP_deref */ \
0x23, 0xe0, 0x1 /* DW_OP_plus_uconst 0xe0 */
diff --git a/crates/fiber/src/stackswitch/x86.rs b/crates/fiber/src/stackswitch/x86.rs
index ec20a3d045..79869e1bc0 100644
--- a/crates/fiber/src/stackswitch/x86.rs
+++ b/crates/fiber/src/stackswitch/x86.rs
@@ -40,49 +40,52 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
)
}
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
+pub(crate) unsafe fn wasmtime_fiber_init(
top_of_stack: *mut u8,
entry_point: extern "C" fn(*mut u8, *mut u8),
entry_arg0: *mut u8,
) {
- naked_asm!(
- "
- mov eax, 4[esp]
-
- // move top_of_stack to the 2nd argument
- mov -0x0c[eax], eax
+ // Our stack from top-to-bottom looks like:
+ //
+ // * 8 bytes of reserved space per unix.rs (two-pointers space)
+ // * 8 bytes of arguments (two arguments wasmtime_fiber_start forwards)
+ // * 4 bytes of return address
+ // * 16 bytes of saved registers
+ //
+ // Note that after the return address the stack is conveniently 16-byte
+ // aligned as required, so we just leave the arguments on the stack in
+ // `wasmtime_fiber_start` and immediately do the call.
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ // state that will get resumed into from a `wasmtime_fiber_switch`
+ // starting up this fiber.
+ edi: *mut u8,
+ esi: *mut u8,
+ ebx: *mut u8,
+ ebp: *mut u8,
+ return_address: *mut u8,
- // move entry_arg0 to the 1st argument
- mov ecx, 12[esp]
- mov -0x10[eax], ecx
+ // two arguments to `entry_point`
+ arg1: *mut u8,
+ arg2: *mut u8,
- // Move our start function to the return address which the `ret` in
- // `wasmtime_fiber_start` will return to.
- lea ecx, {start}
- mov -0x14[eax], ecx
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
- // And move `entry_point` to get loaded into `%ebp` through the context
- // switch. This'll get jumped to in `wasmtime_fiber_start`.
- mov ecx, 8[esp]
- mov -0x18[eax], ecx
-
- // Our stack from top-to-bottom looks like:
- //
- // * 8 bytes of reserved space per unix.rs (two-pointers space)
- // * 8 bytes of arguments (two arguments wasmtime_fiber_start forwards)
- // * 4 bytes of return address
- // * 16 bytes of saved registers
- //
- // Note that after the return address the stack is conveniently 16-byte
- // aligned as required, so we just leave the arguments on the stack in
- // `wasmtime_fiber_start` and immediately do the call.
- lea ecx, -0x24[eax]
- mov -0x08[eax], ecx
- ret
- ",
- start = sym wasmtime_fiber_start,
- );
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ ebp: entry_point as *mut u8,
+ return_address: wasmtime_fiber_start as *mut u8,
+ arg1: entry_arg0,
+ arg2: top_of_stack,
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
}
#[unsafe(naked)]
diff --git a/crates/fiber/src/stackswitch/x86_64.rs b/crates/fiber/src/stackswitch/x86_64.rs
index 4a826bf9a0..8294073fd1 100644
--- a/crates/fiber/src/stackswitch/x86_64.rs
+++ b/crates/fiber/src/stackswitch/x86_64.rs
@@ -43,38 +43,38 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* r
);
}
-#[unsafe(naked)]
-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(
- top_of_stack: *mut u8, // rdi
- entry_point: extern "C" fn(*mut u8, *mut u8), // rsi
- entry_arg0: *mut u8, // rdx
+pub(crate) unsafe fn wasmtime_fiber_init(
+ top_of_stack: *mut u8,
+ entry_point: extern "C" fn(*mut u8, *mut u8),
+ entry_arg0: *mut u8,
) {
- naked_asm!(
- "
- // Here we're going to set up a stack frame as expected by
- // `wasmtime_fiber_switch`. The values we store here will get restored into
- // registers by that function and the `wasmtime_fiber_start` function will
- // take over and understands which values are in which registers.
- //
- // The first 16 bytes of stack are reserved for metadata, so we start
- // storing values beneath that.
- lea rax, {start}[rip]
- mov -0x18[rdi], rax
- mov -0x20[rdi], rdi // loaded into rbp during switch
- mov -0x28[rdi], rsi // loaded into rbx during switch
- mov -0x30[rdi], rdx // loaded into r12 during switch
+ #[repr(C)]
+ #[derive(Default)]
+ struct InitialStack {
+ r15: *mut u8,
+ r14: *mut u8,
+ r13: *mut u8,
+ r12: *mut u8,
+ rbx: *mut u8,
+ rbp: *mut u8,
+ return_address: *mut u8,
- // And then we specify the stack pointer resumption should begin at. Our
- // `wasmtime_fiber_switch` function consumes 6 registers plus a return
- // pointer, and the top 16 bytes are reserved, so that's:
- //
- // (6 + 1) * 16 + 16 = 0x48
- lea rax, -0x48[rdi]
- mov -0x10[rdi], rax
- ret
- ",
- start = sym wasmtime_fiber_start,
- );
+ // unix.rs reserved space
+ last_sp: *mut u8,
+ run_result: *mut u8,
+ }
+
+ unsafe {
+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
+ initial_stack.write(InitialStack {
+ r12: entry_arg0,
+ rbx: entry_point as *mut u8,
+ rbp: top_of_stack,
+ return_address: wasmtime_fiber_start as *mut u8,
+ last_sp: initial_stack.cast(),
+ ..InitialStack::default()
+ });
+ }
}
// This is a pretty special function that has no real signature. Its use is to