A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Rewrite `wasmtime_fiber_init` in Rust (#11860)
Alex Crichton65879713410a
diff --git a/crates/fiber/src/stackswitch/aarch64.rs b/crates/fiber/src/stackswitch/aarch64.rsindex 3711e83920..0d2428516a 100644--- a/crates/fiber/src/stackswitch/aarch64.rs+++ b/crates/fiber/src/stackswitch/aarch64.rs@@ -25,14 +25,10 @@ cfg_if::cfg_if! {macro_rules! paci1716 { () => ("pacib1716\n"); }macro_rules! pacisp { () => ("pacibsp\n"); }macro_rules! autisp { () => ("autibsp\n"); }- macro_rules! sym_adrp { ($s:tt) => (concat!($s, "@PAGE")); }- macro_rules! sym_add { ($s:tt) => (concat!($s, "@PAGEOFF")); }} else {macro_rules! paci1716 { () => ("pacia1716\n"); }macro_rules! pacisp { () => ("paciasp\n"); }macro_rules! autisp { () => ("autiasp\n"); }- macro_rules! sym_adrp { ($s:tt) => (concat!($s, "")); }- macro_rules! sym_add { ($s:tt) => (concat!(":lo12:", $s)); }}}@@ -48,15 +44,15 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x// Save all callee-saved registers on the stack since we're// assuming they're clobbered as a result of the stack switch.stp x29, x30, [sp, -16]!- stp x20, x19, [sp, -16]!- stp x22, x21, [sp, -16]!- stp x24, x23, [sp, -16]!- stp x26, x25, [sp, -16]!- stp x28, x27, [sp, -16]!- stp d9, d8, [sp, -16]!- stp d11, d10, [sp, -16]!- stp d13, d12, [sp, -16]!- stp d15, d14, [sp, -16]!+ stp x27, x28, [sp, -16]!+ stp x25, x26, [sp, -16]!+ stp x23, x24, [sp, -16]!+ stp x21, x22, [sp, -16]!+ stp x19, x20, [sp, -16]!+ stp d14, d15, [sp, -16]!+ stp d12, d13, [sp, -16]!+ stp d10, d11, [sp, -16]!+ stp d8, d9, [sp, -16]!// Load our previously saved stack pointer to resume to, and save// off our current stack pointer on where to come back to@@ -68,15 +64,16 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x// Switch to the new stack and restore all our callee-saved// registers after the switch and return to our new stack.mov sp, x8- ldp d15, d14, [sp], 16- ldp d13, d12, [sp], 16- ldp d11, d10, [sp], 16- ldp d9, d8, [sp], 16- ldp x28, x27, [sp], 16- ldp x26, x25, [sp], 16- ldp x24, x23, [sp], 16- ldp x22, x21, [sp], 16- ldp x20, x19, [sp], 16+ ldp d8, d9, [sp], 16+ ldp d10, d11, [sp], 16+ ldp d12, d13, [sp], 16+ ldp d14, d15, [sp], 16++ ldp x19, x20, [sp], 16+ ldp x21, x22, [sp], 16+ ldp x23, x24, [sp], 16+ ldp x25, x26, [sp], 16+ ldp x27, x28, [sp], 16ldp x29, x30, [sp], 16",autisp!(),@@ -88,54 +85,85 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x));}-// We set up the newly initialized fiber, so that it resumes execution-// from wasmtime_fiber_start(). As a result, we need a signed address-// of this function, so there are 2 requirements:-// * The fiber stack pointer value that is used by the signing operation-// must match the value when the pointer is authenticated inside-// wasmtime_fiber_switch(), otherwise the latter would fault-// * We would like to use an instruction that is executed as a no-op by-// processors that do not support PAuth, so that the code is-// backward-compatible and there is no duplication; `PACIA1716` is a-// suitable one, which has the following operand register-// conventions:-// * X17 contains the pointer value to sign-// * X16 contains the modifier value-//-// TODO: Use the PACGA instruction to authenticate the saved register-// state, which avoids creating signed pointers to-// wasmtime_fiber_start(), and provides wider coverage.-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(- top_of_stack: *mut u8, // x0- entry_point: extern "C" fn(*mut u8, *mut u8), // x1- entry_arg0: *mut u8, // x2+pub(crate) unsafe fn wasmtime_fiber_init(+ top_of_stack: *mut u8,+ entry_point: extern "C" fn(*mut u8, *mut u8),+ entry_arg0: *mut u8, // x2) {- naked_asm!(- concat!(- "- .cfi_startproc- hint #34 // bti c- sub x16, x0, #16- adrp x17, ", sym_adrp!("{fiber}"), "- add x17, x17, ", sym_add!("{fiber}"), "- ",- paci1716!(),- "- str x17, [x16, -0x8] // x17 => lr- str x0, [x16, -0x18] // x0 => x19- stp x2, x1, [x0, -0x38] // x1 => x20, x2 => x21-- // `wasmtime_fiber_switch` has an 0xa0 byte stack, and we add 0x10 more for- // the original reserved 16 bytes.- add x8, x0, -0xb0- str x8, [x0, -0x10]- ret- .cfi_endproc- ",- ),- fiber = sym wasmtime_fiber_start,- );+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ d8: u64,+ d9: u64,+ d10: u64,+ d11: u64,+ d12: u64,+ d13: u64,+ d14: u64,+ d15: u64,++ x19: *mut u8,+ x20: *mut u8,+ x21: *mut u8,+ x22: *mut u8,+ x23: *mut u8,+ x24: *mut u8,+ x25: *mut u8,+ x26: *mut u8,+ x27: *mut u8,+ x28: *mut u8,++ fp: *mut u8,+ lr: *mut u8,++ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }++ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ x19: top_of_stack,+ x20: entry_point as *mut u8,+ x21: entry_arg0,++ // We set up the newly initialized fiber, so that it resumes+ // execution from wasmtime_fiber_start(). As a result, we need a+ // signed address of this function because `wasmtime_fiber_switch`+ // ends with a `auti{a,b}sp` instruction. There are 2 requirements:+ // * We would like to use an instruction that is executed as a no-op+ // by processors that do not support PAuth, so that the code is+ // backward-compatible and there is no duplication; `PACIA1716` is+ // a suitable one.+ // * The fiber stack pointer value that is used by the signing+ // operation must match the value when the pointer is+ // authenticated inside wasmtime_fiber_switch(), which is 16 bytes+ // below the `top_of_stack` which will be `sp` at the time of the+ // `auti{a,b}sp`.+ //+ // TODO: Use the PACGA instruction to authenticate the saved register+ // state, which avoids creating signed pointers to+ // wasmtime_fiber_start(), and provides wider coverage.+ lr: paci1716(wasmtime_fiber_start as *mut u8, top_of_stack.sub(16)),++ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }+}++/// Signs `r17` with the value in `r16` using either `paci{a,b}1716` depending+/// on the platform.+fn paci1716(mut r17: *mut u8, r16: *mut u8) -> *mut u8 {+ unsafe {+ core::arch::asm!(+ paci1716!(),+ inout("x17") r17,+ in("x16") r16,+ );+ r17+ }}// See the x86_64 file for more commentary on what these CFI directives are@@ -152,18 +180,19 @@ unsafe extern "C" fn wasmtime_fiber_start() -> ! {0x6f, /* DW_OP_reg31(%sp) */ \0x06, /* DW_OP_deref */ \0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */- .cfi_rel_offset x29, -0x10.cfi_rel_offset x30, -0x08+ .cfi_rel_offset x29, -0x10.cfi_window_save- .cfi_rel_offset x19, -0x18- .cfi_rel_offset x20, -0x20- .cfi_rel_offset x21, -0x28- .cfi_rel_offset x22, -0x30- .cfi_rel_offset x23, -0x38- .cfi_rel_offset x24, -0x40- .cfi_rel_offset x25, -0x48- .cfi_rel_offset x26, -0x50- .cfi_rel_offset x27, -0x58+ .cfi_rel_offset x28, -0x18+ .cfi_rel_offset x27, -0x20+ .cfi_rel_offset x26, -0x28+ .cfi_rel_offset x25, -0x30+ .cfi_rel_offset x24, -0x38+ .cfi_rel_offset x23, -0x40+ .cfi_rel_offset x22, -0x48+ .cfi_rel_offset x21, -0x50+ .cfi_rel_offset x20, -0x58+ .cfi_rel_offset x19, -0x60// Load our two arguments from the stack, where x1 is our start// procedure and x0 is its first argument. This also blows away thediff --git a/crates/fiber/src/stackswitch/arm.rs b/crates/fiber/src/stackswitch/arm.rsindex 44f88b4252..f02ff757d4 100644--- a/crates/fiber/src/stackswitch/arm.rs+++ b/crates/fiber/src/stackswitch/arm.rs@@ -29,26 +29,40 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* r);}-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(- top_of_stack: *mut u8, // r0- entry_point: extern "C" fn(*mut u8, *mut u8), // r1- entry_arg0: *mut u8, // r2+pub(crate) unsafe fn wasmtime_fiber_init(+ top_of_stack: *mut u8,+ entry_point: extern "C" fn(*mut u8, *mut u8),+ entry_arg0: *mut u8,) {- naked_asm!(- "- adr r3, {start}- str r3, [r0, #-0x0c] // => lr- str r0, [r0, #-0x10] // => r11- str r1, [r0, #-0x14] // => r10- str r2, [r0, #-0x18] // => r9+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ r4: *mut u8,+ r5: *mut u8,+ r6: *mut u8,+ r7: *mut u8,+ r8: *mut u8,+ r9: *mut u8,+ r10: *mut u8,+ r11: *mut u8,+ lr: *mut u8,- add r3, r0, #-0x2c- str r3, [r0, #-0x08]- bx lr- ",- start = sym wasmtime_fiber_start,- );+ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }++ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ r9: entry_arg0,+ r10: entry_point as *mut u8,+ r11: top_of_stack,+ lr: wasmtime_fiber_start as *mut u8,+ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }}#[unsafe(naked)]diff --git a/crates/fiber/src/stackswitch/riscv64.rs b/crates/fiber/src/stackswitch/riscv64.rsindex 4267dfc149..4883004fa5 100644--- a/crates/fiber/src/stackswitch/riscv64.rs+++ b/crates/fiber/src/stackswitch/riscv64.rs@@ -84,27 +84,49 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* a);}-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(- top_of_stack: *mut u8, // a0- entry_point: extern "C" fn(*mut u8, *mut u8), // a1- entry_arg0: *mut u8, // a2+pub(crate) unsafe fn wasmtime_fiber_init(+ top_of_stack: *mut u8,+ entry_point: extern "C" fn(*mut u8, *mut u8),+ entry_arg0: *mut u8,) {- naked_asm!(- "- lla t0,{}- sd t0,-0x18(a0) // ra,first should be wasmtime_fiber_start.- sd a0,-0x20(a0) // fp pointer.- sd a1,-0x28(a0) // entry_point will load to s1.- sd a2,-0x30(a0) // entry_arg0 will load to s2.+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ align_to_16_byte_size: u64,- //- addi t0,a0,-0xe0- sd t0,-0x10(a0)- ret- ",- sym wasmtime_fiber_start,- );+ fs: [f64; 12],++ s11: *mut u8,+ s10: *mut u8,+ s9: *mut u8,+ s8: *mut u8,+ s7: *mut u8,+ s6: *mut u8,+ s5: *mut u8,+ s4: *mut u8,+ s3: *mut u8,+ s2: *mut u8,+ s1: *mut u8,+ fp: *mut u8,++ ra: *mut u8,++ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }++ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ s1: entry_point as *mut u8,+ s2: entry_arg0,+ fp: top_of_stack,+ ra: wasmtime_fiber_start as *mut u8,+ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }}#[unsafe(naked)]diff --git a/crates/fiber/src/stackswitch/s390x.rs b/crates/fiber/src/stackswitch/s390x.rsindex b284ee2b7b..15cb543b16 100644--- a/crates/fiber/src/stackswitch/s390x.rs+++ b/crates/fiber/src/stackswitch/s390x.rs@@ -45,29 +45,65 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x);}-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(- top_of_stack: *mut u8, // x0- entry_point: extern "C" fn(*mut u8, *mut u8), // x1- entry_arg0: *mut u8, // x2+pub(crate) unsafe fn wasmtime_fiber_init(+ top_of_stack: *mut u8,+ entry_point: extern "C" fn(*mut u8, *mut u8),+ entry_arg0: *mut u8,) {- naked_asm!(- "- larl %r1, {}- stg %r1, -48(%r2) // wasmtime_fiber_start - restored into %r14- stg %r2, -112(%r2) // top_of_stack - restored into %r6- stg %r3, -104(%r2) // entry_point - restored into %r7- stg %r4, -96(%r2) // entry_arg0 - restored into %r8- aghi %r2, -160 // 160 bytes register save area- stg %r2, 120(%r2) // bottom of register save area - restored into %r15-- // `wasmtime_fiber_switch` has a 64 byte stack.- aghi %r2, -64- stg %r2, 208(%r2)- br %r14- ",- sym wasmtime_fiber_start,- );+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ f8: *mut u8,+ f9: *mut u8,+ f10: *mut u8,+ f11: *mut u8,+ f12: *mut u8,+ f13: *mut u8,+ f14: *mut u8,+ f15: *mut u8,++ back_chain: *mut u8,+ compiler_reserved: *mut u8,++ r2: *mut u8,+ r3: *mut u8,+ r4: *mut u8,+ r5: *mut u8,++ r6: *mut u8,+ r7: *mut u8,+ r8: *mut u8,+ r9: *mut u8,+ r10: *mut u8,+ r11: *mut u8,+ r12: *mut u8,+ r13: *mut u8,+ r14: *mut u8,+ r15: *mut u8,++ f0: *mut u8,+ f2: *mut u8,+ f4: *mut u8,+ f6: *mut u8,++ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }++ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ r15: (&raw mut (*initial_stack).back_chain).cast(),+ r14: wasmtime_fiber_start as *mut u8,+ r6: top_of_stack,+ r7: entry_point as *mut u8,+ r8: entry_arg0,++ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }}#[unsafe(naked)]@@ -82,7 +118,7 @@ unsafe extern "C" fn wasmtime_fiber_start() -> ! {// match the frame layout in `wasmtime_fiber_switch`..cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \7, /* the byte length of this expression */ \- 0x7f, 0x90, 0x1, /* DW_OP_breg15 0x90 */ \+ 0x7f, 0xa0, 0x1, /* DW_OP_breg15 0x90 */ \0x06, /* DW_OP_deref */ \0x23, 0xe0, 0x1 /* DW_OP_plus_uconst 0xe0 */diff --git a/crates/fiber/src/stackswitch/x86.rs b/crates/fiber/src/stackswitch/x86.rsindex ec20a3d045..79869e1bc0 100644--- a/crates/fiber/src/stackswitch/x86.rs+++ b/crates/fiber/src/stackswitch/x86.rs@@ -40,49 +40,52 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {)}-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(+pub(crate) unsafe fn wasmtime_fiber_init(top_of_stack: *mut u8,entry_point: extern "C" fn(*mut u8, *mut u8),entry_arg0: *mut u8,) {- naked_asm!(- "- mov eax, 4[esp]-- // move top_of_stack to the 2nd argument- mov -0x0c[eax], eax+ // Our stack from top-to-bottom looks like:+ //+ // * 8 bytes of reserved space per unix.rs (two-pointers space)+ // * 8 bytes of arguments (two arguments wasmtime_fiber_start forwards)+ // * 4 bytes of return address+ // * 16 bytes of saved registers+ //+ // Note that after the return address the stack is conveniently 16-byte+ // aligned as required, so we just leave the arguments on the stack in+ // `wasmtime_fiber_start` and immediately do the call.+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ // state that will get resumed into from a `wasmtime_fiber_switch`+ // starting up this fiber.+ edi: *mut u8,+ esi: *mut u8,+ ebx: *mut u8,+ ebp: *mut u8,+ return_address: *mut u8,- // move entry_arg0 to the 1st argument- mov ecx, 12[esp]- mov -0x10[eax], ecx+ // two arguments to `entry_point`+ arg1: *mut u8,+ arg2: *mut u8,- // Move our start function to the return address which the `ret` in- // `wasmtime_fiber_start` will return to.- lea ecx, {start}- mov -0x14[eax], ecx+ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }- // And move `entry_point` to get loaded into `%ebp` through the context- // switch. This'll get jumped to in `wasmtime_fiber_start`.- mov ecx, 8[esp]- mov -0x18[eax], ecx-- // Our stack from top-to-bottom looks like:- //- // * 8 bytes of reserved space per unix.rs (two-pointers space)- // * 8 bytes of arguments (two arguments wasmtime_fiber_start forwards)- // * 4 bytes of return address- // * 16 bytes of saved registers- //- // Note that after the return address the stack is conveniently 16-byte- // aligned as required, so we just leave the arguments on the stack in- // `wasmtime_fiber_start` and immediately do the call.- lea ecx, -0x24[eax]- mov -0x08[eax], ecx- ret- ",- start = sym wasmtime_fiber_start,- );+ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ ebp: entry_point as *mut u8,+ return_address: wasmtime_fiber_start as *mut u8,+ arg1: entry_arg0,+ arg2: top_of_stack,+ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }}#[unsafe(naked)]diff --git a/crates/fiber/src/stackswitch/x86_64.rs b/crates/fiber/src/stackswitch/x86_64.rsindex 4a826bf9a0..8294073fd1 100644--- a/crates/fiber/src/stackswitch/x86_64.rs+++ b/crates/fiber/src/stackswitch/x86_64.rs@@ -43,38 +43,38 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* r);}-#[unsafe(naked)]-pub(crate) unsafe extern "C" fn wasmtime_fiber_init(- top_of_stack: *mut u8, // rdi- entry_point: extern "C" fn(*mut u8, *mut u8), // rsi- entry_arg0: *mut u8, // rdx+pub(crate) unsafe fn wasmtime_fiber_init(+ top_of_stack: *mut u8,+ entry_point: extern "C" fn(*mut u8, *mut u8),+ entry_arg0: *mut u8,) {- naked_asm!(- "- // Here we're going to set up a stack frame as expected by- // `wasmtime_fiber_switch`. The values we store here will get restored into- // registers by that function and the `wasmtime_fiber_start` function will- // take over and understands which values are in which registers.- //- // The first 16 bytes of stack are reserved for metadata, so we start- // storing values beneath that.- lea rax, {start}[rip]- mov -0x18[rdi], rax- mov -0x20[rdi], rdi // loaded into rbp during switch- mov -0x28[rdi], rsi // loaded into rbx during switch- mov -0x30[rdi], rdx // loaded into r12 during switch+ #[repr(C)]+ #[derive(Default)]+ struct InitialStack {+ r15: *mut u8,+ r14: *mut u8,+ r13: *mut u8,+ r12: *mut u8,+ rbx: *mut u8,+ rbp: *mut u8,+ return_address: *mut u8,- // And then we specify the stack pointer resumption should begin at. Our- // `wasmtime_fiber_switch` function consumes 6 registers plus a return- // pointer, and the top 16 bytes are reserved, so that's:- //- // (6 + 1) * 16 + 16 = 0x48- lea rax, -0x48[rdi]- mov -0x10[rdi], rax- ret- ",- start = sym wasmtime_fiber_start,- );+ // unix.rs reserved space+ last_sp: *mut u8,+ run_result: *mut u8,+ }++ unsafe {+ let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);+ initial_stack.write(InitialStack {+ r12: entry_arg0,+ rbx: entry_point as *mut u8,+ rbp: top_of_stack,+ return_address: wasmtime_fiber_start as *mut u8,+ last_sp: initial_stack.cast(),+ ..InitialStack::default()+ });+ }}// This is a pretty special function that has no real signature. Its use is to