A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Refactor `sync_nostd.rs` silghtly (#11875)
Alex Crichton3c46362efe49
diff --git a/crates/wasmtime/src/sync_nostd.rs b/crates/wasmtime/src/sync_nostd.rsindex 39913bbbd3..e820bfa57d 100644--- a/crates/wasmtime/src/sync_nostd.rs+++ b/crates/wasmtime/src/sync_nostd.rs@@ -22,20 +22,12 @@use core::cell::UnsafeCell;use core::mem::MaybeUninit;use core::ops::{Deref, DerefMut};-use core::sync::atomic::{AtomicU8, AtomicU32, Ordering};--#[cfg(has_custom_sync)]-use crate::runtime::vm::capi::{- wasmtime_sync_lock_acquire, wasmtime_sync_lock_free, wasmtime_sync_lock_release,- wasmtime_sync_rwlock_free, wasmtime_sync_rwlock_read, wasmtime_sync_rwlock_read_release,- wasmtime_sync_rwlock_write, wasmtime_sync_rwlock_write_release,-};+use core::sync::atomic::{AtomicU8, Ordering};pub struct OnceLock<T> {val: UnsafeCell<MaybeUninit<T>>,state: AtomicU8,- #[cfg(has_custom_sync)]- mutex: RawMutex,+ mutex: raw::Mutex,}unsafe impl<T: Send> Send for OnceLock<T> {}@@ -50,8 +42,7 @@ impl<T> OnceLock<T> {OnceLock {state: AtomicU8::new(UNINITIALIZED),val: UnsafeCell::new(MaybeUninit::uninit()),- #[cfg(has_custom_sync)]- mutex: RawMutex::new(),+ mutex: raw::Mutex::new(),}}@@ -79,38 +70,8 @@ impl<T> OnceLock<T> {}#[cold]- #[cfg(not(has_custom_sync))]fn try_init<E>(&self, f: impl FnOnce() -> Result<T, E>) -> Result<&T, E> {- match self.state.compare_exchange(- UNINITIALIZED,- INITIALIZING,- Ordering::Acquire,- Ordering::Acquire,- ) {- Ok(UNINITIALIZED) => match f() {- Ok(val) => {- let ret = unsafe { &*(*self.val.get()).write(val) };- let prev = self.state.swap(INITIALIZED, Ordering::Release);- assert_eq!(prev, INITIALIZING);- Ok(ret)- }- Err(e) => match self.state.swap(UNINITIALIZED, Ordering::Release) {- INITIALIZING => Err(e),- _ => unreachable!(),- },- },- Err(INITIALIZING) => panic!(- "concurrent initialization only allowed with `std` or `custom-sync-primitives` features"- ),- Err(INITIALIZED) => Ok(self.get().unwrap()),- _ => unreachable!(),- }- }-- #[cold]- #[cfg(has_custom_sync)]- fn try_init<E>(&self, f: impl FnOnce() -> Result<T, E>) -> Result<&T, E> {- let _guard = OnceLockGuard::acquire(self.mutex.storage.get());+ let _guard = OnceLockGuard::new(&self.mutex);// Check state again under lockmatch self.state.load(Ordering::Acquire) {@@ -153,54 +114,22 @@ impl<T> Default for OnceLock<T> {}}-#[cfg(has_custom_sync)]-#[derive(Debug)]-struct RawMutex {- storage: UnsafeCell<usize>,-}--#[cfg(has_custom_sync)]-impl RawMutex {- const fn new() -> RawMutex {- RawMutex {- storage: UnsafeCell::new(0),- }- }-}--#[cfg(has_custom_sync)]-impl Drop for RawMutex {- fn drop(&mut self) {- // SAFETY: We have exclusive access via &mut self- // The host implementation handles the case where the lock was never used (still zero)- unsafe {- wasmtime_sync_lock_free(self.storage.get());- }- }-}--#[cfg(has_custom_sync)]-struct OnceLockGuard {- lock: *mut usize,+struct OnceLockGuard<'a> {+ lock: &'a raw::Mutex,}-#[cfg(has_custom_sync)]-impl OnceLockGuard {- fn acquire(lock: *mut usize) -> OnceLockGuard {- // SAFETY: The host implementation handles lazy initialization in acquire- unsafe {- wasmtime_sync_lock_acquire(lock);- }+impl<'a> OnceLockGuard<'a> {+ fn new(lock: &'a raw::Mutex) -> OnceLockGuard<'a> {+ lock.lock();OnceLockGuard { lock }}}-#[cfg(has_custom_sync)]-impl Drop for OnceLockGuard {+impl Drop for OnceLockGuard<'_> {fn drop(&mut self) {// SAFETY: We acquired the lock in OnceLockGuard::acquireunsafe {- wasmtime_sync_lock_release(self.lock);+ self.lock.unlock();}}}@@ -208,10 +137,7 @@ impl Drop for OnceLockGuard {#[derive(Debug)]pub struct RwLock<T> {val: UnsafeCell<T>,- #[cfg(not(has_custom_sync))]- state: AtomicU32,- #[cfg(has_custom_sync)]- lock: RawRwLock,+ lock: raw::RwLock,}unsafe impl<T: Send> Send for RwLock<T> {}@@ -221,64 +147,17 @@ impl<T> RwLock<T> {pub const fn new(val: T) -> RwLock<T> {RwLock {val: UnsafeCell::new(val),- #[cfg(not(has_custom_sync))]- state: AtomicU32::new(0),- #[cfg(has_custom_sync)]- lock: RawRwLock::new(),+ lock: raw::RwLock::new(),}}- #[cfg(not(has_custom_sync))]pub fn read(&self) -> impl Deref<Target = T> + '_ {- const READER_LIMIT: u32 = u32::MAX / 2;- match self- .state- .fetch_update(Ordering::Acquire, Ordering::Acquire, |x| match x {- u32::MAX => None,- n => {- let next = n + 1;- if next < READER_LIMIT {- Some(next)- } else {- None- }- }- }) {- Ok(_) => RwLockReadGuard { lock: self },- Err(_) => panic!(- "concurrent read request while locked for writing, must use `std` or `custom-sync-primitives` features to avoid panic"- ),- }- }-- #[cfg(has_custom_sync)]- pub fn read(&self) -> impl Deref<Target = T> + '_ {- // SAFETY: The host implementation handles lazy initialization in read- unsafe {- wasmtime_sync_rwlock_read(self.lock.storage.get());- }+ self.lock.read();RwLockReadGuard { lock: self }}- #[cfg(not(has_custom_sync))]- pub fn write(&self) -> impl DerefMut<Target = T> + '_ {- match self- .state- .compare_exchange(0, u32::MAX, Ordering::Acquire, Ordering::Relaxed)- {- Ok(0) => RwLockWriteGuard { lock: self },- _ => panic!(- "concurrent write request, must use `std` or `custom-sync-primitives` features to avoid panicking"- ),- }- }-- #[cfg(has_custom_sync)]pub fn write(&self) -> impl DerefMut<Target = T> + '_ {- // SAFETY: The host implementation handles lazy initialization in write- unsafe {- wasmtime_sync_rwlock_write(self.lock.storage.get());- }+ self.lock.write();RwLockWriteGuard { lock: self }}}@@ -289,32 +168,6 @@ impl<T: Default> Default for RwLock<T> {}}-#[cfg(has_custom_sync)]-#[derive(Debug)]-struct RawRwLock {- storage: UnsafeCell<usize>,-}--#[cfg(has_custom_sync)]-impl RawRwLock {- const fn new() -> RawRwLock {- RawRwLock {- storage: UnsafeCell::new(0),- }- }-}--#[cfg(has_custom_sync)]-impl Drop for RawRwLock {- fn drop(&mut self) {- // SAFETY: We have exclusive access via &mut self- // The host implementation handles the case where the lock was never used (still zero)- unsafe {- wasmtime_sync_rwlock_free(self.storage.get());- }- }-}-struct RwLockReadGuard<'a, T> {lock: &'a RwLock<T>,}@@ -328,19 +181,12 @@ impl<T> Deref for RwLockReadGuard<'_, T> {}}-#[cfg(not(has_custom_sync))]impl<T> Drop for RwLockReadGuard<'_, T> {fn drop(&mut self) {- self.lock.state.fetch_sub(1, Ordering::Release);- }-}--#[cfg(has_custom_sync)]-impl<T> Drop for RwLockReadGuard<'_, T> {- fn drop(&mut self) {- // SAFETY: We acquired the read lock in RwLock::read()+ // SAFETY: This type represents a safe read lock being held, so it's+ // safe to perform the unlock here at the end.unsafe {- wasmtime_sync_rwlock_read_release(self.lock.lock.storage.get());+ self.lock.lock.read_unlock();}}}@@ -365,22 +211,190 @@ impl<T> DerefMut for RwLockWriteGuard<'_, T> {}}-#[cfg(not(has_custom_sync))]impl<T> Drop for RwLockWriteGuard<'_, T> {fn drop(&mut self) {- match self.lock.state.swap(0, Ordering::Release) {- u32::MAX => {}- _ => unreachable!(),+ // SAFETY: This type represents a safe write lock being held, so it's+ // safe to perform the unlock here at the end.+ unsafe {+ self.lock.lock.write_unlock();+ }+ }+}++#[cfg(not(has_custom_sync))]+use panic_on_contention as raw;+#[cfg(not(has_custom_sync))]+mod panic_on_contention {+ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};++ #[derive(Debug)]+ pub struct Mutex {+ locked: AtomicBool,+ }++ impl Mutex {+ pub const fn new() -> Mutex {+ Mutex {+ locked: AtomicBool::new(false),+ }+ }++ pub fn lock(&self) {+ if self.locked.swap(true, Ordering::Acquire) {+ panic!(+ "concurrent lock request, must use `std` or `custom-sync-primitives` features to avoid panicking"+ );+ }+ }++ pub unsafe fn unlock(&self) {+ self.locked.store(false, Ordering::Release);+ }+ }++ #[derive(Debug)]+ pub struct RwLock {+ state: AtomicU32,+ }++ impl RwLock {+ pub const fn new() -> RwLock {+ RwLock {+ state: AtomicU32::new(0),+ }+ }++ pub fn read(&self) {+ const READER_LIMIT: u32 = u32::MAX / 2;+ match self+ .state+ .fetch_update(Ordering::Acquire, Ordering::Acquire, |x| match x {+ u32::MAX => None,+ n => {+ let next = n + 1;+ if next < READER_LIMIT {+ Some(next)+ } else {+ None+ }+ }+ }) {+ Ok(_) => {}+ Err(_) => panic!(+ "concurrent read request while locked for writing, must use `std` or `custom-sync-primitives` features to avoid panic"+ ),+ }+ }++ pub unsafe fn read_unlock(&self) {+ self.state.fetch_sub(1, Ordering::Release);+ }++ pub fn write(&self) {+ match self+ .state+ .compare_exchange(0, u32::MAX, Ordering::Acquire, Ordering::Relaxed)+ {+ Ok(0) => {}+ _ => panic!(+ "concurrent write request, must use `std` or `custom-sync-primitives` features to avoid panicking"+ ),+ }+ }++ pub unsafe fn write_unlock(&self) {+ match self.state.swap(0, Ordering::Release) {+ u32::MAX => {}+ _ => unreachable!(),+ }}}}#[cfg(has_custom_sync)]-impl<T> Drop for RwLockWriteGuard<'_, T> {- fn drop(&mut self) {- // SAFETY: We acquired the write lock in RwLock::write()- unsafe {- wasmtime_sync_rwlock_write_release(self.lock.lock.storage.get());+use custom_capi as raw;+#[cfg(has_custom_sync)]+mod custom_capi {+ use crate::runtime::vm::capi;+ use core::cell::UnsafeCell;++ #[derive(Debug)]+ pub struct Mutex {+ storage: UnsafeCell<usize>,+ }++ impl Mutex {+ pub const fn new() -> Mutex {+ Mutex {+ storage: UnsafeCell::new(0),+ }+ }++ pub fn lock(&self) {+ unsafe {+ capi::wasmtime_sync_lock_acquire(self.storage.get());+ }+ }++ pub unsafe fn unlock(&self) {+ unsafe {+ capi::wasmtime_sync_lock_release(self.storage.get());+ }+ }+ }++ impl Drop for Mutex {+ fn drop(&mut self) {+ // SAFETY: We have exclusive access via &mut self+ // The host implementation handles the case where the lock was never used (still zero)+ unsafe {+ capi::wasmtime_sync_lock_free(self.storage.get());+ }+ }+ }++ #[derive(Debug)]+ pub struct RwLock {+ storage: UnsafeCell<usize>,+ }++ impl RwLock {+ pub const fn new() -> RwLock {+ RwLock {+ storage: UnsafeCell::new(0),+ }+ }++ pub fn read(&self) {+ unsafe {+ capi::wasmtime_sync_rwlock_read(self.storage.get());+ }+ }++ pub unsafe fn read_unlock(&self) {+ unsafe {+ capi::wasmtime_sync_rwlock_read_release(self.storage.get());+ }+ }++ pub fn write(&self) {+ unsafe {+ capi::wasmtime_sync_rwlock_write(self.storage.get());+ }+ }++ pub unsafe fn write_unlock(&self) {+ unsafe {+ capi::wasmtime_sync_rwlock_write_release(self.storage.get());+ }+ }+ }++ impl Drop for RwLock {+ fn drop(&mut self) {+ unsafe {+ capi::wasmtime_sync_rwlock_free(self.storage.get());+ }}}}diff --git a/examples/min-platform/embedding/wasmtime-platform.c b/examples/min-platform/embedding/wasmtime-platform.cindex ee174dc12f..007abe902d 100644--- a/examples/min-platform/embedding/wasmtime-platform.c+++ b/examples/min-platform/embedding/wasmtime-platform.c@@ -174,7 +174,7 @@ void wasmtime_sync_lock_free(uintptr_t *lock) {}}-void wasmtime_sync_lock_acquire(uintptr_t *lock) {+static pthread_mutex_t *mutex_lazy_init(uintptr_t *lock) {pthread_mutex_t *mutex = (pthread_mutex_t *)*lock;if (mutex == NULL) {pthread_mutex_t *new_mutex = malloc(sizeof(pthread_mutex_t));@@ -190,11 +190,16 @@ void wasmtime_sync_lock_acquire(uintptr_t *lock) {}mutex = (pthread_mutex_t *)*lock;}+ return mutex;+}++void wasmtime_sync_lock_acquire(uintptr_t *lock) {+ pthread_mutex_t *mutex = mutex_lazy_init(lock);pthread_mutex_lock(mutex);}void wasmtime_sync_lock_release(uintptr_t *lock) {- pthread_mutex_t *mutex = (pthread_mutex_t *)*lock;+ pthread_mutex_t *mutex = mutex_lazy_init(lock);pthread_mutex_unlock(mutex);}@@ -207,7 +212,7 @@ void wasmtime_sync_rwlock_free(uintptr_t *lock) {}}-void wasmtime_sync_rwlock_read(uintptr_t *lock) {+static pthread_rwlock_t *rwlock_lazy_init(uintptr_t *lock) {pthread_rwlock_t *rwlock = (pthread_rwlock_t *)*lock;if (rwlock == NULL) {pthread_rwlock_t *new_rwlock = malloc(sizeof(pthread_rwlock_t));@@ -218,41 +223,32 @@ void wasmtime_sync_rwlock_read(uintptr_t *lock) {// Atomically set lock only if it's still NULLif (!__sync_bool_compare_and_swap(lock, 0, (uintptr_t)new_rwlock)) {+ // Another thread won the race, discard our allocationpthread_rwlock_destroy(new_rwlock);free(new_rwlock);}rwlock = (pthread_rwlock_t *)*lock;}+ return rwlock;+}++void wasmtime_sync_rwlock_read(uintptr_t *lock) {+ pthread_rwlock_t *rwlock = rwlock_lazy_init(lock);pthread_rwlock_rdlock(rwlock);}void wasmtime_sync_rwlock_read_release(uintptr_t *lock) {- pthread_rwlock_t *rwlock = (pthread_rwlock_t *)*lock;+ pthread_rwlock_t *rwlock = rwlock_lazy_init(lock);pthread_rwlock_unlock(rwlock);}void wasmtime_sync_rwlock_write(uintptr_t *lock) {- pthread_rwlock_t *rwlock = (pthread_rwlock_t *)*lock;- if (rwlock == NULL) {- pthread_rwlock_t *new_rwlock = malloc(sizeof(pthread_rwlock_t));- if (new_rwlock == NULL) {- abort();- }- pthread_rwlock_init(new_rwlock, NULL);-- // Atomically set lock only if it's still NULL- if (!__sync_bool_compare_and_swap(lock, 0, (uintptr_t)new_rwlock)) {- // Another thread won the race, discard our allocation- pthread_rwlock_destroy(new_rwlock);- free(new_rwlock);- }- rwlock = (pthread_rwlock_t *)*lock;- }+ pthread_rwlock_t *rwlock = rwlock_lazy_init(lock);pthread_rwlock_wrlock(rwlock);}void wasmtime_sync_rwlock_write_release(uintptr_t *lock) {- pthread_rwlock_t *rwlock = (pthread_rwlock_t *)*lock;+ pthread_rwlock_t *rwlock = rwlock_lazy_init(lock);pthread_rwlock_unlock(rwlock);}