The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
//! Epoch fencing for writes to leased storage scopes.
//!
//! A lease ([`super::lease`]) says which node owns a key; it does not stop a
//! node that lost the key from writing. A paused or partitioned holder keeps
//! running its in-process work until its heartbeat notices the loss, and in
//! that window its writes would land on top of the new holder's.
//!
//! A [`LeaseFence`] closes that window for the storage ports. It is one
//! grant (key, node, epoch) plus the scopes that grant protects. While a
//! fence is registered in a [`FenceRegistry`], every write a
//! [`FencedBackend`](super::fenced_backend::FencedBackend) sends to one of
//! those scopes is checked twice.
//!
//! First, on this node, [`LeaseFence::check`] refuses the write with a typed
//! [`FenceError`] when:
//!
//! 1. the fence is latched ([`LeaseFence::fence`]: the heartbeat lost the
//! lease, or an earlier check saw it superseded);
//! 2. the grant ran out by this node's clock, less a skew margin
//! ([`LeaseFence::check_local`], no I/O); or
//! 3. the stored lease record no longer names this node at this epoch, or
//! has expired, or cannot be read (fail closed).
//!
//! Then the storage driver enforces it. The write goes through
//! [`StorageBackend::for_scope_fenced`](tinystoragedrivers::StorageBackend::for_scope_fenced)
//! with [`LeaseFence::driver_fence`], so the driver re-reads the lease record
//! in the same atomic step as the write and refuses it
//! ([`ErrorKind::Fenced`](tinystoragedrivers::ErrorKind::Fenced)) unless the
//! record still names this node at this epoch, unreleased. A holder that
//! passes the first check and is then paused past a takeover cannot land
//! the write: the driver refusal latches the fence and surfaces as
//! [`FenceError::Superseded`].
//!
//! # Where the race is still open
//!
//! The driver step needs [`Capability::Fencing`](tinystoragedrivers::Capability::Fencing)
//! and a lease record in the storage ports. Without either, the write runs
//! after the first check alone, and a holder paused for longer than its
//! remaining grant *after* a successful check can still land one write after
//! a takeover:
//!
//! - MongoDB without transactions (a standalone server) has no fencing, and
//! MongoDB blob writes cannot be fenced (GridFS cannot join a
//! transaction);
//! - writes to a named database ([`StorageBackend::database`](tinystoragedrivers::StorageBackend::database))
//! cannot see the lease record, which lives in the default database;
//! - file leases (hosts without a backend) keep no record in the ports, but
//! they cannot be lost while the process lives either.
//!
//! The skew margin makes that race need a pause longer than the margin
//! after a successful check, rather than any pause at all.
//!
//! Scopes nobody registered pass through untouched, so a backend with an
//! empty registry (every single-user host) behaves exactly as before.
use std::any::Any;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak};
use tinystoragedrivers_types::WriteCondition;
use super::lease::LeaseStore;
use super::StorageError;
/// Why a fenced write was refused.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum FenceError {
/// This node lost the lease (latched).
#[error("lease fence: key lost at epoch {epoch}; this node no longer holds it")]
Fenced {
/// The epoch this node held.
epoch: u64,
},
/// The grant ran out by this node's clock (less the skew margin) before
/// it was renewed.
#[error("lease fence: grant at epoch {epoch} expired before renewal")]
Expired {
/// The epoch this node holds.
epoch: u64,
},
/// The stored record names another holder or a newer epoch.
#[error("lease fence: epoch {held} superseded (stored epoch {stored:?})")]
Superseded {
/// The epoch this node held.
held: u64,
/// The stored record's epoch, if there is a record.
stored: Option<u64>,
},
/// The lease record could not be read, so the write is refused.
#[error("lease fence: could not verify the lease: {0}")]
Unverified(String),
}
impl FenceError {
/// This refusal as a port error ([`ErrorKind::Backend`](tinystoragedrivers::ErrorKind),
/// not retryable, so compare-and-swap loops do not spin on it) carrying
/// `self` as its source; [`fence_error`] recovers it.
pub fn into_storage(self) -> StorageError {
StorageError::backend(self.to_string()).with_source(self)
}
}
/// The [`FenceError`] behind `error`, when a fence refused the write.
pub fn fence_error(error: &StorageError) -> Option<&FenceError> {
std::error::Error::source(error).and_then(|source| source.downcast_ref::<FenceError>())
}
/// Which scopes a fence protects.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ScopeMatch {
/// Exactly this scope.
Exact(String),
/// Every scope starting with this prefix.
Prefix(String),
}
impl ScopeMatch {
fn covers(&self, scope: &str) -> bool {
match self {
Self::Exact(exact) => scope == exact,
Self::Prefix(prefix) => scope.starts_with(prefix.as_str()),
}
}
}
/// One node's grant on a key, guarding the scopes it protects. See the
/// module docs.
pub struct LeaseFence {
key: String,
node: String,
epoch: u64,
expires_at_ms: AtomicU64,
margin_ms: u64,
scopes: Vec<ScopeMatch>,
/// Weak: a fence must not keep its node's lease store (and so a file
/// lock) alive after the host that owns it is gone. A gone store fails
/// the check closed.
leases: Weak<dyn LeaseStore>,
latch: tokio::sync::watch::Sender<bool>,
/// While the fence is latched, it stays registered (refusing writes)
/// until this anchor is gone: the work that could still write is what
/// holds it.
anchor: Mutex<Option<Weak<dyn Any + Send + Sync>>>,
}
impl std::fmt::Debug for LeaseFence {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LeaseFence")
.field("node", &self.node)
.field("epoch", &self.epoch)
.field("fenced", &self.is_fenced())
.finish_non_exhaustive()
}
}
impl LeaseFence {
/// A fence for `node`'s grant on `key` at `epoch`, valid until
/// `expires_at_ms` less `margin_ms`, verified against `leases`.
pub fn new(
leases: Arc<dyn LeaseStore>,
key: impl Into<String>,
node: impl Into<String>,
epoch: u64,
expires_at_ms: u64,
margin_ms: u64,
scopes: Vec<ScopeMatch>,
) -> Self {
Self {
key: key.into(),
node: node.into(),
epoch,
expires_at_ms: AtomicU64::new(expires_at_ms),
margin_ms,
scopes,
leases: Arc::downgrade(&leases),
latch: tokio::sync::watch::channel(false).0,
anchor: Mutex::new(None),
}
}
/// The leased key.
pub fn key(&self) -> &str {
&self.key
}
/// The epoch this fence guards with.
pub fn epoch(&self) -> u64 {
self.epoch
}
/// The node holding the grant.
pub fn node(&self) -> &str {
&self.node
}
/// The condition a storage driver enforces for this grant: the lease record
/// (where the lease store keeps it, [`LeaseStore::write_condition`]) must
/// still name this node at this epoch, unreleased. `None` when the lease
/// store keeps no record in the storage ports (file leases) or is gone;
/// then only [`Self::check`] guards the write.
pub fn write_condition(&self) -> Option<WriteCondition> {
self.leases
.upgrade()?
.write_condition(&self.key, &self.node, self.epoch)
}
/// The refusal for a write the storage driver fenced off
/// ([`ErrorKind::Fenced`](tinystoragedrivers::ErrorKind::Fenced)): the
/// lease record moved on between [`Self::check`] and the write. Re-reads
/// the record for the stored epoch; the fence is latched either way.
pub async fn refused_by_driver(&self, now_ms: u64) -> FenceError {
match self.check(now_ms).await {
// `check` latched it if the record moved on.
Err(error) => {
self.fence();
error
}
Ok(()) => {
// The driver could not match a record this node can still
// read as its own: a guard the driver cannot see. Fail closed.
tracing::error!(
target: "openhuman::storage::fence",
epoch = self.epoch,
"[storage][fence] driver refused a write the lease record still admits; latching"
);
self.fence();
FenceError::Superseded {
held: self.epoch,
stored: None,
}
}
}
}
/// Whether this fence guards `scope`.
pub fn covers(&self, scope: &str) -> bool {
self.scopes.iter().any(|m| m.covers(scope))
}
/// Records a renewal: the grant now lasts until `expires_at_ms`.
pub fn renewed(&self, expires_at_ms: u64) {
self.expires_at_ms.store(expires_at_ms, Ordering::SeqCst);
}
/// Latches the fence: every later check and write is refused. Idempotent.
pub fn fence(&self) {
if !self.latch.send_replace(true) {
tracing::warn!(
target: "openhuman::storage::fence",
epoch = self.epoch,
"[storage][fence] latched; writes to the fenced scopes are refused"
);
}
}
/// Whether the fence is latched.
pub fn is_fenced(&self) -> bool {
*self.latch.borrow()
}
/// Resolves once the fence latches (at once when it already has), so
/// in-process work can stop promptly.
pub async fn fenced(&self) {
let mut rx = self.latch.subscribe();
// `wait_for` only fails when the sender is gone, and `self` owns it.
let _ = rx.wait_for(|fenced| *fenced).await;
}
/// Keeps this fence registered after it latches only for as long as
/// `anchor` is alive (see [`FenceRegistry::lookup`]). Without an anchor
/// a latched fence stays until it is retired or replaced.
pub fn anchor_to(&self, anchor: Weak<dyn Any + Send + Sync>) {
*self.anchor.lock().unwrap_or_else(PoisonError::into_inner) = Some(anchor);
}
fn anchor_alive(&self) -> bool {
self.anchor
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
.is_none_or(|anchor| anchor.strong_count() > 0)
}
fn has_dead_anchor(&self) -> bool {
self.anchor
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
.is_some_and(|anchor| anchor.strong_count() == 0)
}
/// The checks that need no I/O: the latch, and the grant's expiry by
/// this node's clock less the margin.
///
/// # Errors
///
/// [`FenceError::Fenced`] or [`FenceError::Expired`].
pub fn check_local(&self, now_ms: u64) -> Result<(), FenceError> {
if self.is_fenced() {
return Err(FenceError::Fenced { epoch: self.epoch });
}
let deadline = self
.expires_at_ms
.load(Ordering::SeqCst)
.saturating_sub(self.margin_ms);
if now_ms >= deadline {
return Err(FenceError::Expired { epoch: self.epoch });
}
Ok(())
}
/// [`Self::check_local`], then the stored record: it must still name
/// this node at this epoch, unreleased and unexpired. A record that moved
/// on latches the fence.
///
/// # Errors
///
/// Any [`FenceError`]; a lease store failure is
/// [`FenceError::Unverified`] (the write is refused, not risked).
pub async fn check(&self, now_ms: u64) -> Result<(), FenceError> {
self.check_local(now_ms)?;
let Some(leases) = self.leases.upgrade() else {
return Err(FenceError::Unverified("the lease store is gone".into()));
};
let record = leases
.holder(&self.key)
.await
.map_err(|error| FenceError::Unverified(error.to_string()))?;
match record {
Some(record)
if record.owner == self.node && record.epoch == self.epoch && !record.released =>
{
if now_ms >= record.expires_at_ms {
return Err(FenceError::Expired { epoch: self.epoch });
}
Ok(())
}
other => {
let stored = other.map(|record| record.epoch);
tracing::warn!(
target: "openhuman::storage::fence",
held = self.epoch,
stored = ?stored,
"[storage][fence] lease superseded; refusing the write"
);
self.fence();
Err(FenceError::Superseded {
held: self.epoch,
stored,
})
}
}
}
}
/// The fences a process holds. The process has one ([`registry`]); tests
/// make their own so they never see each other's.
pub struct FenceRegistry {
fences: Mutex<Vec<Arc<LeaseFence>>>,
clock: Box<dyn Fn() -> u64 + Send + Sync>,
}
impl std::fmt::Debug for FenceRegistry {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FenceRegistry")
.field("fences", &self.lock().len())
.finish_non_exhaustive()
}
}
impl FenceRegistry {
/// A registry reading `clock` (milliseconds since the epoch) for checks.
pub fn new(clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
Self {
fences: Mutex::new(Vec::new()),
clock: Box::new(clock),
}
}
/// The registry's clock.
pub fn now_ms(&self) -> u64 {
(self.clock)()
}
fn lock(&self) -> std::sync::MutexGuard<'_, Vec<Arc<LeaseFence>>> {
self.fences.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Registers `fence`, replacing any fence on the same key unless that
/// one is from a newer epoch. Fences with dead anchors are pruned first;
/// newer live or unanchored fences retain their epoch protection.
pub fn register(&self, fence: Arc<LeaseFence>) {
let mut fences = self.lock();
// A dead anchor means no work can still write through the old grant.
// Prune it before comparing epochs because a fresh lease store may
// restart its epoch counter while a stale registry entry remains.
fences.retain(|held| !held.has_dead_anchor());
// A fence from a newer epoch is never displaced by an older grant's.
if fences
.iter()
.any(|held| held.key == fence.key && held.epoch > fence.epoch)
{
tracing::debug!(
target: "openhuman::storage::fence",
epoch = fence.epoch,
"[storage][fence] stale registration ignored"
);
return;
}
fences.retain(|held| held.key != fence.key);
tracing::debug!(
target: "openhuman::storage::fence",
epoch = fence.epoch,
"[storage][fence] registered"
);
fences.push(fence);
}
/// Drops the fence on `key` at `epoch` (a clean release); a fence
/// registered since under another epoch stays. Cached handles retain the
/// old fence, so latch it before removing it from the registry.
pub fn retire(&self, key: &str, epoch: u64) {
let mut fences = self.lock();
fences.retain(|held| {
if held.key == key && held.epoch == epoch {
held.fence();
false
} else {
true
}
});
}
/// The fence guarding `scope`, if any. A latched fence whose anchor is
/// gone (nothing that could still write remains) is pruned here.
pub fn lookup(&self, scope: &str) -> Option<Arc<LeaseFence>> {
let mut fences = self.lock();
fences.retain(|held| !held.is_fenced() || held.anchor_alive());
fences.iter().find(|held| held.covers(scope)).cloned()
}
/// Refuses a write to `scope` unless its fence (when it has one) passes
/// [`LeaseFence::check`].
///
/// # Errors
///
/// The fence's refusal, as a port error ([`FenceError::into_storage`]).
pub async fn guard(&self, scope: &str, op: &'static str) -> Result<(), StorageError> {
self.admit(scope, op).await.map(|_| ())
}
/// [`Self::guard`], handing back the fence that admitted the write (if
/// any) so the caller can have the driver enforce it too.
///
/// # Errors
///
/// The fence's refusal, as a port error ([`FenceError::into_storage`]).
pub async fn admit(
&self,
scope: &str,
op: &'static str,
) -> Result<Option<Arc<LeaseFence>>, StorageError> {
let Some(fence) = self.lookup(scope) else {
return Ok(None);
};
match fence.check(self.now_ms()).await {
Ok(()) => Ok(Some(fence)),
Err(error) => {
tracing::warn!(
target: "openhuman::storage::fence",
op,
epoch = fence.epoch,
"[storage][fence] refused a write: {error}"
);
Err(error.into_storage())
}
}
}
/// The port error for a write to `fence`'s scopes that the storage
/// driver refused ([`LeaseFence::refused_by_driver`]).
pub async fn driver_refused(&self, fence: &LeaseFence, op: &'static str) -> StorageError {
let error = fence.refused_by_driver(self.now_ms()).await;
tracing::warn!(
target: "openhuman::storage::fence",
op,
epoch = fence.epoch,
"[storage][fence] the driver refused a write the host check admitted: {error}"
);
error.into_storage()
}
}
/// Milliseconds since the Unix epoch.
pub fn system_now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
.unwrap_or_default()
}
static REGISTRY: LazyLock<Arc<FenceRegistry>> =
LazyLock::new(|| Arc::new(FenceRegistry::new(system_now_ms)));
/// The process's fence registry, which [`super::open`]'s backends consult.
pub fn registry() -> Arc<FenceRegistry> {
Arc::clone(®ISTRY)
}
#[cfg(test)]
#[path = "fence_tests.rs"]
mod tests;