The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
//! Exclusive, expiring ownership of a named key: the lease.
//!
//! A lease says which node owns a key (in SaaS mode: which core process
//! serves a profile) and until when. Two implementations share the
//! [`LeaseStore`] contract:
//!
//! - [`DocumentLeases`] keeps one host-owned record per key in
//! [`LEASE_COLLECTION`] under [`CLUSTER_SCOPE`], using versioned document
//! writes as a compare-and-swap so it is correct across processes on a
//! driver with cross-process CAS ([`super::driver_has_cross_process_cas`]).
//! - [`LocalLeases`] is for hosts without a backend: an exclusive `flock` on
//! `<root>/<sha256(key) hex>/.lease`, which the OS drops when the holder dies.
//!
//! No wall clock is read here: every operation takes `now_ms`, so behaviour
//! is a function of its inputs and the tests are deterministic.
//!
//! # Semantics
//!
//! - **acquire** takes the key when it has no record, its record is
//! released, expired (`now_ms >= expires_at_ms`), or already this node's.
//! Otherwise it fails with [`LeaseError::Held`] carrying the live record.
//! A fresh key starts at epoch 1; every acquisition other than a
//! re-entrant one (this same store instance already holds that epoch)
//! writes `epoch + 1`, so epochs only grow. An expired takeover does not
//! stop the previous holder: resources a lease protects must reject writes
//! carrying an epoch below the current one, and a holder must stop work
//! before `expires_at_ms` minus the maximum clock skew between nodes (each
//! node supplies its own `now_ms`). This module does not check epochs at
//! the protected resource; [`super::fence`] does, for storage-port writes. An epoch at `u64::MAX` cannot be advanced, so a
//! takeover of it fails with a storage error instead of repeating it.
//! - **renew** extends a grant while it is the latest one its holder was
//! issued at that epoch and the stored record still names its holder and
//! epoch. Any takeover, release or newer grant since makes it
//! [`LeaseError::Lost`].
//! - **release** marks the record `released = true` and keeps it, so the next
//! acquirer can tell a clean hand-over from a crash.
//!
//! # `previous_unclean`
//!
//! A grant's `previous_unclean` is `true` exactly when the acquisition
//! replaced a record that was **not released** and that this store instance
//! did not itself hold. That covers a different owner whose lease expired
//! (it stopped renewing: crashed, partitioned or hung) and this node's own
//! id left over from an earlier process (it crashed before releasing). A
//! fresh key, a released record and a re-entrant acquire are clean. The
//! profile host runs workspace recovery when it is set.
//!
//! Node ids must be unique among live processes: a record carrying this
//! node's id but not held by this instance is treated as a dead previous
//! incarnation and taken over at once.
use std::path::PathBuf;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use tinystoragedrivers::Version;
use super::StorageError;
pub use super::lease_documents::{DocumentLeases, LEASE_OWNER};
pub use super::lease_local::LocalLeases;
/// The storage scope lease records live under, shared by every node.
pub const CLUSTER_SCOPE: &str = "cluster";
/// The host-owned collection for process-placement lease records.
pub const LEASE_COLLECTION: &str = "openhuman.leases";
/// The longest key a lease accepts, in bytes.
pub const MAX_KEY_LEN: usize = 200;
/// The stored state of one key's lease.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LeaseRecord {
/// The node that holds (or last held) the key.
pub owner: String,
/// Where that node can be reached, for redirecting callers.
pub endpoint: Option<String>,
/// Bumped on every acquisition that is not re-entrant.
pub epoch: u64,
/// The lease is void from this instant (milliseconds) on.
pub expires_at_ms: u64,
/// The holder let go cleanly.
pub released: bool,
}
impl LeaseRecord {
/// Whether the record still excludes other nodes at `now_ms`.
#[must_use]
pub fn is_live(&self, now_ms: u64) -> bool {
!self.released && now_ms < self.expires_at_ms
}
/// Milliseconds until the record stops excluding others (0 when it
/// already does not).
#[must_use]
pub fn retry_after_ms(&self, now_ms: u64) -> u64 {
if self.released {
0
} else {
self.expires_at_ms.saturating_sub(now_ms)
}
}
}
/// Proof that the caller holds a key, as of its last acquire or renew.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LeaseGrant {
/// The leased key.
pub key: String,
/// The epoch this grant fences with.
pub epoch: u64,
/// Which grant of this epoch it is: every acquire or renew by the holder
/// issues the next one, and only the latest can renew or release.
pub version: Version,
/// The grant is void from this instant (milliseconds) on unless renewed.
pub expires_at_ms: u64,
/// The acquisition replaced a record nobody released (see the module
/// docs): the previous holder's work may be half done.
pub previous_unclean: bool,
}
/// Why a lease operation failed.
#[derive(Debug, thiserror::Error)]
pub enum LeaseError {
/// Another node holds the key.
#[error("lease held by {} (epoch {})", .0.owner, .0.epoch)]
Held(LeaseRecord),
/// The grant is no longer the current one: the key was taken over or
/// released since.
#[error("lease lost")]
Lost,
/// The backend failed, or the key is invalid.
#[error("lease storage: {0}")]
Storage(#[from] StorageError),
}
/// Exclusive, expiring ownership of named keys. See the module docs.
#[async_trait]
pub trait LeaseStore: Send + Sync {
/// Takes `key` for this node.
///
/// # Errors
///
/// [`LeaseError::Held`] when another node's record is live, or
/// [`LeaseError::Storage`].
async fn acquire(&self, key: &str, now_ms: u64) -> Result<LeaseGrant, LeaseError>;
/// Extends `grant` to `now_ms` plus the store's TTL.
///
/// # Errors
///
/// [`LeaseError::Lost`] when the record changed since `grant`, or
/// [`LeaseError::Storage`].
async fn renew(&self, grant: &LeaseGrant, now_ms: u64) -> Result<LeaseGrant, LeaseError>;
/// Lets go of `grant`, leaving a released record behind.
///
/// # Errors
///
/// [`LeaseError::Lost`] when the record changed since `grant`, or
/// [`LeaseError::Storage`].
async fn release(&self, grant: LeaseGrant) -> Result<(), LeaseError>;
/// The stored record for `key`, live or not; check
/// [`LeaseRecord::is_live`].
///
/// # Errors
///
/// [`LeaseError::Storage`].
async fn holder(&self, key: &str) -> Result<Option<LeaseRecord>, LeaseError>;
/// Reads the stored holder without declaring schema or creating local
/// lease files. Implementations that cannot guarantee a read-only lookup
/// fail closed.
async fn holder_readonly(&self, _key: &str) -> Result<Option<LeaseRecord>, LeaseError> {
Err(LeaseError::Storage(StorageError::backend(
"lease store does not support read-only holder inspection",
)))
}
/// The condition a storage driver evaluates for `node`'s grant on `key` at
/// `epoch`, when the record lives in the storage ports at all: the
/// record must still name `node` at `epoch`, unreleased. The driver then
/// checks a write against the record in the same atomic step
/// ([`super::fence::LeaseFence::write_condition`]). `None` (the default) for
/// stores that keep records elsewhere, such as file locks.
fn write_condition(
&self,
_key: &str,
_node: &str,
_epoch: u64,
) -> Option<tinystoragedrivers_types::WriteCondition> {
None
}
}
/// Rejects keys that could escape a directory or a document id: empty,
/// too long, `.`-led, or containing anything but ASCII letters, digits and
/// `-`, `_`, `.`, `@`.
///
/// # Errors
///
/// [`StorageError`] of kind invalid input.
pub fn validate_key(key: &str) -> Result<(), StorageError> {
if key.is_empty() || key.len() > MAX_KEY_LEN {
return Err(StorageError::invalid_input(format!(
"lease key must be 1..={MAX_KEY_LEN} bytes"
)));
}
if key.starts_with('.') {
return Err(StorageError::invalid_input(
"lease key must not start with '.'",
));
}
if !key
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'@'))
{
return Err(StorageError::invalid_input(
"lease key may hold only ASCII letters, digits and - _ . @",
));
}
Ok(())
}
/// What an acquire does with the record it found.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Takeover {
/// Write this epoch; `unclean` is the grant's `previous_unclean`.
Take { epoch: u64, unclean: bool },
/// Someone else's live record stands.
Refuse,
/// The record's epoch cannot be advanced.
Exhausted,
}
/// The acquire rule both stores share. `held_epoch` is the epoch this store
/// instance currently holds for the key, if any.
pub(crate) fn decide(
found: Option<&LeaseRecord>,
node: &str,
held_epoch: Option<u64>,
now_ms: u64,
) -> Takeover {
let Some(record) = found else {
return Takeover::Take {
epoch: 1,
unclean: false,
};
};
let live_reentrant = record.owner == node
&& held_epoch == Some(record.epoch)
&& !record.released
&& now_ms < record.expires_at_ms;
if live_reentrant {
return Takeover::Take {
epoch: record.epoch,
unclean: false,
};
}
let Some(next) = record.epoch.checked_add(1) else {
return Takeover::Exhausted;
};
if record.released {
return Takeover::Take {
epoch: next,
unclean: false,
};
}
if record.owner == node {
return Takeover::Take {
epoch: next,
unclean: true,
};
}
if now_ms >= record.expires_at_ms {
return Takeover::Take {
epoch: next,
unclean: true,
};
}
Takeover::Refuse
}
/// The directory a [`LocalLeases`] keeps `key`'s lock and record in.
///
/// The directory name is the SHA-256 of the key, so keys that differ only by
/// case, trailing dots or reserved device names never alias on
/// case-insensitive or Windows filesystems.
pub(crate) fn local_dir(root: &std::path::Path, key: &str) -> PathBuf {
use sha2::{Digest, Sha256};
root.join(hex::encode(Sha256::digest(key.as_bytes())))
}
#[cfg(test)]
#[path = "lease_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "lease_model_tests.rs"]
mod lease_model_tests;