Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(storage): add fenced backend implementation

Steven Enamakel013d31f79bdb
diff --git a/crates/openhuman-core/src/storage/fenced_backend.rs b/crates/openhuman-core/src/storage/fenced_backend.rs
new file mode 100644
index 0000000000..341489c400
--- /dev/null
+++ b/crates/openhuman-core/src/storage/fenced_backend.rs
@@ -0,0 +1,685 @@
+//! [`FencedBackend`]: a storage backend whose writes pass the lease fence.
+//!
+//! Wraps any backend. Every scope it hands out is the inner backend's, with
+//! the document, stream and blob handles wrapped so that each write first
+//! asks the [`FenceRegistry`] for the scope's fence and runs only if that
+//! fence passes ([`FenceRegistry::admit`]). Reads, `ensure_collection`
+//! (idempotent schema) and scopes with no fence go straight through.
+//!
+//! A write the host check admits then goes through the inner backend's
+//! fenced handles ([`StorageBackend::for_scope_fenced`] with
+//! [`LeaseFence::driver_fence`](super::fence::LeaseFence::driver_fence)), so
+//! the driver re-checks the lease record atomically with the write. A
+//! driver refusal ([`ErrorKind::Fenced`]) latches the fence and surfaces as
+//! [`FenceError::Superseded`](super::fence::FenceError::Superseded). Where
+//! the driver cannot fence (no [`Capability::Fencing`], a write it answers
+//! with `Unsupported(Fencing)`, or a lease store with no
+//! record in the ports) the write runs after the host check alone.
+//!
+//! The fence is looked up per write, not when the scope is bound, because
+//! long-lived consumers (the session store caches its per-agent handles)
+//! must follow the profile as it is opened, fenced and re-opened.
+//!
+//! See [`super::fence`] for what the checks do and do not guarantee.
+
+use std::ops::Range;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use async_trait::async_trait;
+use serde_json::Value;
+use tinystoragedrivers::{
+ Blob, BlobMeta, BlobStore, BlobStream, Capabilities, Capability, CollectionSpec, ErrorKind,
+ Filter, Page, Precondition, Query, ReadCheck, Result, SearchHit, Sort, StorageError,
+ StreamEntry, StreamStore, Version, Versioned, WriteOp, WriteResult,
+};
+
+use super::fence::{FenceRegistry, LeaseFence};
+use super::{DocumentStore, Scope, ScopedStorage, StorageBackend};
+
+/// A backend whose writes are fenced. See the module docs.
+pub struct FencedBackend {
+ inner: Arc<dyn StorageBackend>,
+ fences: Arc<FenceRegistry>,
+}
+
+impl std::fmt::Debug for FencedBackend {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("FencedBackend")
+ .field("inner", &self.inner)
+ .finish_non_exhaustive()
+ }
+}
+
+impl FencedBackend {
+ /// `inner`, its writes checked against `fences`.
+ pub fn new(inner: Arc<dyn StorageBackend>, fences: Arc<FenceRegistry>) -> Self {
+ Self { inner, fences }
+ }
+}
+
+impl FencedBackend {
+ /// `scoped`'s handles with every write passing the scope's fence.
+ fn wrap_scoped(&self, scope: &Scope, scoped: &ScopedStorage) -> ScopedStorage {
+ let guard = Guard {
+ scope: scope.clone(),
+ fences: Arc::clone(&self.fences),
+ backend: Some(Arc::clone(&self.inner)),
+ };
+ ScopedStorage::new(
+ scope.clone(),
+ scoped.driver(),
+ Arc::new(FencedDocuments {
+ inner: Arc::clone(scoped.documents()),
+ guard: guard.clone(),
+ }),
+ Arc::new(FencedStreams {
+ inner: Arc::clone(scoped.streams()),
+ guard: guard.clone(),
+ }),
+ Arc::new(FencedBlobs {
+ inner: Arc::clone(scoped.blobs()),
+ guard,
+ }),
+ )
+ }
+}
+
+/// `inner` fenced by the process's registry ([`super::fence::registry`]).
+pub fn wrap(inner: Arc<dyn StorageBackend>) -> Arc<dyn StorageBackend> {
+ Arc::new(FencedBackend::new(inner, super::fence::registry()))
+}
+
+#[async_trait]
+impl StorageBackend for FencedBackend {
+ fn driver(&self) -> &'static str {
+ self.inner.driver()
+ }
+
+ fn capabilities(&self) -> Capabilities {
+ self.inner.capabilities()
+ }
+
+ fn for_scope(&self, scope: &Scope) -> Result<ScopedStorage> {
+ let scoped = self.inner.for_scope(scope)?;
+ Ok(self.wrap_scoped(scope, &scoped))
+ }
+
+ fn document_driver(&self) -> &'static str {
+ self.inner.document_driver()
+ }
+
+ fn existing_schema(&self) -> Result<Arc<dyn StorageBackend>> {
+ Ok(Arc::new(Self {
+ inner: self.inner.existing_schema()?,
+ fences: Arc::clone(&self.fences),
+ }))
+ }
+
+ async fn prepare_schema(&self) -> Result<()> {
+ self.inner.prepare_schema().await
+ }
+
+ /// The driver's own fenced handles; the fence wrapper itself builds on
+ /// these, so they are not wrapped a second time.
+ fn for_scope_fenced(
+ &self,
+ scope: &Scope,
+ fence: &tinystoragedrivers::Fence,
+ ) -> Result<ScopedStorage> {
+ self.inner.for_scope_fenced(scope, fence)
+ }
+
+ fn for_scope_generation(
+ &self,
+ scope: &Scope,
+ generation: tinystoragedrivers::ScopeGeneration,
+ ) -> Result<ScopedStorage> {
+ let scoped = self.inner.for_scope_generation(scope, generation)?;
+ Ok(self.wrap_scoped(scope, &scoped))
+ }
+
+ fn for_scope_fenced_generation(
+ &self,
+ scope: &Scope,
+ generation: tinystoragedrivers::ScopeGeneration,
+ fence: &tinystoragedrivers::Fence,
+ ) -> Result<ScopedStorage> {
+ self.inner
+ .for_scope_fenced_generation(scope, generation, fence)
+ }
+
+ async fn maintain(
+ &self,
+ budget: tinystoragedrivers::MaintenanceBudget,
+ ) -> Result<tinystoragedrivers::MaintenanceReport> {
+ self.inner.maintain(budget).await
+ }
+
+ async fn checkpoint(&self, mode: tinystoragedrivers::Checkpoint) -> Result<()> {
+ self.inner.checkpoint(mode).await
+ }
+
+ async fn scopes(&self, prefix: &str) -> Result<Vec<Scope>> {
+ self.inner.scopes(prefix).await
+ }
+
+ async fn scopes_of_tenant(&self, tenant: &Scope) -> Result<Vec<Scope>> {
+ self.inner.scopes_of_tenant(tenant).await
+ }
+
+ /// Reclaiming blobs that already read as absent changes nothing a reader
+ /// can see, so it needs no fence: it goes straight to the inner driver's
+ /// own implementation rather than the trait default over the fenced view.
+ async fn sweep_expired_blobs(&self, scope: &Scope) -> Result<u64> {
+ self.inner.sweep_expired_blobs(scope).await
+ }
+}
+
+#[derive(Clone)]
+struct Guard {
+ scope: Scope,
+ fences: Arc<FenceRegistry>,
+ /// The inner backend, whose driver may enforce the fence.
+ backend: Option<Arc<dyn StorageBackend>>,
+}
+
+impl Guard {
+ /// The host check. `Ok(Some(..))` when the driver should enforce the
+ /// fence too: the fence and the scope's fenced handles to write through.
+ async fn admit(&self, op: &'static str) -> Result<Option<(Arc<LeaseFence>, ScopedStorage)>> {
+ let Some(lease) = self.fences.admit(self.scope.as_str(), op).await? else {
+ return Ok(None);
+ };
+ let Some(backend) = &self.backend else {
+ return Ok(None);
+ };
+ if !backend.capabilities().contains(Capability::Fencing) {
+ return Ok(None);
+ }
+ let Some(fence) = lease.driver_fence() else {
+ return Ok(None);
+ };
+ match backend.for_scope_fenced(&self.scope, &fence) {
+ Ok(scoped) => Ok(Some((lease, scoped))),
+ Err(error) if unfenceable(&error) => Ok(None),
+ Err(error) => Err(error),
+ }
+ }
+
+ /// What a driver-fenced write's refusal means: a [`ErrorKind::Fenced`]
+ /// becomes the latched fence's error; anything else passes through.
+ async fn settle<T>(
+ &self,
+ lease: &LeaseFence,
+ op: &'static str,
+ outcome: Result<T>,
+ ) -> Result<T> {
+ match outcome {
+ Err(error) if error.kind() == ErrorKind::Fenced => {
+ Err(self.fences.driver_refused(lease, op).await)
+ }
+ other => other,
+ }
+ }
+}
+
+/// The driver cannot make this write atomic with the fence.
+fn unfenceable(error: &StorageError) -> bool {
+ error.kind() == ErrorKind::Unsupported(Capability::Fencing)
+}
+
+/// One fenced write: the host check, then `$port.$method(args)` through the
+/// driver's fenced handles when it can enforce the fence, else (or when it
+/// answers `Unsupported(Fencing)` for this write) through `$self.inner`.
+/// Arguments are cloned for the fenced attempt so the fallback can reuse
+/// them.
+macro_rules! fenced_write {
+ ($self:ident, $op:literal, $port:ident . $method:ident ( $($arg:expr),* $(,)? )) => {{
+ match $self.guard.admit($op).await? {
+ None => $self.inner.$method($($arg),*).await,
+ Some((lease, scoped)) => {
+ // `&str` and `Copy` arguments clone trivially; the owned ones
+ // must survive for the fallback.
+ #[allow(clippy::clone_on_copy)]
+ let outcome = scoped.$port().$method($(Clone::clone(&$arg)),*).await;
+ match outcome {
+ Err(error) if unfenceable(&error) => {
+ tracing::debug!(
+ target: "openhuman::storage::fence",
+ op = $op,
+ "[storage][fence] driver cannot fence this write; host check only"
+ );
+ $self.inner.$method($($arg),*).await
+ }
+ outcome => $self.guard.settle(&lease, $op, outcome).await,
+ }
+ }
+ }
+ }};
+}
+
+struct FencedDocuments {
+ inner: Arc<dyn DocumentStore>,
+ guard: Guard,
+}
+
+impl std::fmt::Debug for FencedDocuments {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("FencedDocuments")
+ .field("inner", &self.inner)
+ .finish_non_exhaustive()
+ }
+}
+
+#[async_trait]
+impl DocumentStore for FencedDocuments {
+ fn capabilities(&self) -> Capabilities {
+ self.inner.capabilities()
+ }
+
+ async fn ensure_collection(&self, spec: &CollectionSpec) -> Result<()> {
+ self.inner.ensure_collection(spec).await
+ }
+
+ async fn get(&self, collection: &str, id: &str) -> Result<Option<Versioned<Value>>> {
+ self.inner.get(collection, id).await
+ }
+
+ async fn put(
+ &self,
+ collection: &str,
+ id: &str,
+ doc: Value,
+ precondition: Precondition,
+ ) -> Result<Version> {
+ fenced_write!(
+ self,
+ "documents.put",
+ documents.put(collection, id, doc, precondition)
+ )
+ }
+
+ /// Forwarded (and fenced): without it the trait default refuses a write that
+ /// declares a schema version or expiry, which is every owner's write.
+ async fn put_with(
+ &self,
+ collection: &str,
+ id: &str,
+ doc: Value,
+ precondition: Precondition,
+ options: tinystoragedrivers::WriteOptions,
+ ) -> Result<Version> {
+ fenced_write!(
+ self,
+ "documents.put_with",
+ documents.put_with(collection, id, doc, precondition, options)
+ )
+ }
+
+ async fn delete(&self, collection: &str, id: &str, precondition: Precondition) -> Result<bool> {
+ fenced_write!(
+ self,
+ "documents.delete",
+ documents.delete(collection, id, precondition)
+ )
+ }
+
+ async fn query(&self, collection: &str, query: &Query) -> Result<Page<Versioned<Value>>> {
+ self.inner.query(collection, query).await
+ }
+
+ async fn count(&self, collection: &str, filter: &Filter) -> Result<u64> {
+ self.inner.count(collection, filter).await
+ }
+
+ async fn delete_where(&self, collection: &str, filter: &Filter) -> Result<u64> {
+ fenced_write!(
+ self,
+ "documents.delete_where",
+ documents.delete_where(collection, filter)
+ )
+ }
+
+ async fn claim(
+ &self,
+ collection: &str,
+ filter: &Filter,
+ sort: &[Sort],
+ patch: &Value,
+ ) -> Result<Option<Versioned<Value>>> {
+ fenced_write!(
+ self,
+ "documents.claim",
+ documents.claim(collection, filter, sort, patch)
+ )
+ }
+
+ async fn atomic_batch(&self, ops: Vec<WriteOp>) -> Result<Vec<WriteResult>> {
+ fenced_write!(self, "documents.atomic_batch", documents.atomic_batch(ops))
+ }
+
+ async fn atomic_batch_checked(
+ &self,
+ checks: Vec<ReadCheck>,
+ ops: Vec<WriteOp>,
+ ) -> Result<Vec<WriteResult>> {
+ fenced_write!(
+ self,
+ "documents.atomic_batch_checked",
+ documents.atomic_batch_checked(checks, ops)
+ )
+ }
+
+ async fn search(&self, collection: &str, text: &str, limit: usize) -> Result<Vec<SearchHit>> {
+ self.inner.search(collection, text, limit).await
+ }
+
+ async fn drop_collection(&self, collection: &str) -> Result<()> {
+ fenced_write!(
+ self,
+ "documents.drop_collection",
+ documents.drop_collection(collection)
+ )
+ }
+
+ async fn watch(
+ &self,
+ collection: &str,
+ filter: &Filter,
+ options: &tinystoragedrivers::WatchOptions,
+ ) -> Result<Box<dyn tinystoragedrivers::DocumentWatch>> {
+ self.inner.watch(collection, filter, options).await
+ }
+
+ async fn validate_collection_spec(&self, required: &CollectionSpec) -> Result<()> {
+ self.inner.validate_collection_spec(required).await
+ }
+
+ async fn collections(&self, prefix: &str) -> Result<Vec<String>> {
+ self.inner.collections(prefix).await
+ }
+}
+
+struct FencedStreams {
+ inner: Arc<dyn StreamStore>,
+ guard: Guard,
+}
+
+impl std::fmt::Debug for FencedStreams {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("FencedStreams")
+ .field("inner", &self.inner)
+ .finish_non_exhaustive()
+ }
+}
+
+#[async_trait]
+impl StreamStore for FencedStreams {
+ async fn append(&self, stream: &str, value: Value) -> Result<u64> {
+ fenced_write!(self, "streams.append", streams.append(stream, value))
+ }
+
+ async fn append_batch(&self, stream: &str, values: Vec<Value>) -> Result<u64> {
+ fenced_write!(
+ self,
+ "streams.append_batch",
+ streams.append_batch(stream, values)
+ )
+ }
+
+ /// Forwarded (and fenced): without it the trait default answers
+ /// `Unsupported(ConditionalStreamAppend)` even on a driver that has it,
+ /// which refuses every conversation append.
+ async fn append_batch_if_offset(
+ &self,
+ stream: &str,
+ expected_next_offset: u64,
+ values: Vec<Value>,
+ ) -> Result<u64> {
+ fenced_write!(
+ self,
+ "streams.append_batch_if_offset",
+ streams.append_batch_if_offset(stream, expected_next_offset, values)
+ )
+ }
+
+ async fn read_window(&self, stream: &str, from: u64, limit: usize) -> Result<Vec<StreamEntry>> {
+ self.inner.read_window(stream, from, limit).await
+ }
+
+ async fn len(&self, stream: &str) -> Result<u64> {
+ self.inner.len(stream).await
+ }
+
+ async fn truncate_before(&self, stream: &str, offset: u64) -> Result<u64> {
+ fenced_write!(
+ self,
+ "streams.truncate_before",
+ streams.truncate_before(stream, offset)
+ )
+ }
+
+ async fn delete_stream(&self, stream: &str) -> Result<bool> {
+ fenced_write!(self, "streams.delete_stream", streams.delete_stream(stream))
+ }
+
+ async fn streams(&self, prefix: &str) -> Result<Vec<String>> {
+ self.inner.streams(prefix).await
+ }
+
+ async fn info(&self, stream: &str) -> Result<Option<tinystoragedrivers::StreamInfo>> {
+ self.inner.info(stream).await
+ }
+
+ /// Forwarded (and fenced): sealing a finished stream is a write.
+ async fn seal(&self, stream: &str) -> Result<tinystoragedrivers::SealOutcome> {
+ fenced_write!(self, "streams.seal", streams.seal(stream))
+ }
+
+ async fn delete_sealed_stream(&self, stream: &str, sealed_ms: i64) -> Result<bool> {
+ fenced_write!(
+ self,
+ "streams.delete_sealed_stream",
+ streams.delete_sealed_stream(stream, sealed_ms)
+ )
+ }
+}
+
+struct FencedBlobs {
+ inner: Arc<dyn BlobStore>,
+ guard: Guard,
+}
+
+/// Two handles to one stream, used only to recover the input when a native
+/// fenced handle rejects streaming before it polls the reader.
+#[derive(Clone)]
+struct SharedBlobReader {
+ reader: Arc<Mutex<tinystoragedrivers::BlobReader>>,
+ polled: Arc<AtomicBool>,
+}
+
+impl tokio::io::AsyncRead for SharedBlobReader {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut tokio::io::ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ let this = self.get_mut();
+ this.polled.store(true, Ordering::SeqCst);
+ let mut reader = match this.reader.lock() {
+ Ok(reader) => reader,
+ Err(_) => {
+ return Poll::Ready(Err(std::io::Error::other(
+ "shared blob reader lock was poisoned",
+ )));
+ }
+ };
+ reader.as_mut().poll_read(cx, buf)
+ }
+}
+
+fn shared_blob_readers(
+ reader: tinystoragedrivers::BlobReader,
+) -> (
+ tinystoragedrivers::BlobReader,
+ tinystoragedrivers::BlobReader,
+ Arc<AtomicBool>,
+) {
+ let reader = Arc::new(Mutex::new(reader));
+ let polled = Arc::new(AtomicBool::new(false));
+ let handle = || {
+ Box::pin(SharedBlobReader {
+ reader: Arc::clone(&reader),
+ polled: Arc::clone(&polled),
+ }) as tinystoragedrivers::BlobReader
+ };
+ (handle(), handle(), polled)
+}
+
+impl std::fmt::Debug for FencedBlobs {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("FencedBlobs")
+ .field("inner", &self.inner)
+ .finish_non_exhaustive()
+ }
+}
+
+#[async_trait]
+impl BlobStore for FencedBlobs {
+ fn max_key_len(&self) -> usize {
+ self.inner.max_key_len()
+ }
+
+ /// The driver's abilities, minus presigned URLs: a presigned PUT is a write
+ /// the fence never sees, so the view does not offer them.
+ fn capabilities(&self) -> Capabilities {
+ self.inner.capabilities().without(Capability::PresignedUrls)
+ }
+
+ async fn put_expiring(
+ &self,
+ key: &str,
+ bytes: Vec<u8>,
+ content_type: Option<&str>,
+ ttl: Duration,
+ ) -> Result<BlobMeta> {
+ fenced_write!(
+ self,
+ "blobs.put_expiring",
+ blobs.put_expiring(key, bytes, content_type, ttl)
+ )
+ }
+
+ async fn get_stream(&self, key: &str) -> Result<Option<BlobStream>> {
+ self.inner.get_stream(key).await
+ }
+
+ async fn put_stream(
+ &self,
+ key: &str,
+ reader: tinystoragedrivers::BlobReader,
+ len: u64,
+ content_type: Option<&str>,
+ ) -> Result<BlobMeta> {
+ match self.guard.admit("blobs.put_stream").await? {
+ None => self.inner.put_stream(key, reader, len, content_type).await,
+ Some((lease, scoped)) => {
+ let (attempt, fallback, polled) = shared_blob_readers(reader);
+ let outcome = scoped
+ .blobs()
+ .put_stream(key, attempt, len, content_type)
+ .await;
+ match outcome {
+ Err(error) if unfenceable(&error) && !polled.load(Ordering::SeqCst) => {
+ tracing::debug!(
+ target: "openhuman::storage::fence",
+ op = "blobs.put_stream",
+ "[storage][fence] driver cannot fence this stream write; host check only"
+ );
+ self.inner
+ .put_stream(key, fallback, len, content_type)
+ .await
+ }
+ outcome => self.guard.settle(&lease, "blobs.put_stream", outcome).await,
+ }
+ }
+ }
+ }
+
+ async fn put_stream_expiring(
+ &self,
+ key: &str,
+ reader: tinystoragedrivers::BlobReader,
+ len: u64,
+ content_type: Option<&str>,
+ ttl: Duration,
+ ) -> Result<BlobMeta> {
+ match self.guard.admit("blobs.put_stream_expiring").await? {
+ None => {
+ self.inner
+ .put_stream_expiring(key, reader, len, content_type, ttl)
+ .await
+ }
+ Some((lease, scoped)) => {
+ let (attempt, fallback, polled) = shared_blob_readers(reader);
+ let outcome = scoped
+ .blobs()
+ .put_stream_expiring(key, attempt, len, content_type, ttl)
+ .await;
+ match outcome {
+ Err(error) if unfenceable(&error) && !polled.load(Ordering::SeqCst) => {
+ tracing::debug!(
+ target: "openhuman::storage::fence",
+ op = "blobs.put_stream_expiring",
+ "[storage][fence] driver cannot fence this stream write; host check only"
+ );
+ self.inner
+ .put_stream_expiring(key, fallback, len, content_type, ttl)
+ .await
+ }
+ outcome => {
+ self.guard
+ .settle(&lease, "blobs.put_stream_expiring", outcome)
+ .await
+ }
+ }
+ }
+ }
+ }
+
+ async fn presign_get(&self, key: &str, ttl: Duration) -> Result<String> {
+ self.inner.presign_get(key, ttl).await
+ }
+
+ async fn put(&self, key: &str, bytes: Vec<u8>, content_type: Option<&str>) -> Result<BlobMeta> {
+ fenced_write!(self, "blobs.put", blobs.put(key, bytes, content_type))
+ }
+
+ async fn get(&self, key: &str) -> Result<Option<Blob>> {
+ self.inner.get(key).await
+ }
+
+ async fn get_range(&self, key: &str, range: Range<u64>) -> Result<Option<Vec<u8>>> {
+ self.inner.get_range(key, range).await
+ }
+
+ async fn head(&self, key: &str) -> Result<Option<BlobMeta>> {
+ self.inner.head(key).await
+ }
+
+ async fn delete(&self, key: &str) -> Result<bool> {
+ fenced_write!(self, "blobs.delete", blobs.delete(key))
+ }
+
+ async fn list(&self, prefix: &str) -> Result<Vec<BlobMeta>> {
+ self.inner.list(prefix).await
+ }
+}
+
+#[cfg(test)]
+#[path = "fenced_backend_tests.rs"]
+mod tests;