Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): update storage API to use Storage instead of ScopedStorage

Steven Enamakel7edb63047477
diff --git a/crates/openhuman-core/src/storage/documents.rs b/crates/openhuman-core/src/storage/documents.rs
index f23f243805..2e78310250 100644
--- a/crates/openhuman-core/src/storage/documents.rs
+++ b/crates/openhuman-core/src/storage/documents.rs
@@ -14,11 +14,10 @@ use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak};
use anyhow::{anyhow, Context, Result};
use serde_json::Value;
-use tinystoragedrivers::{CollectionSpec, Repo as TsdRepo, RetryPolicy, Versioned};
+use tinystoragedrivers::{CollectionSpec, ErrorKind, Versioned};
use super::{
- block_on, current_scope, installed, DocumentStore, Scope, ScopedStorage, StorageBackend,
- StorageError,
+ block_on, current_scope, installed, DocumentStore, Scope, Storage, StorageBackend, StorageError,
};
/// Collections already declared, per process: `(backend address, scope,
@@ -79,7 +78,7 @@ struct Origin {
/// One domain's document store under one storage scope.
#[derive(Clone)]
pub struct Repo {
- scoped: ScopedStorage,
+ scoped: Storage,
docs: Arc<dyn DocumentStore>,
domain: &'static str,
collections: fn() -> Vec<CollectionSpec>,
@@ -148,7 +147,7 @@ impl Repo {
/// The repo over an already scoped handle (tests, explicit scopes).
pub fn over(
- scoped: &ScopedStorage,
+ scoped: &Storage,
domain: &'static str,
collections: fn() -> Vec<CollectionSpec>,
) -> Self {
@@ -163,7 +162,7 @@ impl Repo {
/// The scoped storage this repo was opened on, for the primitives that
/// bind to a whole scope (`tsd.migrations`).
- pub fn scoped(&self) -> &ScopedStorage {
+ pub fn scoped(&self) -> &Storage {
&self.scoped
}
@@ -243,9 +242,8 @@ impl Repo {
/// and returns what was stored, or `None` when the document is missing or
/// `change` declines (it is no longer in the state the change requires).
///
-/// The loop is tsd's [`tinystoragedrivers::Repo::update`] with its default
-/// [`RetryPolicy`] (32 attempts, jittered 1 to 8 ms backoff); the host keeps
-/// no loop of its own.
+/// The operation retries a conditional write up to 32 times when another
+/// writer wins the compare-and-swap race.
///
/// # Errors
///
@@ -257,16 +255,32 @@ pub async fn compare_and_swap(
id: &str,
change: impl Fn(&Value) -> Option<Value> + Send + Sync,
) -> Result<Option<Versioned<Value>>, StorageError> {
- let repo = TsdRepo::<Value>::new(docs, collection)?.with_retry(RetryPolicy::default());
- let written = repo
- .update(id, |current| Ok(current.and_then(|doc| change(&doc))))
- .await?;
- Ok(written.map(|(doc, version)| Versioned {
- id: id.to_string(),
- version,
- doc,
- meta: tinystoragedrivers::Meta::unknown(),
- }))
+ for _ in 0..32 {
+ let Some(current) = docs.get(collection, id).await? else {
+ return Ok(None);
+ };
+ let Some(next) = change(&current.doc) else {
+ return Ok(None);
+ };
+ match docs
+ .put(collection, id, next.clone(), current.unchanged())
+ .await
+ {
+ Ok(version) => {
+ return Ok(Some(Versioned {
+ id: id.to_owned(),
+ version,
+ doc: next,
+ meta: current.meta,
+ }));
+ }
+ Err(error) if error.kind() == ErrorKind::Conflict => continue,
+ Err(error) => return Err(error),
+ }
+ }
+ Err(StorageError::conflict(format!(
+ "document `{collection}/{id}` kept changing during compare-and-swap"
+ )))
}
/// The string field `field` of `doc`, when present.
diff --git a/crates/openhuman-core/src/storage/local.rs b/crates/openhuman-core/src/storage/local.rs
index 5d0d5f6322..24a597cf56 100644
--- a/crates/openhuman-core/src/storage/local.rs
+++ b/crates/openhuman-core/src/storage/local.rs
@@ -22,7 +22,7 @@ use std::sync::{Arc, LazyLock, Mutex, PoisonError};
use anyhow::{anyhow, Context, Result};
-use super::{current_scope, installed, Scope, ScopedStorage, StorageBackend};
+use super::{current_scope, installed, Scope, Storage, StorageBackend};
/// A backend and the scope a store's records live under in it.
#[derive(Clone)]
@@ -39,7 +39,7 @@ impl Opened {
/// # Errors
///
/// When the backend refuses the scope.
- pub fn scoped(&self) -> Result<ScopedStorage> {
+ pub fn scoped(&self) -> Result<Storage> {
self.backend
.for_scope(&self.scope)
.map_err(|error| anyhow!("open the storage scope: {error}"))
diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rs
index 154c12026f..e3d51621e2 100644
--- a/crates/openhuman-core/src/storage/mod.rs
+++ b/crates/openhuman-core/src/storage/mod.rs
@@ -71,12 +71,41 @@ use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
pub use tinystoragedrivers::{
CollectionSpec, Precondition, StorageBackend, StorageConfig as StorageUrl, StorageError,
};
-pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, Storage};
+pub use tinystoragedrivers_api::{DocumentStore, Storage};
/// SQLite is used only by the read-only compatibility importers while they
/// consume historical per-domain files. It is not the process storage API.
pub use tinystoragedrivers::sqlite::rusqlite;
+/// Typed conveniences for document consumers, including a bounded cursor
+/// walk for callers that need every matching record.
+#[async_trait::async_trait]
+pub trait DocumentStoreExt: tinystoragedrivers_api::DocumentStoreExt {
+ /// Read every record matching `query`, following the driver's cursor until
+ /// the final page. Each request is bounded to 256 documents.
+ async fn query_all(
+ &self,
+ collection: &str,
+ query: &tinystoragedrivers_types::Query,
+ ) -> Result<Vec<tinystoragedrivers_types::Versioned<serde_json::Value>>, StorageError> {
+ let mut page_query = query.clone();
+ page_query.limit = Some(256);
+ page_query.cursor = None;
+ let mut records = Vec::new();
+ loop {
+ let page = self.query(collection, &page_query).await?;
+ records.extend(page.items);
+ let Some(cursor) = page.next else {
+ break;
+ };
+ page_query.cursor = Some(cursor);
+ }
+ Ok(records)
+ }
+}
+
+impl<T: tinystoragedrivers_api::DocumentStoreExt + ?Sized> DocumentStoreExt for T {}
+
/// Dedicated runtime for synchronous domain surfaces that bridge to the
/// asynchronous storage ports. Keeping it on its own thread also permits a
/// synchronous facade to be called while the caller is on a current-thread
@@ -378,7 +407,10 @@ pub fn driver_has_cross_process_cas(driver: &str) -> bool {
/// Whether the installed backend may be shared with other processes; `false`
/// when none is installed. See [`driver_is_shared`].
pub fn installed_is_shared() -> bool {
- installed().is_some_and(|backend| driver_is_shared(backend.driver()))
+ installed_url()
+ .as_deref()
+ .and_then(|url| StorageUrl::parse(url).ok())
+ .is_some_and(|config| driver_is_shared(config.driver()))
}
/// A stable key for a legacy import source (a workspace directory), for the