Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): update storage schema definitions and dependencies

Steven Enamakel7a19d1a0b7d1
diff --git a/Cargo.lock b/Cargo.lock
index c016984503..3bff246b06 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5472,8 +5472,11 @@ dependencies = [
"tinyskills",
"tinyskills-drivers",
"tinystoragedrivers",
+ "tinystoragedrivers-api",
"tinystoragedrivers-bus",
"tinystoragedrivers-busproxy 2.0.2",
+ "tinystoragedrivers-testkit",
+ "tinystoragedrivers-types",
"tinytools",
"tinytools-agent",
"tinytools-std",
@@ -10046,6 +10049,26 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "tinystoragedrivers-testkit"
+version = "2.0.2"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "serde_json",
+ "tinystoragedrivers-api",
+ "tinystoragedrivers-blob",
+ "tinystoragedrivers-document",
+ "tinystoragedrivers-kv",
+ "tinystoragedrivers-queue",
+ "tinystoragedrivers-secrets",
+ "tinystoragedrivers-types",
+ "tokio",
+ "ulid",
+ "zeroize",
+]
+
[[package]]
name = "tinystoragedrivers-types"
version = "2.0.2"
@@ -12489,11 +12512,3 @@ checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
dependencies = [
"zune-core",
]
-
-[[patch.unused]]
-name = "tinystoragedrivers-testkit"
-version = "2.0.2"
-
-[[patch.unused]]
-name = "tinystoragedrivers-testkit"
-version = "2.0.2"
diff --git a/crates/openhuman-core/Cargo.toml b/crates/openhuman-core/Cargo.toml
index 144b3aa9c7..547b2338ef 100644
--- a/crates/openhuman-core/Cargo.toml
+++ b/crates/openhuman-core/Cargo.toml
@@ -116,6 +116,7 @@ tinyagents-session = { path = "../../vendor/tinyagents/crates/tinyagents-session
# The storage ports and the URL-to-backend facade (`storage` domain). No driver
# by default: the `storage-*` features below add them.
tinystoragedrivers = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.2", default-features = false }
+tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.2" }
tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.2" }
# The storage bus: the wire contract and the reference host server that the
# `modules::storage_host` interface answers loaded modules with. Modules only.
@@ -466,6 +467,7 @@ windows-sys = { version = "0.61", features = [
tokio-tungstenite = { version = "0.29", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] }
[dev-dependencies]
+tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.2" }
# Legacy TinyChannels provider/runtime fixtures remain available to tests only;
# production channel execution uses the TinyBus module contract above.
tinychannels = { path = "../../vendor/tinychannels", features = ["relay-websocket", "email", "lark"] }
diff --git a/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs b/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
index 23c24cc6ad..8923fbcf72 100644
--- a/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
+++ b/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
@@ -11,8 +11,8 @@
//! saved since. The driver's `updated_ms` is the time of the write, not of the
//! session's last use.
-use serde_json::{Map, Value};
use anyhow::Result;
+use serde_json::{Map, Value};
use tinystoragedrivers::CollectionSpec;
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/artifacts/storage_schema.rs b/crates/openhuman-core/src/artifacts/storage_schema.rs
index 4ba7587c8a..7639d6d87a 100644
--- a/crates/openhuman-core/src/artifacts/storage_schema.rs
+++ b/crates/openhuman-core/src/artifacts/storage_schema.rs
@@ -12,8 +12,8 @@
//! `{"_args": <value>}`, so the body is always an object and
//! [`args_of_body`] reads either form back.
-use serde_json::{json, Value};
use anyhow::Result;
+use serde_json::{json, Value};
use tinystoragedrivers::{CollectionSpec, IndexSpec};
use super::types::ArtifactMeta;
diff --git a/crates/openhuman-core/src/config/schema/load/source/storage_schema.rs b/crates/openhuman-core/src/config/schema/load/source/storage_schema.rs
index 3a27acac15..0ab2c19ff6 100644
--- a/crates/openhuman-core/src/config/schema/load/source/storage_schema.rs
+++ b/crates/openhuman-core/src/config/schema/load/source/storage_schema.rs
@@ -26,12 +26,10 @@ pub(crate) fn upgrade_records(
batch
.into_iter()
.map(|record| {
- Ok(LegacyTargetRecord::new(
- CONFIG_COLLECTION,
- key_of(&record.id)?,
- record.body,
+ Ok(
+ LegacyTargetRecord::new(CONFIG_COLLECTION, key_of(&record.id)?, record.body)
+ .created_ms(record.meta.created_ms),
)
- .created_ms(record.meta.created_ms))
})
.collect()
})
diff --git a/crates/openhuman-core/src/desktop/notifications/storage_schema.rs b/crates/openhuman-core/src/desktop/notifications/storage_schema.rs
index e3008cdf33..53312cf514 100644
--- a/crates/openhuman-core/src/desktop/notifications/storage_schema.rs
+++ b/crates/openhuman-core/src/desktop/notifications/storage_schema.rs
@@ -13,8 +13,8 @@
//! provider-controlled documents whose keys a document-store driver must never
//! see as field names.
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, IndexSpec, Key};
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/integrations/task_sources/storage_schema.rs b/crates/openhuman-core/src/integrations/task_sources/storage_schema.rs
index 10225f52f9..5b869b2829 100644
--- a/crates/openhuman-core/src/integrations/task_sources/storage_schema.rs
+++ b/crates/openhuman-core/src/integrations/task_sources/storage_schema.rs
@@ -10,8 +10,8 @@
//! provider-defined documents whose keys a document-store driver (MongoDB)
//! must never see as field names.
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, IndexSpec, Key};
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/platform/cost/storage_schema.rs b/crates/openhuman-core/src/platform/cost/storage_schema.rs
index 1a2adcc10f..4ab040134e 100644
--- a/crates/openhuman-core/src/platform/cost/storage_schema.rs
+++ b/crates/openhuman-core/src/platform/cost/storage_schema.rs
@@ -5,8 +5,8 @@
//! |---|---|---|
//! | `cost_records` (`usage.timestamp` RFC 3339 beside `ts_ms`) | `cost.records` | `usage.timestamp` dropped; `ts_ms` is the instant |
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, IndexSpec};
use super::types::CostRecord;
diff --git a/crates/openhuman-core/src/profiles/storage_schema.rs b/crates/openhuman-core/src/profiles/storage_schema.rs
index b763f854d7..49c74d2c5e 100644
--- a/crates/openhuman-core/src/profiles/storage_schema.rs
+++ b/crates/openhuman-core/src/profiles/storage_schema.rs
@@ -13,8 +13,8 @@
//! visits it; [`migrate_cluster`] runs this owner's steps over it, the first
//! time a registry opens the shared backend.
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, Storage};
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/security/approval/storage_schema.rs b/crates/openhuman-core/src/security/approval/storage_schema.rs
index d00ea682f4..2ce466bf70 100644
--- a/crates/openhuman-core/src/security/approval/storage_schema.rs
+++ b/crates/openhuman-core/src/security/approval/storage_schema.rs
@@ -12,8 +12,8 @@
//! stay JSON text: they carry arbitrary tool arguments whose keys a
//! document-store driver (MongoDB) must never see as field names.
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, IndexSpec, Key};
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/security/devices/storage_schema.rs b/crates/openhuman-core/src/security/devices/storage_schema.rs
index ab7e34bfac..91b727d3ad 100644
--- a/crates/openhuman-core/src/security/devices/storage_schema.rs
+++ b/crates/openhuman-core/src/security/devices/storage_schema.rs
@@ -5,8 +5,8 @@
//! |---|---|---|
//! | `paired_devices` (`created_at`, `last_seen_at` RFC 3339) | `devices.paired` (`created_ms`, `last_seen_ms`) | tenant |
-use serde_json::Value;
use anyhow::Result;
+use serde_json::Value;
use tinystoragedrivers::{CollectionSpec, IndexSpec};
use crate::storage::schema_support::{
diff --git a/crates/openhuman-core/src/storage/boot_tests.rs b/crates/openhuman-core/src/storage/boot_tests.rs
index 41db1f8382..3b965045c3 100644
--- a/crates/openhuman-core/src/storage/boot_tests.rs
+++ b/crates/openhuman-core/src/storage/boot_tests.rs
@@ -1,42 +1,31 @@
use super::*;
-use serde_json::json;
-use tinystoragedrivers::{MemoryStorage, Precondition};
#[tokio::test]
-async fn an_empty_backend_prepares_and_migrates_cleanly() {
- let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());
+async fn boot_prepares_the_composed_owner_collections_on_host_scopes() {
+ let backend: Arc<dyn StorageBackend> =
+ Arc::new(tinystoragedrivers_testkit::MemoryBackend::new());
+
let report = prepare_and_migrate(&backend).await.unwrap();
- assert!(report.failures.is_empty(), "{:?}", report.failures);
- assert!(report.scopes >= 1);
- // A second boot has nothing left to do.
- let again = prepare_and_migrate(&backend).await.unwrap();
- assert!(again.failures.is_empty(), "{:?}", again.failures);
- assert_eq!(again.steps_run, 0);
+ let declared = super::super::schema::owner_collections().unwrap();
+ assert_eq!(report.scopes, 2);
+ assert_eq!(report.collections, declared.len() * report.scopes);
+
+ // Owner declarations are idempotent: a retry after an interrupted boot
+ // may safely ensure every collection again.
+ let retry = prepare_and_migrate(&backend).await.unwrap();
+ assert_eq!(retry, report);
}
#[tokio::test]
-async fn a_v1_agent_scope_is_moved_before_its_owner_steps_run() {
- let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());
- let old = backend.for_scope(&Scope::new("asha").unwrap()).unwrap();
- old.documents()
- .put(
- "harness_embedding_cache",
- "legacy-id",
- json!({"sig": "s", "key": "k", "vector": [1.0]}),
- Precondition::None,
- )
- .await
- .unwrap();
- let report = prepare_and_migrate(&backend).await.unwrap();
- assert!(report.failures.is_empty(), "{:?}", report.failures);
+async fn tenant_scopes_use_the_same_owner_collection_declarations() {
+ let backend = tinystoragedrivers_testkit::MemoryBackend::new();
+ let scope = super::super::scope::scope_for_profile("profile-a");
+ let storage = backend.for_scope(&scope).unwrap();
+
+ let prepared = prepare_scope(&storage).await.unwrap();
- let new = backend
- .for_scope(&Scope::local().agent("asha").unwrap())
- .unwrap();
- assert!(new
- .documents()
- .get("harness.embedding_cache", "s/k")
- .await
- .unwrap()
- .is_some());
+ assert_eq!(
+ prepared,
+ super::super::schema::owner_collections().unwrap().len()
+ );
}
diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rs
index 4ef7ee9d8a..154c12026f 100644
--- a/crates/openhuman-core/src/storage/mod.rs
+++ b/crates/openhuman-core/src/storage/mod.rs
@@ -68,11 +68,40 @@ use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
/// tinystoragedrivers owns the one copy; the re-export goes when the importers
/// do (the one-time legacy import window closes). Re-checked for #7768: both
/// consumers are still present, so it stays.
-pub use tinystoragedrivers::sqlite::rusqlite;
pub use tinystoragedrivers::{
- Blocking, CollectionSpec, DocumentStore, DocumentStoreExt, MemoryStorage, Precondition,
- ScopedStorage, StorageBackend, StorageConfig as StorageUrl, StorageError,
+ CollectionSpec, Precondition, StorageBackend, StorageConfig as StorageUrl, StorageError,
};
+pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, Storage};
+
+/// SQLite is used only by the read-only compatibility importers while they
+/// consume historical per-domain files. It is not the process storage API.
+pub use tinystoragedrivers::sqlite::rusqlite;
+
+/// Dedicated runtime for synchronous domain surfaces that bridge to the
+/// asynchronous storage ports. Keeping it on its own thread also permits a
+/// synchronous facade to be called while the caller is on a current-thread
+/// Tokio runtime.
+struct Blocking(tokio::runtime::Runtime);
+
+impl Blocking {
+ fn new() -> anyhow::Result<Self> {
+ Ok(Self(
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .enable_all()
+ .thread_name("openhuman-storage")
+ .build()?,
+ ))
+ }
+
+ fn run<T, F>(&self, future: F) -> Result<T, StorageError>
+ where
+ F: Future<Output = Result<T, StorageError>> + Send + 'static,
+ T: Send + 'static,
+ {
+ Ok(self.0.block_on(future)?)
+ }
+}
use crate::config::schema::storage::redact_url;
use crate::config::Config;
@@ -271,7 +300,7 @@ pub fn scope_from(
///
/// When the scope cannot be resolved ([`current_scope`]) or the backend
/// refuses it.
-pub fn current_scoped() -> Result<Option<ScopedStorage>, StorageError> {
+pub fn current_scoped() -> Result<Option<Storage>, StorageError> {
installed()
.map(|backend| backend.for_scope(&current_scope()?))
.transpose()