Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(storage): run the tinyagents graph and harness migrations when a scope is prepared

M3gA-Mind84db5337b1af
diff --git a/crates/openhuman-core/src/agent/orchestration/delegation.rs b/crates/openhuman-core/src/agent/orchestration/delegation.rs
index 8022188c84..d2fdeea43e 100644
--- a/crates/openhuman-core/src/agent/orchestration/delegation.rs
+++ b/crates/openhuman-core/src/agent/orchestration/delegation.rs
@@ -353,7 +353,11 @@ pub fn open_delegation_checkpointer(
config: &Config,
) -> Result<Arc<dyn Checkpointer<DelegationState>>, String> {
let scoped = crate::storage::local::open("delegation", &config.workspace_dir)
- .and_then(|opened| opened.scoped())
+ .and_then(|opened| {
+ // Carries v1 `delegation_graph_*` checkpoints forward before the read.
+ crate::storage::documents::prepare_once(&opened.backend, &opened.scope)?;
+ opened.scoped()
+ })
.map_err(|e| format!("open the storage for graph checkpoints: {e:#}"))?;
Ok(Arc::new(
tinyagents_graph::checkpoint::DriverCheckpointer::<DelegationState>::with_namespace(
diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs
index 25647a0b3b..a206dbfbd4 100644
--- a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs
+++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs
@@ -27,7 +27,11 @@ pub fn embedding_tool_ranker_in(
workspace_dir: &std::path::Path,
) -> EmbeddingToolRanker {
let ranker = embedding_tool_ranker(provider);
- match crate::storage::local::open("tool_search", workspace_dir).and_then(|o| o.scoped()) {
+ let opened = crate::storage::local::open("tool_search", workspace_dir).and_then(|o| {
+ crate::storage::documents::prepare_once(&o.backend, &o.scope)?;
+ o.scoped()
+ });
+ match opened {
Ok(scoped) => ranker.with_cache(Arc::new(DriverEmbeddingCache::new(Arc::clone(
scoped.documents(),
)))),
diff --git a/crates/openhuman-core/src/modules/storage_grant_schemas.rs b/crates/openhuman-core/src/modules/storage_grant_schemas.rs
index 24a858697f..85fc5df151 100644
--- a/crates/openhuman-core/src/modules/storage_grant_schemas.rs
+++ b/crates/openhuman-core/src/modules/storage_grant_schemas.rs
@@ -118,7 +118,7 @@ pub(super) async fn migrate_imported(ctx: &crate::core::runtime::DispatchCtx, mo
};
let result = async {
let storage = backend.for_scope(&scope)?;
- crate::storage::boot::prepare_scope(&storage).await
+ crate::storage::boot::prepare_scope(&storage, &scope).await
}
.await;
if let Err(error) = result {
diff --git a/crates/openhuman-core/src/skills/run_log.rs b/crates/openhuman-core/src/skills/run_log.rs
index b29ce90484..cdd1ab1f60 100644
--- a/crates/openhuman-core/src/skills/run_log.rs
+++ b/crates/openhuman-core/src/skills/run_log.rs
@@ -45,8 +45,9 @@ fn run_log_store(workspace: &Path) -> anyhow::Result<RunLogStore> {
let opened = crate::storage::local::open("skills::run_log", workspace)?;
let storage = opened.scoped()?;
let prepared = storage.clone();
+ let scope = opened.scope.to_storage_scope();
crate::storage::block_on(async move {
- crate::storage::boot::prepare_scope(&prepared).await?;
+ crate::storage::boot::prepare_scope(&prepared, &scope).await?;
Ok(())
})?;
Ok(RunLogStore::new(Arc::clone(storage.streams())))
diff --git a/crates/openhuman-core/src/storage/README.md b/crates/openhuman-core/src/storage/README.md
index db9349a9af..d2b4895548 100644
--- a/crates/openhuman-core/src/storage/README.md
+++ b/crates/openhuman-core/src/storage/README.md
@@ -371,6 +371,6 @@ storage itself.
- `scope.rs`: the one scope grammar (`tenant_scope`, `scope_for_profile`, `scope_for_agent`, `scope_for_module`), built on tsd's `Scope::tenant/local/agent/module`.
- `schema.rs`: `collection_owners()` validates linked crates' `CollectionSpec` declarations without merging owners; `module_collections(owner)` feeds module grants.
-- `boot.rs`: `prepare_and_migrate`, called by `open`, ensures each linked crate's collections in the host's local and cluster scopes. Tenant, agent and module scopes are prepared by their owners when opened.
+- `boot.rs`: `prepare_and_migrate`, called by `open`, ensures each linked crate's collections in the host's local and cluster scopes and runs the tinyagents graph and harness owner migrations there (a failure is logged and retried at the next boot). Tenant, agent and module scopes are prepared (`prepare_scope`, the same migrations included) by their owners when opened.
- `scope_move.rs`: the resumable copy of v1 scopes (`profile:`, `<profile>~<agent>`, raw agent ids, `module:<id>@<base>`, the `*-sha256:` fallbacks) into the v2 scopes.
- The generated reference is `docs/storage-schema.md`; `node scripts/ci/check-storage-schema.mjs` lints the current collection names, indexes, expiry fields and search fields.
diff --git a/crates/openhuman-core/src/storage/boot.rs b/crates/openhuman-core/src/storage/boot.rs
index 45684265ae..30b17c94bd 100644
--- a/crates/openhuman-core/src/storage/boot.rs
+++ b/crates/openhuman-core/src/storage/boot.rs
@@ -4,6 +4,10 @@
//! Collection declarations belong to each record owner. The host asks each
//! linked crate for its declarations and prepares them on the shared local and
//! cluster scopes; tenant scopes are prepared when their owner is created.
+//!
+//! Preparing a scope also runs the tinyagents graph and harness owner-local
+//! migrations (`migrate_owners`), which carry their v1 collection names and
+//! shapes forward. Both are idempotent, so a prepared scope re-runs nothing.
use std::sync::Arc;
@@ -16,6 +20,8 @@ pub struct BootReport {
pub scopes: usize,
/// Collection declarations ensured across those scopes.
pub collections: usize,
+ /// Owner migrations that failed (logged; the next boot retries them).
+ pub failures: usize,
}
/// Ensures each linked crate's collections in the host's shared scopes.
@@ -25,7 +31,9 @@ pub struct BootReport {
///
/// # Errors
/// Returns a collection validation, scope binding, or preparation error.
-/// Preparation is idempotent and may be retried on the next boot.
+/// Preparation is idempotent and may be retried on the next boot. A failed
+/// owner migration is logged and counted in [`BootReport::failures`] instead,
+/// so one scope's migration never stops the boot.
pub async fn prepare_and_migrate(
backend: &Arc<dyn StorageBackend>,
) -> Result<BootReport, StorageError> {
@@ -40,22 +48,33 @@ pub async fn prepare_and_migrate(
report.collections += 1;
}
}
+ if let Err(error) = migrate_owners(&storage, &scope).await {
+ tracing::warn!(
+ target: "openhuman::storage",
+ scope = scope.as_str(),
+ %error,
+ "[storage][boot] owner migration failed; retried at the next boot"
+ );
+ report.failures += 1;
+ }
report.scopes += 1;
}
tracing::info!(
target: "openhuman::storage",
scopes = report.scopes,
collections = report.collections,
+ failures = report.failures,
"[storage][boot] owner collections prepared"
);
Ok(report)
}
-/// Prepares a tenant or module scope before its owner starts writing there.
+/// Prepares a tenant or module scope before its owner starts writing there,
+/// including the graph and harness owner migrations (`migrate_owners`).
///
/// # Errors
-/// Returns the first owner declaration or backend error.
-pub async fn prepare_scope(storage: &Storage) -> Result<usize, StorageError> {
+/// Returns the first owner declaration, migration or backend error.
+pub async fn prepare_scope(storage: &Storage, scope: &Scope) -> Result<usize, StorageError> {
let owners = super::schema::collection_owners()?;
let mut count = 0;
for owner in &owners {
@@ -64,9 +83,20 @@ pub async fn prepare_scope(storage: &Storage) -> Result<usize, StorageError> {
count += 1;
}
}
+ migrate_owners(storage, scope).await?;
Ok(count)
}
+/// Runs the tinyagents graph and harness owner-local migrations on `scope`:
+/// v1 checkpoints, leases, task cache, run status, journals, blobs and
+/// key-value stores move to their v2 names. Idempotent.
+async fn migrate_owners(storage: &Storage, scope: &Scope) -> Result<(), StorageError> {
+ let scoped = tinyagents_storage_kit::ScopedStorage::new(storage.clone(), scope.clone());
+ tinyagents_graph::storage::open_storage(scoped.clone()).await?;
+ tinyagents_harness::storage::open_storage(scoped).await?;
+ Ok(())
+}
+
#[cfg(test)]
#[path = "boot_tests.rs"]
mod tests;
diff --git a/crates/openhuman-core/src/storage/boot_tests.rs b/crates/openhuman-core/src/storage/boot_tests.rs
index 89547d028b..8d04aa0503 100644
--- a/crates/openhuman-core/src/storage/boot_tests.rs
+++ b/crates/openhuman-core/src/storage/boot_tests.rs
@@ -25,7 +25,7 @@ async fn tenant_scopes_use_each_linked_crates_collections() {
let scope = super::super::scope::scope_for_profile("profile-a");
let storage = backend.for_scope(&scope).unwrap();
- let prepared = prepare_scope(&storage).await.unwrap();
+ let prepared = prepare_scope(&storage, &scope).await.unwrap();
let declared = super::super::schema::collection_owners()
.unwrap()
.iter()
@@ -33,3 +33,95 @@ async fn tenant_scopes_use_each_linked_crates_collections() {
.sum::<usize>();
assert_eq!(prepared, declared);
}
+
+/// v1 rows an older build left in `storage`: a delegation graph checkpoint
+/// (thread `t`, checkpoint `d1`) and a harness run status (`run.abc`).
+async fn put_v1_rows(storage: &Storage) {
+ use tinystoragedrivers::Precondition;
+ let checkpoint = tinyagents_graph::checkpoint::Checkpoint::new(7_i32, Vec::new())
+ .with_thread_id("t".to_string())
+ .with_checkpoint_id("d1".to_string());
+ let docs = storage.documents();
+ let rows = [
+ (
+ "delegation_graph_checkpoints",
+ format!("1:t/20:{:020}", 0),
+ serde_json::json!({
+ "thread": "t", "namespace": "", "seq": 0, "checkpoint_id": "d1",
+ "record": serde_json::to_value(&checkpoint).unwrap(),
+ }),
+ ),
+ (
+ "delegation_graph_threads",
+ "1:t".to_string(),
+ serde_json::json!({"thread": "t", "next": 1}),
+ ),
+ (
+ "harness_run_status",
+ "8f3a-hash".to_string(),
+ serde_json::json!({
+ "runId": "run.abc", "rootRunId": "run.abc", "threadId": "t", "active": false,
+ "record": {
+ "run_id": "run.abc", "root_run_id": "run.abc", "thread_id": "t",
+ "component": "model", "status": "completed", "current_phase": "model",
+ "model_calls": 1, "tool_calls": 0, "active_tool_calls": [],
+ "usage": {}, "cost": {},
+ "started_at": 1_700_000_000_u64, "updated_at": 1_700_000_010_u64,
+ "metadata": {}
+ }
+ }),
+ ),
+ ];
+ for (collection, id, body) in rows {
+ docs.put(collection, &id, body, Precondition::None)
+ .await
+ .unwrap();
+ }
+}
+
+/// The v1 rows are readable through the v2 readers.
+async fn assert_v1_rows_migrated(storage: &Storage) {
+ use tinyagents_graph::checkpoint::{Checkpointer, DriverCheckpointer};
+ let checkpointer =
+ DriverCheckpointer::<i32>::with_namespace(Arc::clone(storage.documents()), "delegation");
+ let found = checkpointer
+ .get("t", None)
+ .await
+ .unwrap()
+ .expect("the v1 delegation checkpoint is readable after preparation");
+ assert_eq!((found.checkpoint_id.as_str(), found.state), ("d1", 7));
+ let status = storage
+ .documents()
+ .get("harness.run_status", "run.abc")
+ .await
+ .unwrap();
+ assert!(
+ status.is_some(),
+ "the v1 harness run status is readable after preparation"
+ );
+}
+
+#[tokio::test]
+async fn boot_migrates_v1_graph_and_harness_rows_in_the_local_scope() {
+ let backend: Arc<dyn StorageBackend> =
+ Arc::new(tinystoragedrivers_testkit::MemoryBackend::new());
+ let local = backend.for_scope(&Scope::new("local").unwrap()).unwrap();
+ put_v1_rows(&local).await;
+
+ let report = prepare_and_migrate(&backend).await.unwrap();
+
+ assert_eq!(report.failures, 0);
+ assert_v1_rows_migrated(&local).await;
+}
+
+#[tokio::test]
+async fn prepare_scope_migrates_v1_graph_and_harness_rows() {
+ let backend = tinystoragedrivers_testkit::MemoryBackend::new();
+ let scope = super::super::scope::scope_for_profile("profile-a");
+ let storage = backend.for_scope(&scope).unwrap();
+ put_v1_rows(&storage).await;
+
+ prepare_scope(&storage, &scope).await.unwrap();
+
+ assert_v1_rows_migrated(&storage).await;
+}
diff --git a/crates/openhuman-core/src/storage/documents.rs b/crates/openhuman-core/src/storage/documents.rs
index bc85ce177a..55aa66733a 100644
--- a/crates/openhuman-core/src/storage/documents.rs
+++ b/crates/openhuman-core/src/storage/documents.rs
@@ -72,6 +72,42 @@ type PreparedKey = (usize, String);
static PREPARED_SCOPES: LazyLock<Mutex<HashMap<PreparedKey, Weak<dyn StorageBackend>>>> =
LazyLock::new(Mutex::default);
+/// Prepares `scope` on `backend` ([`super::boot::prepare_scope`]: owner
+/// collections and the graph and harness migrations) once per process, for
+/// sync stores that open a scope directly.
+///
+/// # Errors
+///
+/// When the backend refuses the scope or the preparation fails; the next
+/// call retries.
+pub(crate) fn prepare_once(
+ backend: &Arc<dyn StorageBackend>,
+ scope: &tinystoragedrivers::Scope,
+) -> Result<(), StorageError> {
+ let prepared_key = (backend_addr(backend), scope.as_str().to_owned());
+ let already_prepared = PREPARED_SCOPES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .get(&prepared_key)
+ .and_then(Weak::upgrade)
+ .is_some_and(|alive| Arc::ptr_eq(&alive, backend));
+ if already_prepared {
+ return Ok(());
+ }
+ let to_prepare = backend.for_scope(scope)?;
+ let scope = scope.clone();
+ block_on(async move {
+ super::boot::prepare_scope(&to_prepare, &scope).await?;
+ Ok(())
+ })?;
+ let mut prepared = PREPARED_SCOPES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ prepared.retain(|_, weak| weak.strong_count() > 0);
+ prepared.insert(prepared_key, Arc::downgrade(backend));
+ Ok(())
+}
+
/// The backend and scope a [`Repo`] was opened on, for declaring its
/// collections once per process instead of on every call.
#[derive(Clone)]
@@ -142,26 +178,8 @@ impl Repo {
let scoped = backend
.for_scope(scope)
.with_context(|| format!("[{domain}] open the storage scope"))?;
- let prepared_key = (backend_addr(backend), scope.as_str().to_owned());
- let already_prepared = PREPARED_SCOPES
- .lock()
- .unwrap_or_else(PoisonError::into_inner)
- .get(&prepared_key)
- .and_then(Weak::upgrade)
- .is_some_and(|alive| Arc::ptr_eq(&alive, backend));
- if !already_prepared {
- let to_prepare = scoped.clone();
- block_on(async move {
- super::boot::prepare_scope(&to_prepare).await?;
- Ok(())
- })
+ prepare_once(backend, scope)
.with_context(|| format!("[{domain}] prepare owner collections"))?;
- let mut prepared = PREPARED_SCOPES
- .lock()
- .unwrap_or_else(PoisonError::into_inner);
- prepared.retain(|_, weak| weak.strong_count() > 0);
- prepared.insert(prepared_key, Arc::downgrade(backend));
- }
let mut repo = Self::over(&scoped, domain, collections);
repo.origin = Some(Origin {
backend: Arc::clone(backend),
diff --git a/crates/openhuman-core/src/storage/sessions/tenant_router.rs b/crates/openhuman-core/src/storage/sessions/tenant_router.rs
index da26d0b1ca..3da734eb61 100644
--- a/crates/openhuman-core/src/storage/sessions/tenant_router.rs
+++ b/crates/openhuman-core/src/storage/sessions/tenant_router.rs
@@ -101,8 +101,9 @@ impl TenantSessionStores {
let scope = PortScope::new(owner_scope.as_str())?;
let storage = backend.for_scope(&owner_scope)?;
let prepare_storage = storage.clone();
+ let prepare_scope = scope.clone();
super::super::block_on(async move {
- crate::storage::boot::prepare_scope(&prepare_storage).await
+ crate::storage::boot::prepare_scope(&prepare_storage, &prepare_scope).await
})?;
Ok(ScopedStorage::new(storage, scope))
});
diff --git a/crates/openhuman-core/src/storage/sessions/workspace.rs b/crates/openhuman-core/src/storage/sessions/workspace.rs
index 81e14bb3a5..5d6a3d02a5 100644
--- a/crates/openhuman-core/src/storage/sessions/workspace.rs
+++ b/crates/openhuman-core/src/storage/sessions/workspace.rs
@@ -223,6 +223,10 @@ fn tenant_scope_for(workspace_dir: &Path, what: &str) -> Option<(String, ScopedS
}
};
let scope = session_scope(&key);
+ if let Err(error) = crate::storage::documents::prepare_once(&backend, &scope) {
+ log::warn!("[session_store] {what} scope not prepared agent={key}: {error}");
+ return None;
+ }
match backend.for_scope(&scope) {
Ok(scoped) => match tinyagents_storage_kit::Scope::new(scope.as_str()) {
Ok(port_scope) => Some((key, ScopedStorage::new(scoped, port_scope))),
@@ -279,7 +283,12 @@ pub fn current_documents_for(workspace_dir: &Path) -> Option<Arc<dyn DocumentSto
return None;
}
};
- match backend.for_scope(&session_scope(&key)) {
+ let scope = session_scope(&key);
+ if let Err(error) = crate::storage::documents::prepare_once(&backend, &scope) {
+ log::warn!("[session_store] documents scope not prepared agent={key}: {error}");
+ return None;
+ }
+ match backend.for_scope(&scope) {
Ok(scoped) => Some(Arc::clone(scoped.documents())),
Err(error) => {
log::warn!("[session_store] documents scope refused agent={key}: {error}");
@@ -306,7 +315,12 @@ pub fn status_store_for(
return None;
}
};
- let scoped = match backend.for_scope(&session_scope(super::DEFAULT_AGENT)) {
+ let scope = session_scope(super::DEFAULT_AGENT);
+ if let Err(error) = crate::storage::documents::prepare_once(&backend, &scope) {
+ log::warn!("[session_store] local run-status scope not prepared: {error}");
+ return None;
+ }
+ let scoped = match backend.for_scope(&scope) {
Ok(scoped) => scoped,
Err(error) => {
log::warn!("[session_store] local run-status scope refused: {error}");