The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): tidy imports and simplify boot migration
Steven Enamakel7e55513043c9
diff --git a/crates/openhuman-core/src/storage/documents.rs b/crates/openhuman-core/src/storage/documents.rsindex 0c652c7f4e..bc85ce177a 100644--- a/crates/openhuman-core/src/storage/documents.rs+++ b/crates/openhuman-core/src/storage/documents.rs@@ -14,8 +14,8 @@ use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak};use anyhow::{anyhow, Context, Result};use serde_json::Value;-use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Versioned};use tinystoragedrivers::WriteCondition;+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Versioned};use super::{block_on, current_scope, installed, DocumentStore, Scope, Storage, StorageBackend, StorageError,@@ -204,14 +204,19 @@ impl Repo {/// claim-bound imports.pub(crate) fn conditioned(&self, condition: &WriteCondition) -> Result<Self> {let origin = self.origin.as_ref().ok_or_else(|| {- anyhow!("[{}] claim-bound storage requires an opened backend scope", self.domain)+ anyhow!(+ "[{}] claim-bound storage requires an opened backend scope",+ self.domain+ )})?;let scope = Scope::new(&origin.scope).with_context(|| format!("[{}] parse the storage scope", self.domain))?;let scoped = origin.backend.for_scope_conditioned(&scope, condition)- .with_context(|| format!("[{}] bind the import claim to storage writes", self.domain))?;+ .with_context(|| {+ format!("[{}] bind the import claim to storage writes", self.domain)+ })?;let mut repo = Self::over(&scoped, self.domain, self.collections);repo.origin = Some(origin.clone());Ok(repo)diff --git a/crates/openhuman-core/src/storage/fenced_backend_tests.rs b/crates/openhuman-core/src/storage/fenced_backend_tests.rsindex f872801a7d..f0ea8c2004 100644--- a/crates/openhuman-core/src/storage/fenced_backend_tests.rs+++ b/crates/openhuman-core/src/storage/fenced_backend_tests.rs@@ -6,8 +6,8 @@ use crate::storage::fence::{FenceError, LeaseFence, ScopeMatch};use crate::storage::lease::{DocumentLeases, LeaseGrant, LeaseStore};use serde_json::json;use std::sync::{- Arc,atomic::{AtomicU64, Ordering},+ Arc,};use std::time::Duration;use tinystoragedrivers_api::StorageBackend as _;@@ -112,14 +112,12 @@ async fn cached_handles_enforce_local_expiry_margin_and_accept_same_epoch_renewafence_error(&error),Some(FenceError::Expired { .. })));- assert!(- storage- .documents()- .get("items", "too-late")- .await- .unwrap()- .is_none()- );+ assert!(storage+ .documents()+ .get("items", "too-late")+ .await+ .unwrap()+ .is_none());}#[tokio::test]@@ -162,45 +160,39 @@ async fn a_cached_handle_cannot_mutate_after_cross_process_takeover_on_any_port(// A's local clock is deliberately behind. The local precheck succeeds;// the driver's atomic condition is what refuses each cached-handle write.a.set_now(1_000);- assert!(- storage- .documents()- .put("items", "late", json!({}), Precondition::Absent)- .await- .is_err()- );+ assert!(storage+ .documents()+ .put("items", "late", json!({}), Precondition::Absent)+ .await+ .is_err());assert!(storage.streams().append("events", json!(1)).await.is_err());- assert!(- storage- .blobs()- .put("blob", b"x".to_vec(), None)- .await- .is_err()- );+ assert!(storage+ .blobs()+ .put("blob", b"x".to_vec(), None)+ .await+ .is_err());assert!(storage.kv().put("key", json!(1)).await.is_err());assert!(storage.queues().ack("work", &receipt).await.is_err());let raw = shared.for_scope(&Scope::new("profile:alice").unwrap()).unwrap();- assert!(- raw.documents()- .get("items", "late")- .await- .unwrap()- .is_none()- );+ assert!(raw+ .documents()+ .get("items", "late")+ .await+ .unwrap()+ .is_none());assert_eq!(raw.streams().len("events").await.unwrap(), 0);assert!(raw.blobs().get("blob").await.unwrap().is_none());assert!(raw.kv().get("key").await.unwrap().is_none());assert_eq!(raw.queues().stats("work").await.unwrap().leased, 1);- assert!(- raw.documents()- .get("items", "kept")- .await- .unwrap()- .is_some()- );+ assert!(raw+ .documents()+ .get("items", "kept")+ .await+ .unwrap()+ .is_some());}#[tokio::test]@@ -215,23 +207,19 @@ async fn a_manually_latched_cached_fence_refuses_mutations_before_driver_io() {.await.unwrap();fence.fence();- assert!(- storage- .documents()- .put("items", "late", json!({}), Precondition::Absent)- .await- .is_err()- );- assert!(- shared- .for_scope(&Scope::new("profile:alice").unwrap())- .unwrap()- .documents()- .get("items", "late")- .await- .unwrap()- .is_none()- );+ assert!(storage+ .documents()+ .put("items", "late", json!({}), Precondition::Absent)+ .await+ .is_err());+ assert!(shared+ .for_scope(&Scope::new("profile:alice").unwrap())+ .unwrap()+ .documents()+ .get("items", "late")+ .await+ .unwrap()+ .is_none());}#[tokio::test]diff --git a/crates/openhuman-core/src/storage/fenced_ports.rs b/crates/openhuman-core/src/storage/fenced_ports.rsindex 598aabd497..34ccb4d1ac 100644--- a/crates/openhuman-core/src/storage/fenced_ports.rs+++ b/crates/openhuman-core/src/storage/fenced_ports.rs@@ -20,8 +20,8 @@ use tinystoragedrivers_types::{};use super::{- StorageError,fence::{FenceRegistry, LeaseFence},+ StorageError,};fn check(fence: &LeaseFence, clock: &FenceRegistry) -> Result<()> {diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rsindex ff291263fa..607816cccc 100644--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs@@ -1,6 +1,6 @@use super::*;use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};-use crate::storage::layout::{Target, legacy};+use crate::storage::layout::{legacy, Target};use tempfile::TempDir;#[test]diff --git a/crates/openhuman-core/src/storage/import/mod_tests.rs b/crates/openhuman-core/src/storage/import/mod_tests.rsindex 79c47d1b92..7d142a9472 100644--- a/crates/openhuman-core/src/storage/import/mod_tests.rs+++ b/crates/openhuman-core/src/storage/import/mod_tests.rs@@ -166,16 +166,11 @@ fn an_import_claimed_by_another_process_waits_for_it_and_does_not_repeat_it() {let finisher = {let (markers, step) = (markers.clone(), step.clone());std::thread::spawn(move || {- crate::storage::import_state::with_claim(- &markers,- MARKER_OWNER,- &step.clone(),- |_| {+ crate::storage::import_state::with_claim(&markers, MARKER_OWNER, &step.clone(), |_| {held_tx.send(()).unwrap();std::thread::sleep(std::time::Duration::from_millis(400));crate::storage::import_state::set_done(&markers, MARKER_OWNER, &step)- },- )+ }).unwrap().expect("the first claimant runs");})@@ -257,18 +252,16 @@ fn processes_importing_one_workspace_at_once_import_it_once() {.filter(|o| matches!(o, Outcome::Imported { .. })).count();assert_eq!(imports, 1, "exactly one process imports: {outcomes:?}");- assert!(- outcomes- .iter()- .all(|o| !matches!(o, Outcome::Imported { .. })- || matches!(- o,- Outcome::Imported {- found: 20,- written: 20- }- ))- );+ assert!(outcomes+ .iter()+ .all(|o| !matches!(o, Outcome::Imported { .. })+ || matches!(+ o,+ Outcome::Imported {+ found: 20,+ written: 20+ }+ )));assert!(is_done(&backend, &config, &source).unwrap());assert_eq!(dump(&backend, &source, "devices.paired").len(), 20);}@@ -435,11 +428,9 @@ async fn without_an_installed_backend_the_host_import_does_nothing() {legacy_device(&config, "chan-a", "Phone", false);assert!(crate::storage::installed().is_none());- assert!(- run(&crate::core::runtime::DispatchCtx::unbound(), &config)- .await- .is_empty()- );+ assert!(run(&crate::core::runtime::DispatchCtx::unbound(), &config)+ .await+ .is_empty());}#[test]diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rsindex 51247b6d56..9effd3af44 100644--- a/crates/openhuman-core/src/storage/import/scope_move.rs+++ b/crates/openhuman-core/src/storage/import/scope_move.rs@@ -9,7 +9,7 @@ use std::collections::HashSet;use std::path::Path;use crate::storage::rusqlite::{self, Connection, OpenFlags};-use anyhow::{Context, Result, bail};+use anyhow::{bail, Context, Result};use serde_json::Value;use tinystoragedrivers::Meta;diff --git a/crates/openhuman-core/src/storage/import/session_db.rs b/crates/openhuman-core/src/storage/import/session_db.rsindex d10bfcaa1f..56c2090e42 100644--- a/crates/openhuman-core/src/storage/import/session_db.rs+++ b/crates/openhuman-core/src/storage/import/session_db.rs@@ -6,9 +6,9 @@use std::path::Path;-use anyhow::{Context, Result, bail};+use anyhow::{bail, Context, Result};use serde::de::DeserializeOwned;-use serde_json::{Map, Number, Value, json};+use serde_json::{json, Map, Number, Value};use tinyagents_session::run_ledger::tool_effects::{ToolEffectRow, ToolEffectStatus};use tinyagents_session::run_ledger::{AgentRun, AgentTeam, AgentTeamMember, AgentTeamTask, RunEvent, RunTelemetry, WorkflowRun,@@ -410,9 +410,7 @@ fn decode_value(value: Value, decode: Decode) -> Result<Value> {Ok(match decode {Decode::Plain => value,Decode::JsonOrEmpty => {- let raw = value- .as_str()- .context("legacy JSON column is not text")?;+ let raw = value.as_str().context("legacy JSON column is not text")?;serde_json::from_str(raw).unwrap_or_else(|_| json!({}))}Decode::OptionalJson => valuediff --git a/crates/openhuman-core/src/storage/import/session_db_tests.rs b/crates/openhuman-core/src/storage/import/session_db_tests.rsindex 46f0ca3b4f..e43bc11d48 100644--- a/crates/openhuman-core/src/storage/import/session_db_tests.rs+++ b/crates/openhuman-core/src/storage/import/session_db_tests.rs@@ -2,7 +2,7 @@ use super::*;use std::path::Path;-use crate::storage::rusqlite::{Connection, params};+use crate::storage::rusqlite::{params, Connection};fn classic_session_file(path: &Path) {let connection = Connection::open(path).unwrap();@@ -124,9 +124,7 @@ fn rejects_a_database_without_classic_session_tables() {let error = SessionDatabase::read(&path).unwrap_err();- assert!(- error- .to_string()- .contains("none of the classic session database tables")- );+ assert!(error+ .to_string()+ .contains("none of the classic session database tables"));}diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rsindex 8b9b844fba..52e116c95c 100644--- a/crates/openhuman-core/src/storage/import/sessions.rs+++ b/crates/openhuman-core/src/storage/import/sessions.rs@@ -32,15 +32,15 @@ use std::path::{Path, PathBuf};use std::sync::Arc;use anyhow::{Context, Result};-use tinyagents_session::DriverSessionStores;use tinyagents_session::port::TurnStates;use tinyagents_session::run_ledger::{ImportCounts, LegacyDatabaseReport, LegacySessionImport};use tinyagents_session::transcript::{- CompactionMarker, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory,+ session_stem, CompactionMarker, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory,TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptPartial, TranscriptTurn,- TurnUsage, session_stem,+ TurnUsage,};use tinyagents_session::turn_state::TurnState;+use tinyagents_session::DriverSessionStores;use super::tinyflows::{finish, marked};use super::{Outcome, Report, Snapshot};@@ -71,8 +71,10 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>if !db.is_file() && !raw.is_dir() && !turns.is_dir() {return Ok(None);}- let claimed_backend =- super::super::import_state::backend_with_condition(Arc::clone(backend), condition.clone());+ let claimed_backend = super::super::import_state::backend_with_condition(+ Arc::clone(backend),+ condition.clone(),+ );let provider = DriverSessionStores::new(claimed_backend, Scope::local()).map_err(|error| anyhow::anyhow!("start the session store bridge: {error}"))?.recover_on_open(false);@@ -411,8 +413,8 @@ fn decode_transcript(Option<serde_json::Value>,Vec<DisplayRecord>,)> {- let bytes = std::fs::read(path)- .with_context(|| format!("read transcript {}", path.display()))?;+ let bytes =+ std::fs::read(path).with_context(|| format!("read transcript {}", path.display()))?;let mut meta = None;let mut tools = None;let mut records = Vec::new();diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rsindex 74b984a6a2..81d3413c64 100644--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs@@ -1,6 +1,6 @@use super::*;-use crate::storage::MemoryStorage;use crate::storage::rusqlite;+use crate::storage::MemoryStorage;use crate::util::classic_transcript_fixture::write_classic;use sha2::Digest as _;use std::collections::BTreeMap;diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rsindex ffc4be463a..0e2050a2f1 100644--- a/crates/openhuman-core/src/storage/import/tinyflows.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs@@ -27,7 +27,7 @@ use std::sync::{LazyLock, Mutex, Weak};use anyhow::{Context, Result};-use super::{MARKER_OWNER, Outcome, Report, marker_spec, marker_step};+use super::{marker_spec, marker_step, Outcome, Report, MARKER_OWNER};use crate::config::Config;use crate::storage::documents::Repo;use crate::storage::import_state;diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rsindex abfe4b32ef..3989529f1c 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs@@ -11,7 +11,7 @@ use tinyflows_bus::catalog::{MAX_IMPORT_RECORD_PAYLOAD_BYTES,};-use super::legacy_db::{LegacyDb, column_or};+use super::legacy_db::{column_or, LegacyDb};/// Reads legacy flow database rows and draft files as bounded ordered batches.///@@ -120,9 +120,9 @@ async fn page_rows<T, ToRecord, F, Fut>(what: &'static str,query: &str,decode: impl Fn(&crate::storage::rusqlite::Row<'_>) -> crate::storage::rusqlite::Result<T>- + Send- + Sync- + 'static,+ + Send+ + Sync+ + 'static,to_record: ToRecord,batches: &mut BatchSender,send: &mut F,diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rsindex d7d500f288..b949b6f755 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs@@ -14,7 +14,7 @@use std::collections::HashSet;use std::path::Path;-use anyhow::{Context, Result, anyhow};+use anyhow::{anyhow, Context, Result};use chrono::{DateTime, Utc};use std::collections::HashMap;use tinyflows_bus::schedule::types::{@@ -22,7 +22,7 @@ use tinyflows_bus::schedule::types::{SessionTarget,};-use super::legacy_db::{LegacyDb, column_or};+use super::legacy_db::{column_or, LegacyDb};use super::report::ImportReport;/// What `jobs.db` held, decoded.diff --git a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs b/crates/openhuman-core/src/storage/import/tinyflows_tests.rsindex 99d67b476a..ea1acf138a 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs@@ -1,14 +1,14 @@use super::*;use crate::cron::client::testing as cron_store;use crate::storage::{MemoryStorage, Scope};-use serde_json::Value;use serde_json::json;+use serde_json::Value;use sha2::Digest;use std::collections::BTreeMap;use tempfile::TempDir;use tinyflows::engine::Checkpointer;-use tinyflows_drivers::DriverCheckpointer;use tinyflows_drivers::catalog::FlowCatalogDocuments;+use tinyflows_drivers::DriverCheckpointer;fn fixtures() -> PathBuf {Path::new(env!("CARGO_MANIFEST_DIR")).join("src/storage/import/fixtures/tinyflows")@@ -522,22 +522,18 @@ async fn catalog_and_schedule_owner_migrations_cover_existing_scopes() {for scope in [Scope::local(), profile] {let scoped = backend.for_scope(&scope).unwrap();- assert!(- scoped- .documents()- .get("flows.definitions", "missing")- .await- .unwrap()- .is_none()- );- assert!(- scoped- .documents()- .get("cron.jobs", "missing")- .await- .unwrap()- .is_none()- );+ assert!(scoped+ .documents()+ .get("flows.definitions", "missing")+ .await+ .unwrap()+ .is_none());+ assert!(scoped+ .documents()+ .get("cron.jobs", "missing")+ .await+ .unwrap()+ .is_none());}}diff --git a/crates/openhuman-core/src/storage/import_state.rs b/crates/openhuman-core/src/storage/import_state.rsindex 699896f84d..f5d0c1b194 100644--- a/crates/openhuman-core/src/storage/import_state.rs+++ b/crates/openhuman-core/src/storage/import_state.rs@@ -15,8 +15,8 @@ use tinystoragedrivers::{};use tinystoragedrivers_api::DocumentStore;-use super::DocumentStoreExt;use super::documents::Repo;+use super::DocumentStoreExt;/// How long a claim blocks other processes before it can be taken over.pub const CLAIM_TTL_MS: i64 = 10 * 60 * 1000;diff --git a/crates/openhuman-core/src/storage/import_state_tests.rs b/crates/openhuman-core/src/storage/import_state_tests.rsindex 96b86ffa48..31a2fdd4f0 100644--- a/crates/openhuman-core/src/storage/import_state_tests.rs+++ b/crates/openhuman-core/src/storage/import_state_tests.rs@@ -1,7 +1,7 @@use super::*;use crate::storage::{Scope, StorageBackend};-use std::sync::Arc;use std::sync::mpsc;+use std::sync::Arc;use tinystoragedrivers::CollectionSpec;use tinystoragedrivers_testkit::MemoryBackend;@@ -187,11 +187,10 @@ fn a_legacy_marker_row_is_honoured_and_removed() {.await?);assert!(!done(&owned, "storage", &step_id("storage_import", "other@x")).await?);- assert!(- docs.query_all("storage_import", &tinystoragedrivers::Query::all())- .await?- .is_empty()- );+ assert!(docs+ .query_all("storage_import", &tinystoragedrivers::Query::all())+ .await?+ .is_empty());assert_eq!(migrate_legacy_markers(&owned).await?, 0);assert!(done(diff --git a/crates/openhuman-core/src/storage/layout/db.rs b/crates/openhuman-core/src/storage/layout/db.rsindex 4f4ac45686..ac49f061d6 100644--- a/crates/openhuman-core/src/storage/layout/db.rs+++ b/crates/openhuman-core/src/storage/layout/db.rs@@ -4,4 +4,4 @@//! the `legacy-import` feature. Layout migration code depends on this narrow//! wrapper and does not open or query SQLite itself.-pub use crate::storage::import::layout_db::{Counts, DbCopy, checkpoint, copy, table_counts};+pub use crate::storage::import::layout_db::{checkpoint, copy, table_counts, Counts, DbCopy};diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rsindex 7275ab7bd3..429fead89e 100644--- a/crates/openhuman-core/src/storage/mod.rs+++ b/crates/openhuman-core/src/storage/mod.rs@@ -292,16 +292,9 @@ pub async fn open(url: &str) -> Result<Arc<dyn StorageBackend>, StorageError> {// Every opened backend checks the process's lease fences on write; with// no fence registered (every single-user host) that is a pass-through.let backend = tinystoragedrivers::open(&parsed).await?;- // Schema, scope move and migrations run on the raw backend, before any- // lease fence exists to refuse them.- let report = boot::prepare_and_migrate(&backend).await?;- if !report.failures.is_empty() {- tracing::warn!(- target: "openhuman::storage",- failures = report.failures.len(),- "[storage] opened with unfinished migrations; they retry at the next boot"- );- }+ // Prepare the composed owner collections before installing lease fences.+ // Legacy physical-source migration is handled separately by importers.+ boot::prepare_and_migrate(&backend).await?;Ok(fenced_backend::wrap(backend))}diff --git a/crates/openhuman-core/src/storage/state_kv.rs b/crates/openhuman-core/src/storage/state_kv.rsindex 14fcdb75fa..7065ae1ed9 100644--- a/crates/openhuman-core/src/storage/state_kv.rs+++ b/crates/openhuman-core/src/storage/state_kv.rs@@ -411,11 +411,7 @@ impl StateKv {/// # Errors////// A storage error, or whatever `body` returns.- pub fn import_once(- &self,- name: &str,- body: impl FnOnce(&StateKv) -> Result<()>,- ) -> Result<()> {+ pub fn import_once(&self, name: &str, body: impl FnOnce(&StateKv) -> Result<()>) -> Result<()> {let key = format!("{}/{}/{name}", self.owner, self.ns);let step = import_state::step_id(IMPORT_KIND, &key);self.repo.once_per_backend(&key, || {