Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): tidy imports and simplify boot migration

Steven Enamakel7e55513043c9
diff --git a/crates/openhuman-core/src/storage/documents.rs b/crates/openhuman-core/src/storage/documents.rs
index 0c652c7f4e..bc85ce177a 100644
--- a/crates/openhuman-core/src/storage/documents.rs
+++ b/crates/openhuman-core/src/storage/documents.rs
@@ -14,8 +14,8 @@ use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak};
use anyhow::{anyhow, Context, Result};
use serde_json::Value;
-use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Versioned};
use tinystoragedrivers::WriteCondition;
+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Versioned};
use super::{
block_on, current_scope, installed, DocumentStore, Scope, Storage, StorageBackend, StorageError,
@@ -204,14 +204,19 @@ impl Repo {
/// claim-bound imports.
pub(crate) fn conditioned(&self, condition: &WriteCondition) -> Result<Self> {
let origin = self.origin.as_ref().ok_or_else(|| {
- anyhow!("[{}] claim-bound storage requires an opened backend scope", self.domain)
+ anyhow!(
+ "[{}] claim-bound storage requires an opened backend scope",
+ self.domain
+ )
})?;
let scope = Scope::new(&origin.scope)
.with_context(|| format!("[{}] parse the storage scope", self.domain))?;
let scoped = origin
.backend
.for_scope_conditioned(&scope, condition)
- .with_context(|| format!("[{}] bind the import claim to storage writes", self.domain))?;
+ .with_context(|| {
+ format!("[{}] bind the import claim to storage writes", self.domain)
+ })?;
let mut repo = Self::over(&scoped, self.domain, self.collections);
repo.origin = Some(origin.clone());
Ok(repo)
diff --git a/crates/openhuman-core/src/storage/fenced_backend_tests.rs b/crates/openhuman-core/src/storage/fenced_backend_tests.rs
index f872801a7d..f0ea8c2004 100644
--- a/crates/openhuman-core/src/storage/fenced_backend_tests.rs
+++ b/crates/openhuman-core/src/storage/fenced_backend_tests.rs
@@ -6,8 +6,8 @@ use crate::storage::fence::{FenceError, LeaseFence, ScopeMatch};
use crate::storage::lease::{DocumentLeases, LeaseGrant, LeaseStore};
use serde_json::json;
use std::sync::{
- Arc,
atomic::{AtomicU64, Ordering},
+ Arc,
};
use std::time::Duration;
use tinystoragedrivers_api::StorageBackend as _;
@@ -112,14 +112,12 @@ async fn cached_handles_enforce_local_expiry_margin_and_accept_same_epoch_renewa
fence_error(&error),
Some(FenceError::Expired { .. })
));
- assert!(
- storage
- .documents()
- .get("items", "too-late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(storage
+ .documents()
+ .get("items", "too-late")
+ .await
+ .unwrap()
+ .is_none());
}
#[tokio::test]
@@ -162,45 +160,39 @@ async fn a_cached_handle_cannot_mutate_after_cross_process_takeover_on_any_port(
// A's local clock is deliberately behind. The local precheck succeeds;
// the driver's atomic condition is what refuses each cached-handle write.
a.set_now(1_000);
- assert!(
- storage
- .documents()
- .put("items", "late", json!({}), Precondition::Absent)
- .await
- .is_err()
- );
+ assert!(storage
+ .documents()
+ .put("items", "late", json!({}), Precondition::Absent)
+ .await
+ .is_err());
assert!(storage.streams().append("events", json!(1)).await.is_err());
- assert!(
- storage
- .blobs()
- .put("blob", b"x".to_vec(), None)
- .await
- .is_err()
- );
+ assert!(storage
+ .blobs()
+ .put("blob", b"x".to_vec(), None)
+ .await
+ .is_err());
assert!(storage.kv().put("key", json!(1)).await.is_err());
assert!(storage.queues().ack("work", &receipt).await.is_err());
let raw = shared
.for_scope(&Scope::new("profile:alice").unwrap())
.unwrap();
- assert!(
- raw.documents()
- .get("items", "late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(raw
+ .documents()
+ .get("items", "late")
+ .await
+ .unwrap()
+ .is_none());
assert_eq!(raw.streams().len("events").await.unwrap(), 0);
assert!(raw.blobs().get("blob").await.unwrap().is_none());
assert!(raw.kv().get("key").await.unwrap().is_none());
assert_eq!(raw.queues().stats("work").await.unwrap().leased, 1);
- assert!(
- raw.documents()
- .get("items", "kept")
- .await
- .unwrap()
- .is_some()
- );
+ assert!(raw
+ .documents()
+ .get("items", "kept")
+ .await
+ .unwrap()
+ .is_some());
}
#[tokio::test]
@@ -215,23 +207,19 @@ async fn a_manually_latched_cached_fence_refuses_mutations_before_driver_io() {
.await
.unwrap();
fence.fence();
- assert!(
- storage
- .documents()
- .put("items", "late", json!({}), Precondition::Absent)
- .await
- .is_err()
- );
- assert!(
- shared
- .for_scope(&Scope::new("profile:alice").unwrap())
- .unwrap()
- .documents()
- .get("items", "late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(storage
+ .documents()
+ .put("items", "late", json!({}), Precondition::Absent)
+ .await
+ .is_err());
+ assert!(shared
+ .for_scope(&Scope::new("profile:alice").unwrap())
+ .unwrap()
+ .documents()
+ .get("items", "late")
+ .await
+ .unwrap()
+ .is_none());
}
#[tokio::test]
diff --git a/crates/openhuman-core/src/storage/fenced_ports.rs b/crates/openhuman-core/src/storage/fenced_ports.rs
index 598aabd497..34ccb4d1ac 100644
--- a/crates/openhuman-core/src/storage/fenced_ports.rs
+++ b/crates/openhuman-core/src/storage/fenced_ports.rs
@@ -20,8 +20,8 @@ use tinystoragedrivers_types::{
};
use super::{
- StorageError,
fence::{FenceRegistry, LeaseFence},
+ StorageError,
};
fn check(fence: &LeaseFence, clock: &FenceRegistry) -> Result<()> {
diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
index ff291263fa..607816cccc 100644
--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs
+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
@@ -1,6 +1,6 @@
use super::*;
use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};
-use crate::storage::layout::{Target, legacy};
+use crate::storage::layout::{legacy, Target};
use tempfile::TempDir;
#[test]
diff --git a/crates/openhuman-core/src/storage/import/mod_tests.rs b/crates/openhuman-core/src/storage/import/mod_tests.rs
index 79c47d1b92..7d142a9472 100644
--- a/crates/openhuman-core/src/storage/import/mod_tests.rs
+++ b/crates/openhuman-core/src/storage/import/mod_tests.rs
@@ -166,16 +166,11 @@ fn an_import_claimed_by_another_process_waits_for_it_and_does_not_repeat_it() {
let finisher = {
let (markers, step) = (markers.clone(), step.clone());
std::thread::spawn(move || {
- crate::storage::import_state::with_claim(
- &markers,
- MARKER_OWNER,
- &step.clone(),
- |_| {
+ crate::storage::import_state::with_claim(&markers, MARKER_OWNER, &step.clone(), |_| {
held_tx.send(()).unwrap();
std::thread::sleep(std::time::Duration::from_millis(400));
crate::storage::import_state::set_done(&markers, MARKER_OWNER, &step)
- },
- )
+ })
.unwrap()
.expect("the first claimant runs");
})
@@ -257,18 +252,16 @@ fn processes_importing_one_workspace_at_once_import_it_once() {
.filter(|o| matches!(o, Outcome::Imported { .. }))
.count();
assert_eq!(imports, 1, "exactly one process imports: {outcomes:?}");
- assert!(
- outcomes
- .iter()
- .all(|o| !matches!(o, Outcome::Imported { .. })
- || matches!(
- o,
- Outcome::Imported {
- found: 20,
- written: 20
- }
- ))
- );
+ assert!(outcomes
+ .iter()
+ .all(|o| !matches!(o, Outcome::Imported { .. })
+ || matches!(
+ o,
+ Outcome::Imported {
+ found: 20,
+ written: 20
+ }
+ )));
assert!(is_done(&backend, &config, &source).unwrap());
assert_eq!(dump(&backend, &source, "devices.paired").len(), 20);
}
@@ -435,11 +428,9 @@ async fn without_an_installed_backend_the_host_import_does_nothing() {
legacy_device(&config, "chan-a", "Phone", false);
assert!(crate::storage::installed().is_none());
- assert!(
- run(&crate::core::runtime::DispatchCtx::unbound(), &config)
- .await
- .is_empty()
- );
+ assert!(run(&crate::core::runtime::DispatchCtx::unbound(), &config)
+ .await
+ .is_empty());
}
#[test]
diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rs
index 51247b6d56..9effd3af44 100644
--- a/crates/openhuman-core/src/storage/import/scope_move.rs
+++ b/crates/openhuman-core/src/storage/import/scope_move.rs
@@ -9,7 +9,7 @@ use std::collections::HashSet;
use std::path::Path;
use crate::storage::rusqlite::{self, Connection, OpenFlags};
-use anyhow::{Context, Result, bail};
+use anyhow::{bail, Context, Result};
use serde_json::Value;
use tinystoragedrivers::Meta;
diff --git a/crates/openhuman-core/src/storage/import/session_db.rs b/crates/openhuman-core/src/storage/import/session_db.rs
index d10bfcaa1f..56c2090e42 100644
--- a/crates/openhuman-core/src/storage/import/session_db.rs
+++ b/crates/openhuman-core/src/storage/import/session_db.rs
@@ -6,9 +6,9 @@
use std::path::Path;
-use anyhow::{Context, Result, bail};
+use anyhow::{bail, Context, Result};
use serde::de::DeserializeOwned;
-use serde_json::{Map, Number, Value, json};
+use serde_json::{json, Map, Number, Value};
use tinyagents_session::run_ledger::tool_effects::{ToolEffectRow, ToolEffectStatus};
use tinyagents_session::run_ledger::{
AgentRun, AgentTeam, AgentTeamMember, AgentTeamTask, RunEvent, RunTelemetry, WorkflowRun,
@@ -410,9 +410,7 @@ fn decode_value(value: Value, decode: Decode) -> Result<Value> {
Ok(match decode {
Decode::Plain => value,
Decode::JsonOrEmpty => {
- let raw = value
- .as_str()
- .context("legacy JSON column is not text")?;
+ let raw = value.as_str().context("legacy JSON column is not text")?;
serde_json::from_str(raw).unwrap_or_else(|_| json!({}))
}
Decode::OptionalJson => value
diff --git a/crates/openhuman-core/src/storage/import/session_db_tests.rs b/crates/openhuman-core/src/storage/import/session_db_tests.rs
index 46f0ca3b4f..e43bc11d48 100644
--- a/crates/openhuman-core/src/storage/import/session_db_tests.rs
+++ b/crates/openhuman-core/src/storage/import/session_db_tests.rs
@@ -2,7 +2,7 @@ use super::*;
use std::path::Path;
-use crate::storage::rusqlite::{Connection, params};
+use crate::storage::rusqlite::{params, Connection};
fn classic_session_file(path: &Path) {
let connection = Connection::open(path).unwrap();
@@ -124,9 +124,7 @@ fn rejects_a_database_without_classic_session_tables() {
let error = SessionDatabase::read(&path).unwrap_err();
- assert!(
- error
- .to_string()
- .contains("none of the classic session database tables")
- );
+ assert!(error
+ .to_string()
+ .contains("none of the classic session database tables"));
}
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rs
index 8b9b844fba..52e116c95c 100644
--- a/crates/openhuman-core/src/storage/import/sessions.rs
+++ b/crates/openhuman-core/src/storage/import/sessions.rs
@@ -32,15 +32,15 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
-use tinyagents_session::DriverSessionStores;
use tinyagents_session::port::TurnStates;
use tinyagents_session::run_ledger::{ImportCounts, LegacyDatabaseReport, LegacySessionImport};
use tinyagents_session::transcript::{
- CompactionMarker, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory,
+ session_stem, CompactionMarker, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory,
TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptPartial, TranscriptTurn,
- TurnUsage, session_stem,
+ TurnUsage,
};
use tinyagents_session::turn_state::TurnState;
+use tinyagents_session::DriverSessionStores;
use super::tinyflows::{finish, marked};
use super::{Outcome, Report, Snapshot};
@@ -71,8 +71,10 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>
if !db.is_file() && !raw.is_dir() && !turns.is_dir() {
return Ok(None);
}
- let claimed_backend =
- super::super::import_state::backend_with_condition(Arc::clone(backend), condition.clone());
+ let claimed_backend = super::super::import_state::backend_with_condition(
+ Arc::clone(backend),
+ condition.clone(),
+ );
let provider = DriverSessionStores::new(claimed_backend, Scope::local())
.map_err(|error| anyhow::anyhow!("start the session store bridge: {error}"))?
.recover_on_open(false);
@@ -411,8 +413,8 @@ fn decode_transcript(
Option<serde_json::Value>,
Vec<DisplayRecord>,
)> {
- let bytes = std::fs::read(path)
- .with_context(|| format!("read transcript {}", path.display()))?;
+ let bytes =
+ std::fs::read(path).with_context(|| format!("read transcript {}", path.display()))?;
let mut meta = None;
let mut tools = None;
let mut records = Vec::new();
diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rs
index 74b984a6a2..81d3413c64 100644
--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs
+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs
@@ -1,6 +1,6 @@
use super::*;
-use crate::storage::MemoryStorage;
use crate::storage::rusqlite;
+use crate::storage::MemoryStorage;
use crate::util::classic_transcript_fixture::write_classic;
use sha2::Digest as _;
use std::collections::BTreeMap;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rs
index ffc4be463a..0e2050a2f1 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs
@@ -27,7 +27,7 @@ use std::sync::{LazyLock, Mutex, Weak};
use anyhow::{Context, Result};
-use super::{MARKER_OWNER, Outcome, Report, marker_spec, marker_step};
+use super::{marker_spec, marker_step, Outcome, Report, MARKER_OWNER};
use crate::config::Config;
use crate::storage::documents::Repo;
use crate::storage::import_state;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
index abfe4b32ef..3989529f1c 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
@@ -11,7 +11,7 @@ use tinyflows_bus::catalog::{
MAX_IMPORT_RECORD_PAYLOAD_BYTES,
};
-use super::legacy_db::{LegacyDb, column_or};
+use super::legacy_db::{column_or, LegacyDb};
/// Reads legacy flow database rows and draft files as bounded ordered batches.
///
@@ -120,9 +120,9 @@ async fn page_rows<T, ToRecord, F, Fut>(
what: &'static str,
query: &str,
decode: impl Fn(&crate::storage::rusqlite::Row<'_>) -> crate::storage::rusqlite::Result<T>
- + Send
- + Sync
- + 'static,
+ + Send
+ + Sync
+ + 'static,
to_record: ToRecord,
batches: &mut BatchSender,
send: &mut F,
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
index d7d500f288..b949b6f755 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
@@ -14,7 +14,7 @@
use std::collections::HashSet;
use std::path::Path;
-use anyhow::{Context, Result, anyhow};
+use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use tinyflows_bus::schedule::types::{
@@ -22,7 +22,7 @@ use tinyflows_bus::schedule::types::{
SessionTarget,
};
-use super::legacy_db::{LegacyDb, column_or};
+use super::legacy_db::{column_or, LegacyDb};
use super::report::ImportReport;
/// What `jobs.db` held, decoded.
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
index 99d67b476a..ea1acf138a 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
@@ -1,14 +1,14 @@
use super::*;
use crate::cron::client::testing as cron_store;
use crate::storage::{MemoryStorage, Scope};
-use serde_json::Value;
use serde_json::json;
+use serde_json::Value;
use sha2::Digest;
use std::collections::BTreeMap;
use tempfile::TempDir;
use tinyflows::engine::Checkpointer;
-use tinyflows_drivers::DriverCheckpointer;
use tinyflows_drivers::catalog::FlowCatalogDocuments;
+use tinyflows_drivers::DriverCheckpointer;
fn fixtures() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("src/storage/import/fixtures/tinyflows")
@@ -522,22 +522,18 @@ async fn catalog_and_schedule_owner_migrations_cover_existing_scopes() {
for scope in [Scope::local(), profile] {
let scoped = backend.for_scope(&scope).unwrap();
- assert!(
- scoped
- .documents()
- .get("flows.definitions", "missing")
- .await
- .unwrap()
- .is_none()
- );
- assert!(
- scoped
- .documents()
- .get("cron.jobs", "missing")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(scoped
+ .documents()
+ .get("flows.definitions", "missing")
+ .await
+ .unwrap()
+ .is_none());
+ assert!(scoped
+ .documents()
+ .get("cron.jobs", "missing")
+ .await
+ .unwrap()
+ .is_none());
}
}
diff --git a/crates/openhuman-core/src/storage/import_state.rs b/crates/openhuman-core/src/storage/import_state.rs
index 699896f84d..f5d0c1b194 100644
--- a/crates/openhuman-core/src/storage/import_state.rs
+++ b/crates/openhuman-core/src/storage/import_state.rs
@@ -15,8 +15,8 @@ use tinystoragedrivers::{
};
use tinystoragedrivers_api::DocumentStore;
-use super::DocumentStoreExt;
use super::documents::Repo;
+use super::DocumentStoreExt;
/// How long a claim blocks other processes before it can be taken over.
pub const CLAIM_TTL_MS: i64 = 10 * 60 * 1000;
diff --git a/crates/openhuman-core/src/storage/import_state_tests.rs b/crates/openhuman-core/src/storage/import_state_tests.rs
index 96b86ffa48..31a2fdd4f0 100644
--- a/crates/openhuman-core/src/storage/import_state_tests.rs
+++ b/crates/openhuman-core/src/storage/import_state_tests.rs
@@ -1,7 +1,7 @@
use super::*;
use crate::storage::{Scope, StorageBackend};
-use std::sync::Arc;
use std::sync::mpsc;
+use std::sync::Arc;
use tinystoragedrivers::CollectionSpec;
use tinystoragedrivers_testkit::MemoryBackend;
@@ -187,11 +187,10 @@ fn a_legacy_marker_row_is_honoured_and_removed() {
.await?
);
assert!(!done(&owned, "storage", &step_id("storage_import", "other@x")).await?);
- assert!(
- docs.query_all("storage_import", &tinystoragedrivers::Query::all())
- .await?
- .is_empty()
- );
+ assert!(docs
+ .query_all("storage_import", &tinystoragedrivers::Query::all())
+ .await?
+ .is_empty());
assert_eq!(migrate_legacy_markers(&owned).await?, 0);
assert!(
done(
diff --git a/crates/openhuman-core/src/storage/layout/db.rs b/crates/openhuman-core/src/storage/layout/db.rs
index 4f4ac45686..ac49f061d6 100644
--- a/crates/openhuman-core/src/storage/layout/db.rs
+++ b/crates/openhuman-core/src/storage/layout/db.rs
@@ -4,4 +4,4 @@
//! the `legacy-import` feature. Layout migration code depends on this narrow
//! wrapper and does not open or query SQLite itself.
-pub use crate::storage::import::layout_db::{Counts, DbCopy, checkpoint, copy, table_counts};
+pub use crate::storage::import::layout_db::{checkpoint, copy, table_counts, Counts, DbCopy};
diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rs
index 7275ab7bd3..429fead89e 100644
--- a/crates/openhuman-core/src/storage/mod.rs
+++ b/crates/openhuman-core/src/storage/mod.rs
@@ -292,16 +292,9 @@ pub async fn open(url: &str) -> Result<Arc<dyn StorageBackend>, StorageError> {
// Every opened backend checks the process's lease fences on write; with
// no fence registered (every single-user host) that is a pass-through.
let backend = tinystoragedrivers::open(&parsed).await?;
- // Schema, scope move and migrations run on the raw backend, before any
- // lease fence exists to refuse them.
- let report = boot::prepare_and_migrate(&backend).await?;
- if !report.failures.is_empty() {
- tracing::warn!(
- target: "openhuman::storage",
- failures = report.failures.len(),
- "[storage] opened with unfinished migrations; they retry at the next boot"
- );
- }
+ // Prepare the composed owner collections before installing lease fences.
+ // Legacy physical-source migration is handled separately by importers.
+ boot::prepare_and_migrate(&backend).await?;
Ok(fenced_backend::wrap(backend))
}
diff --git a/crates/openhuman-core/src/storage/state_kv.rs b/crates/openhuman-core/src/storage/state_kv.rs
index 14fcdb75fa..7065ae1ed9 100644
--- a/crates/openhuman-core/src/storage/state_kv.rs
+++ b/crates/openhuman-core/src/storage/state_kv.rs
@@ -411,11 +411,7 @@ impl StateKv {
/// # Errors
///
/// A storage error, or whatever `body` returns.
- pub fn import_once(
- &self,
- name: &str,
- body: impl FnOnce(&StateKv) -> Result<()>,
- ) -> Result<()> {
+ pub fn import_once(&self, name: &str, body: impl FnOnce(&StateKv) -> Result<()>) -> Result<()> {
let key = format!("{}/{}/{name}", self.owner, self.ns);
let step = import_state::step_id(IMPORT_KIND, &key);
self.repo.once_per_backend(&key, || {