The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(import): clean up import logic and tests
Steven Enamakel010926bcb577
diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rsindex ff291263fa..607816cccc 100644--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs@@ -1,6 +1,6 @@use super::*;use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};-use crate::storage::layout::{Target, legacy};+use crate::storage::layout::{legacy, Target};use tempfile::TempDir;#[test]diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rsindex 2cc99de865..cc4d778e19 100644--- a/crates/openhuman-core/src/storage/import/mod.rs+++ b/crates/openhuman-core/src/storage/import/mod.rs@@ -73,20 +73,20 @@ use std::sync::Arc;use anyhow::Result;#[cfg(feature = "legacy-import")]-use anyhow::{Context, anyhow};+use anyhow::{anyhow, Context};#[cfg(feature = "legacy-import")]use sha2::Digest;use tinystoragedrivers::CollectionSpec;#[cfg(feature = "legacy-import")]use tinystoragedrivers::{ErrorKind, Precondition};-use super::StorageBackend;#[cfg(feature = "legacy-import")]use super::block_on;#[cfg(feature = "legacy-import")]use super::documents::Repo;#[cfg(feature = "legacy-import")]use super::rusqlite;+use super::StorageBackend;use crate::config::Config;#[cfg(feature = "legacy-import")]use rusqlite::Connection;diff --git a/crates/openhuman-core/src/storage/import/mod_tests.rs b/crates/openhuman-core/src/storage/import/mod_tests.rsindex d3245b1453..402526212c 100644--- a/crates/openhuman-core/src/storage/import/mod_tests.rs+++ b/crates/openhuman-core/src/storage/import/mod_tests.rs@@ -252,18 +252,16 @@ fn processes_importing_one_workspace_at_once_import_it_once() {.filter(|o| matches!(o, Outcome::Imported { .. })).count();assert_eq!(imports, 1, "exactly one process imports: {outcomes:?}");- assert!(- outcomes- .iter()- .all(|o| !matches!(o, Outcome::Imported { .. })- || matches!(- o,- Outcome::Imported {- found: 20,- written: 20- }- ))- );+ assert!(outcomes+ .iter()+ .all(|o| !matches!(o, Outcome::Imported { .. })+ || matches!(+ o,+ Outcome::Imported {+ found: 20,+ written: 20+ }+ )));assert!(is_done(&backend, &config, &source).unwrap());assert_eq!(dump(&backend, &source, "devices.paired").len(), 20);}@@ -430,11 +428,9 @@ async fn without_an_installed_backend_the_host_import_does_nothing() {legacy_device(&config, "chan-a", "Phone", false);assert!(crate::storage::installed().is_none());- assert!(- run(&crate::core::runtime::DispatchCtx::unbound(), &config)- .await- .is_empty()- );+ assert!(run(&crate::core::runtime::DispatchCtx::unbound(), &config)+ .await+ .is_empty());}#[test]diff --git a/crates/openhuman-core/src/storage/import/schemas.rs b/crates/openhuman-core/src/storage/import/schemas.rsindex b3114aff7c..66cb2187fe 100644--- a/crates/openhuman-core/src/storage/import/schemas.rs+++ b/crates/openhuman-core/src/storage/import/schemas.rs@@ -3,10 +3,10 @@use crate::core::runtime::DispatchCtx;use serde::Deserialize;-use serde_json::{Map, Value, json};+use serde_json::{json, Map, Value};-use crate::core::Outcome;use crate::core::all::{ControllerFuture, RegisteredController};+use crate::core::Outcome;use crate::core::{ControllerSchema, FieldSchema, TypeSchema};#[derive(Debug, Deserialize, Default)]diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rsindex b8724bc43a..c2b6d38bd8 100644--- a/crates/openhuman-core/src/storage/import/sessions.rs+++ b/crates/openhuman-core/src/storage/import/sessions.rs@@ -32,12 +32,12 @@ use std::path::{Path, PathBuf};use std::sync::Arc;use anyhow::{Context, Result};-use tinyagents_session::DriverSessionStores;use tinyagents_session::legacy::{- ImportCounts, LegacyDatabaseReport, QuiescentSource, SourceOwner, import_transcripts,- import_turn_states,+ import_transcripts, import_turn_states, ImportCounts, LegacyDatabaseReport, QuiescentSource,+ SourceOwner,};-use tinyagents_session::transcript::{SessionRef, session_stem};+use tinyagents_session::transcript::{session_stem, SessionRef};+use tinyagents_session::DriverSessionStores;use super::tinyflows::{finish, marked};use super::{Outcome, Report, Snapshot};diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rsindex 9953e4340a..38e1e2bdc0 100644--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs@@ -1,6 +1,6 @@use super::*;-use crate::storage::MemoryStorage;use crate::storage::rusqlite;+use crate::storage::MemoryStorage;use crate::util::classic_transcript_fixture::write_classic;use sha2::Digest as _;use std::collections::BTreeMap;diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rsindex f4482ae4b1..3ba4d579ce 100644--- a/crates/openhuman-core/src/storage/import/tinyflows.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs@@ -27,7 +27,7 @@ use std::sync::{LazyLock, Mutex, Weak};use anyhow::{Context, Result};-use super::{MARKER_OWNER, Outcome, Report, marker_spec, marker_step};+use super::{marker_spec, marker_step, Outcome, Report, MARKER_OWNER};use crate::config::Config;use crate::storage::documents::Repo;use crate::storage::import_state;@@ -358,9 +358,18 @@ async fn migrate_owner_stores(backend: Arc<dyn StorageBackend>,config: Config,) -> (Vec<Report>, bool) {- let scopes = crate::storage::boot::scopes_to_migrate(&backend).await;- #[cfg(feature = "flows")]- let local = Scope::local();+ // v2 has no global scope registry. Prepare only the owner scopes whose+ // legacy stores this importer can actually discover: the workspace and+ // agent cron directories. The catalog remains workspace-local.+ let local = crate::storage::tenant_scope(None);+ let mut scopes = vec![local.clone()];+ scopes.extend(+ agent_cron_dirs(&config.workspace_dir)+ .into_iter()+ .map(|(agent, _)| crate::storage::scope_for_agent(&agent)),+ );+ scopes.sort_by(|left, right| left.as_str().cmp(right.as_str()));+ scopes.dedup();#[cfg(feature = "flows")]let mut local_catalog_ready = false;#[cfg(not(feature = "flows"))]@@ -374,7 +383,7 @@ async fn migrate_owner_stores(// second import change the database.if matches!(scope.parsed().map(|parsed| parsed.principal),- Ok(Some(tinystoragedrivers::Principal::Module(_)))+ Ok(Some(crate::storage::Principal::Module(_)))) {continue;}diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rsindex 94ffa9c137..b5184e0f6b 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs@@ -62,14 +62,13 @@ whereFut: Future<Output = Result<()>>,{let mut batches = BatchSender::default();- if let Some(sqlite_path) = sqlite_path- && let Some(db) = LegacyDb::open(sqlite_path)?- {- let tables = db.tables().await?;- if tables.contains("checkpoints") {- let mut offset = 0_i64;- loop {- let rows = db+ if let Some(sqlite_path) = sqlite_path {+ if let Some(db) = LegacyDb::open(sqlite_path)? {+ let tables = db.tables().await?;+ if tables.contains("checkpoints") {+ let mut offset = 0_i64;+ loop {+ let rows = db.read("checkpoints", move |connection| {let mut statement = connection.prepare("SELECT CASE WHEN length(CAST(record AS BLOB)) <= ?1 THEN CAST(record AS BLOB) END, seq FROM checkpoints ORDER BY seq LIMIT ?2 OFFSET ?3",@@ -87,12 +86,12 @@ where.collect::<std::result::Result<Vec<_>, _>>()}).await?;- if rows.is_empty() {- break;- }- offset += rows.len() as i64;- for (raw, sequence) in rows {- let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for (raw, sequence) in rows {+ let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {Some(raw_json) => LegacyImportRecord::Checkpoint { raw_json },None => LegacyImportRecord::Invalid {record_kind: LegacyImportKind::Checkpoint,@@ -100,14 +99,14 @@ wheremessage: "checkpoint source record exceeds the import byte limit or is not UTF-8".into(),},};- batches.push(record, &mut send).await?;+ batches.push(record, &mut send).await?;+ }}}- }- if tables.contains("checkpoint_writes") {- let mut offset = 0_i64;- loop {- let rows = db.read("checkpoint writes", move |connection| {+ if tables.contains("checkpoint_writes") {+ let mut offset = 0_i64;+ loop {+ let rows = db.read("checkpoint writes", move |connection| {let mut statement = connection.prepare("WITH source_rows AS (SELECT rowid, thread_id, namespace, checkpoint_id, task_id, idx, node, channel, payload, length(COALESCE(CAST(thread_id AS BLOB), X'')) + length(COALESCE(CAST(namespace AS BLOB), X'')) + length(COALESCE(CAST(checkpoint_id AS BLOB), X'')) + length(COALESCE(CAST(task_id AS BLOB), X'')) + length(COALESCE(CAST(node AS BLOB), X'')) + length(COALESCE(CAST(channel AS BLOB), X'')) + length(COALESCE(CAST(payload AS BLOB), X'')) AS raw_bytes FROM checkpoint_writes) SELECT rowid, raw_bytes <= ?1, CASE WHEN raw_bytes <= ?1 THEN CAST(thread_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(namespace AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(checkpoint_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(task_id AS BLOB) END, idx, CASE WHEN raw_bytes <= ?1 THEN CAST(node AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(channel AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(payload AS BLOB) END FROM source_rows ORDER BY rowid LIMIT ?2 OFFSET ?3",)?;@@ -134,54 +133,56 @@ where)?;mapped.collect::<std::result::Result<Vec<_>, _>>()}).await?;- if rows.is_empty() {- break;- }- offset += rows.len() as i64;- for (- row_id,- within_limit,- thread,- namespace,- checkpoint,- task,- idx,- node,- channel,- payload,- ) in rows- {- let record = if within_limit {- let decoded = (|| {- Some(LegacySqliteCheckpointWrite {- thread_id: String::from_utf8(thread?).ok()?,- namespace_json: String::from_utf8(namespace?).ok()?,- checkpoint_id: String::from_utf8(checkpoint?).ok()?,- task_id: String::from_utf8(task?).ok()?,- idx,- node: String::from_utf8(node?).ok()?,- channel: String::from_utf8(channel?).ok()?,- payload_json: String::from_utf8(payload?).ok()?,- })- })();- decoded.map_or_else(- || LegacyImportRecord::Invalid {+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for (+ row_id,+ within_limit,+ thread,+ namespace,+ checkpoint,+ task,+ idx,+ node,+ channel,+ payload,+ ) in rows+ {+ let record = if within_limit {+ let decoded = (|| {+ Some(LegacySqliteCheckpointWrite {+ thread_id: String::from_utf8(thread?).ok()?,+ namespace_json: String::from_utf8(namespace?).ok()?,+ checkpoint_id: String::from_utf8(checkpoint?).ok()?,+ task_id: String::from_utf8(task?).ok()?,+ idx,+ node: String::from_utf8(node?).ok()?,+ channel: String::from_utf8(channel?).ok()?,+ payload_json: String::from_utf8(payload?).ok()?,+ })+ })();+ decoded.map_or_else(+ || LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Write,+ identity: Some(format!("row:{row_id}")),+ message:+ "checkpoint write row contains a missing or non-UTF-8 field"+ .into(),+ },+ LegacyImportRecord::SqliteCheckpointWrite,+ )+ } else {+ LegacyImportRecord::Invalid {record_kind: LegacyImportKind::Write,identity: Some(format!("row:{row_id}")),- message:- "checkpoint write row contains a missing or non-UTF-8 field"- .into(),- },- LegacyImportRecord::SqliteCheckpointWrite,- )- } else {- LegacyImportRecord::Invalid {- record_kind: LegacyImportKind::Write,- identity: Some(format!("row:{row_id}")),- message: "checkpoint write row exceeds the import byte limit".into(),- }- };- batches.push(record, &mut send).await?;+ message: "checkpoint write row exceeds the import byte limit"+ .into(),+ }+ };+ batches.push(record, &mut send).await?;+ }}}}diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rsindex abfe4b32ef..3989529f1c 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs@@ -11,7 +11,7 @@ use tinyflows_bus::catalog::{MAX_IMPORT_RECORD_PAYLOAD_BYTES,};-use super::legacy_db::{LegacyDb, column_or};+use super::legacy_db::{column_or, LegacyDb};/// Reads legacy flow database rows and draft files as bounded ordered batches.///@@ -120,9 +120,9 @@ async fn page_rows<T, ToRecord, F, Fut>(what: &'static str,query: &str,decode: impl Fn(&crate::storage::rusqlite::Row<'_>) -> crate::storage::rusqlite::Result<T>- + Send- + Sync- + 'static,+ + Send+ + Sync+ + 'static,to_record: ToRecord,batches: &mut BatchSender,send: &mut F,diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rsindex d7d500f288..b949b6f755 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs@@ -14,7 +14,7 @@use std::collections::HashSet;use std::path::Path;-use anyhow::{Context, Result, anyhow};+use anyhow::{anyhow, Context, Result};use chrono::{DateTime, Utc};use std::collections::HashMap;use tinyflows_bus::schedule::types::{@@ -22,7 +22,7 @@ use tinyflows_bus::schedule::types::{SessionTarget,};-use super::legacy_db::{LegacyDb, column_or};+use super::legacy_db::{column_or, LegacyDb};use super::report::ImportReport;/// What `jobs.db` held, decoded.diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rsindex e8911f75d7..ae67bfcce8 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs@@ -10,5 +10,5 @@ mod report;pub(super) use checkpoint_reader::read_legacy_checkpoint_batches;pub(super) use flow_reader::read_legacy_flow_catalog;-pub(super) use legacy_cron::{LegacyCron, read_legacy_cron};+pub(super) use legacy_cron::{read_legacy_cron, LegacyCron};pub(super) use report::ImportReport;diff --git a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs b/crates/openhuman-core/src/storage/import/tinyflows_tests.rsindex 99d67b476a..ea1acf138a 100644--- a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs@@ -1,14 +1,14 @@use super::*;use crate::cron::client::testing as cron_store;use crate::storage::{MemoryStorage, Scope};-use serde_json::Value;use serde_json::json;+use serde_json::Value;use sha2::Digest;use std::collections::BTreeMap;use tempfile::TempDir;use tinyflows::engine::Checkpointer;-use tinyflows_drivers::DriverCheckpointer;use tinyflows_drivers::catalog::FlowCatalogDocuments;+use tinyflows_drivers::DriverCheckpointer;fn fixtures() -> PathBuf {Path::new(env!("CARGO_MANIFEST_DIR")).join("src/storage/import/fixtures/tinyflows")@@ -522,22 +522,18 @@ async fn catalog_and_schedule_owner_migrations_cover_existing_scopes() {for scope in [Scope::local(), profile] {let scoped = backend.for_scope(&scope).unwrap();- assert!(- scoped- .documents()- .get("flows.definitions", "missing")- .await- .unwrap()- .is_none()- );- assert!(- scoped- .documents()- .get("cron.jobs", "missing")- .await- .unwrap()- .is_none()- );+ assert!(scoped+ .documents()+ .get("flows.definitions", "missing")+ .await+ .unwrap()+ .is_none());+ assert!(scoped+ .documents()+ .get("cron.jobs", "missing")+ .await+ .unwrap()+ .is_none());}}