Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(import): clean up import logic and tests

Steven Enamakel010926bcb577
diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
index ff291263fa..607816cccc 100644
--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs
+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
@@ -1,6 +1,6 @@
use super::*;
use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};
-use crate::storage::layout::{Target, legacy};
+use crate::storage::layout::{legacy, Target};
use tempfile::TempDir;
#[test]
diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rs
index 2cc99de865..cc4d778e19 100644
--- a/crates/openhuman-core/src/storage/import/mod.rs
+++ b/crates/openhuman-core/src/storage/import/mod.rs
@@ -73,20 +73,20 @@ use std::sync::Arc;
use anyhow::Result;
#[cfg(feature = "legacy-import")]
-use anyhow::{Context, anyhow};
+use anyhow::{anyhow, Context};
#[cfg(feature = "legacy-import")]
use sha2::Digest;
use tinystoragedrivers::CollectionSpec;
#[cfg(feature = "legacy-import")]
use tinystoragedrivers::{ErrorKind, Precondition};
-use super::StorageBackend;
#[cfg(feature = "legacy-import")]
use super::block_on;
#[cfg(feature = "legacy-import")]
use super::documents::Repo;
#[cfg(feature = "legacy-import")]
use super::rusqlite;
+use super::StorageBackend;
use crate::config::Config;
#[cfg(feature = "legacy-import")]
use rusqlite::Connection;
diff --git a/crates/openhuman-core/src/storage/import/mod_tests.rs b/crates/openhuman-core/src/storage/import/mod_tests.rs
index d3245b1453..402526212c 100644
--- a/crates/openhuman-core/src/storage/import/mod_tests.rs
+++ b/crates/openhuman-core/src/storage/import/mod_tests.rs
@@ -252,18 +252,16 @@ fn processes_importing_one_workspace_at_once_import_it_once() {
.filter(|o| matches!(o, Outcome::Imported { .. }))
.count();
assert_eq!(imports, 1, "exactly one process imports: {outcomes:?}");
- assert!(
- outcomes
- .iter()
- .all(|o| !matches!(o, Outcome::Imported { .. })
- || matches!(
- o,
- Outcome::Imported {
- found: 20,
- written: 20
- }
- ))
- );
+ assert!(outcomes
+ .iter()
+ .all(|o| !matches!(o, Outcome::Imported { .. })
+ || matches!(
+ o,
+ Outcome::Imported {
+ found: 20,
+ written: 20
+ }
+ )));
assert!(is_done(&backend, &config, &source).unwrap());
assert_eq!(dump(&backend, &source, "devices.paired").len(), 20);
}
@@ -430,11 +428,9 @@ async fn without_an_installed_backend_the_host_import_does_nothing() {
legacy_device(&config, "chan-a", "Phone", false);
assert!(crate::storage::installed().is_none());
- assert!(
- run(&crate::core::runtime::DispatchCtx::unbound(), &config)
- .await
- .is_empty()
- );
+ assert!(run(&crate::core::runtime::DispatchCtx::unbound(), &config)
+ .await
+ .is_empty());
}
#[test]
diff --git a/crates/openhuman-core/src/storage/import/schemas.rs b/crates/openhuman-core/src/storage/import/schemas.rs
index b3114aff7c..66cb2187fe 100644
--- a/crates/openhuman-core/src/storage/import/schemas.rs
+++ b/crates/openhuman-core/src/storage/import/schemas.rs
@@ -3,10 +3,10 @@
use crate::core::runtime::DispatchCtx;
use serde::Deserialize;
-use serde_json::{Map, Value, json};
+use serde_json::{json, Map, Value};
-use crate::core::Outcome;
use crate::core::all::{ControllerFuture, RegisteredController};
+use crate::core::Outcome;
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
#[derive(Debug, Deserialize, Default)]
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rs
index b8724bc43a..c2b6d38bd8 100644
--- a/crates/openhuman-core/src/storage/import/sessions.rs
+++ b/crates/openhuman-core/src/storage/import/sessions.rs
@@ -32,12 +32,12 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
-use tinyagents_session::DriverSessionStores;
use tinyagents_session::legacy::{
- ImportCounts, LegacyDatabaseReport, QuiescentSource, SourceOwner, import_transcripts,
- import_turn_states,
+ import_transcripts, import_turn_states, ImportCounts, LegacyDatabaseReport, QuiescentSource,
+ SourceOwner,
};
-use tinyagents_session::transcript::{SessionRef, session_stem};
+use tinyagents_session::transcript::{session_stem, SessionRef};
+use tinyagents_session::DriverSessionStores;
use super::tinyflows::{finish, marked};
use super::{Outcome, Report, Snapshot};
diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rs
index 9953e4340a..38e1e2bdc0 100644
--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs
+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs
@@ -1,6 +1,6 @@
use super::*;
-use crate::storage::MemoryStorage;
use crate::storage::rusqlite;
+use crate::storage::MemoryStorage;
use crate::util::classic_transcript_fixture::write_classic;
use sha2::Digest as _;
use std::collections::BTreeMap;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rs
index f4482ae4b1..3ba4d579ce 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs
@@ -27,7 +27,7 @@ use std::sync::{LazyLock, Mutex, Weak};
use anyhow::{Context, Result};
-use super::{MARKER_OWNER, Outcome, Report, marker_spec, marker_step};
+use super::{marker_spec, marker_step, Outcome, Report, MARKER_OWNER};
use crate::config::Config;
use crate::storage::documents::Repo;
use crate::storage::import_state;
@@ -358,9 +358,18 @@ async fn migrate_owner_stores(
backend: Arc<dyn StorageBackend>,
config: Config,
) -> (Vec<Report>, bool) {
- let scopes = crate::storage::boot::scopes_to_migrate(&backend).await;
- #[cfg(feature = "flows")]
- let local = Scope::local();
+ // v2 has no global scope registry. Prepare only the owner scopes whose
+ // legacy stores this importer can actually discover: the workspace and
+ // agent cron directories. The catalog remains workspace-local.
+ let local = crate::storage::tenant_scope(None);
+ let mut scopes = vec![local.clone()];
+ scopes.extend(
+ agent_cron_dirs(&config.workspace_dir)
+ .into_iter()
+ .map(|(agent, _)| crate::storage::scope_for_agent(&agent)),
+ );
+ scopes.sort_by(|left, right| left.as_str().cmp(right.as_str()));
+ scopes.dedup();
#[cfg(feature = "flows")]
let mut local_catalog_ready = false;
#[cfg(not(feature = "flows"))]
@@ -374,7 +383,7 @@ async fn migrate_owner_stores(
// second import change the database.
if matches!(
scope.parsed().map(|parsed| parsed.principal),
- Ok(Some(tinystoragedrivers::Principal::Module(_)))
+ Ok(Some(crate::storage::Principal::Module(_)))
) {
continue;
}
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs
index 94ffa9c137..b5184e0f6b 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs
@@ -62,14 +62,13 @@ where
Fut: Future<Output = Result<()>>,
{
let mut batches = BatchSender::default();
- if let Some(sqlite_path) = sqlite_path
- && let Some(db) = LegacyDb::open(sqlite_path)?
- {
- let tables = db.tables().await?;
- if tables.contains("checkpoints") {
- let mut offset = 0_i64;
- loop {
- let rows = db
+ if let Some(sqlite_path) = sqlite_path {
+ if let Some(db) = LegacyDb::open(sqlite_path)? {
+ let tables = db.tables().await?;
+ if tables.contains("checkpoints") {
+ let mut offset = 0_i64;
+ loop {
+ let rows = db
.read("checkpoints", move |connection| {
let mut statement = connection.prepare(
"SELECT CASE WHEN length(CAST(record AS BLOB)) <= ?1 THEN CAST(record AS BLOB) END, seq FROM checkpoints ORDER BY seq LIMIT ?2 OFFSET ?3",
@@ -87,12 +86,12 @@ where
.collect::<std::result::Result<Vec<_>, _>>()
})
.await?;
- if rows.is_empty() {
- break;
- }
- offset += rows.len() as i64;
- for (raw, sequence) in rows {
- let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for (raw, sequence) in rows {
+ let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {
Some(raw_json) => LegacyImportRecord::Checkpoint { raw_json },
None => LegacyImportRecord::Invalid {
record_kind: LegacyImportKind::Checkpoint,
@@ -100,14 +99,14 @@ where
message: "checkpoint source record exceeds the import byte limit or is not UTF-8".into(),
},
};
- batches.push(record, &mut send).await?;
+ batches.push(record, &mut send).await?;
+ }
}
}
- }
- if tables.contains("checkpoint_writes") {
- let mut offset = 0_i64;
- loop {
- let rows = db.read("checkpoint writes", move |connection| {
+ if tables.contains("checkpoint_writes") {
+ let mut offset = 0_i64;
+ loop {
+ let rows = db.read("checkpoint writes", move |connection| {
let mut statement = connection.prepare(
"WITH source_rows AS (SELECT rowid, thread_id, namespace, checkpoint_id, task_id, idx, node, channel, payload, length(COALESCE(CAST(thread_id AS BLOB), X'')) + length(COALESCE(CAST(namespace AS BLOB), X'')) + length(COALESCE(CAST(checkpoint_id AS BLOB), X'')) + length(COALESCE(CAST(task_id AS BLOB), X'')) + length(COALESCE(CAST(node AS BLOB), X'')) + length(COALESCE(CAST(channel AS BLOB), X'')) + length(COALESCE(CAST(payload AS BLOB), X'')) AS raw_bytes FROM checkpoint_writes) SELECT rowid, raw_bytes <= ?1, CASE WHEN raw_bytes <= ?1 THEN CAST(thread_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(namespace AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(checkpoint_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(task_id AS BLOB) END, idx, CASE WHEN raw_bytes <= ?1 THEN CAST(node AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(channel AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(payload AS BLOB) END FROM source_rows ORDER BY rowid LIMIT ?2 OFFSET ?3",
)?;
@@ -134,54 +133,56 @@ where
)?;
mapped.collect::<std::result::Result<Vec<_>, _>>()
}).await?;
- if rows.is_empty() {
- break;
- }
- offset += rows.len() as i64;
- for (
- row_id,
- within_limit,
- thread,
- namespace,
- checkpoint,
- task,
- idx,
- node,
- channel,
- payload,
- ) in rows
- {
- let record = if within_limit {
- let decoded = (|| {
- Some(LegacySqliteCheckpointWrite {
- thread_id: String::from_utf8(thread?).ok()?,
- namespace_json: String::from_utf8(namespace?).ok()?,
- checkpoint_id: String::from_utf8(checkpoint?).ok()?,
- task_id: String::from_utf8(task?).ok()?,
- idx,
- node: String::from_utf8(node?).ok()?,
- channel: String::from_utf8(channel?).ok()?,
- payload_json: String::from_utf8(payload?).ok()?,
- })
- })();
- decoded.map_or_else(
- || LegacyImportRecord::Invalid {
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for (
+ row_id,
+ within_limit,
+ thread,
+ namespace,
+ checkpoint,
+ task,
+ idx,
+ node,
+ channel,
+ payload,
+ ) in rows
+ {
+ let record = if within_limit {
+ let decoded = (|| {
+ Some(LegacySqliteCheckpointWrite {
+ thread_id: String::from_utf8(thread?).ok()?,
+ namespace_json: String::from_utf8(namespace?).ok()?,
+ checkpoint_id: String::from_utf8(checkpoint?).ok()?,
+ task_id: String::from_utf8(task?).ok()?,
+ idx,
+ node: String::from_utf8(node?).ok()?,
+ channel: String::from_utf8(channel?).ok()?,
+ payload_json: String::from_utf8(payload?).ok()?,
+ })
+ })();
+ decoded.map_or_else(
+ || LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Write,
+ identity: Some(format!("row:{row_id}")),
+ message:
+ "checkpoint write row contains a missing or non-UTF-8 field"
+ .into(),
+ },
+ LegacyImportRecord::SqliteCheckpointWrite,
+ )
+ } else {
+ LegacyImportRecord::Invalid {
record_kind: LegacyImportKind::Write,
identity: Some(format!("row:{row_id}")),
- message:
- "checkpoint write row contains a missing or non-UTF-8 field"
- .into(),
- },
- LegacyImportRecord::SqliteCheckpointWrite,
- )
- } else {
- LegacyImportRecord::Invalid {
- record_kind: LegacyImportKind::Write,
- identity: Some(format!("row:{row_id}")),
- message: "checkpoint write row exceeds the import byte limit".into(),
- }
- };
- batches.push(record, &mut send).await?;
+ message: "checkpoint write row exceeds the import byte limit"
+ .into(),
+ }
+ };
+ batches.push(record, &mut send).await?;
+ }
}
}
}
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
index abfe4b32ef..3989529f1c 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
@@ -11,7 +11,7 @@ use tinyflows_bus::catalog::{
MAX_IMPORT_RECORD_PAYLOAD_BYTES,
};
-use super::legacy_db::{LegacyDb, column_or};
+use super::legacy_db::{column_or, LegacyDb};
/// Reads legacy flow database rows and draft files as bounded ordered batches.
///
@@ -120,9 +120,9 @@ async fn page_rows<T, ToRecord, F, Fut>(
what: &'static str,
query: &str,
decode: impl Fn(&crate::storage::rusqlite::Row<'_>) -> crate::storage::rusqlite::Result<T>
- + Send
- + Sync
- + 'static,
+ + Send
+ + Sync
+ + 'static,
to_record: ToRecord,
batches: &mut BatchSender,
send: &mut F,
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
index d7d500f288..b949b6f755 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
@@ -14,7 +14,7 @@
use std::collections::HashSet;
use std::path::Path;
-use anyhow::{Context, Result, anyhow};
+use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use tinyflows_bus::schedule::types::{
@@ -22,7 +22,7 @@ use tinyflows_bus::schedule::types::{
SessionTarget,
};
-use super::legacy_db::{LegacyDb, column_or};
+use super::legacy_db::{column_or, LegacyDb};
use super::report::ImportReport;
/// What `jobs.db` held, decoded.
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs
index e8911f75d7..ae67bfcce8 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs
@@ -10,5 +10,5 @@ mod report;
pub(super) use checkpoint_reader::read_legacy_checkpoint_batches;
pub(super) use flow_reader::read_legacy_flow_catalog;
-pub(super) use legacy_cron::{LegacyCron, read_legacy_cron};
+pub(super) use legacy_cron::{read_legacy_cron, LegacyCron};
pub(super) use report::ImportReport;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
index 99d67b476a..ea1acf138a 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_tests.rs
@@ -1,14 +1,14 @@
use super::*;
use crate::cron::client::testing as cron_store;
use crate::storage::{MemoryStorage, Scope};
-use serde_json::Value;
use serde_json::json;
+use serde_json::Value;
use sha2::Digest;
use std::collections::BTreeMap;
use tempfile::TempDir;
use tinyflows::engine::Checkpointer;
-use tinyflows_drivers::DriverCheckpointer;
use tinyflows_drivers::catalog::FlowCatalogDocuments;
+use tinyflows_drivers::DriverCheckpointer;
fn fixtures() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("src/storage/import/fixtures/tinyflows")
@@ -522,22 +522,18 @@ async fn catalog_and_schedule_owner_migrations_cover_existing_scopes() {
for scope in [Scope::local(), profile] {
let scoped = backend.for_scope(&scope).unwrap();
- assert!(
- scoped
- .documents()
- .get("flows.definitions", "missing")
- .await
- .unwrap()
- .is_none()
- );
- assert!(
- scoped
- .documents()
- .get("cron.jobs", "missing")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(scoped
+ .documents()
+ .get("flows.definitions", "missing")
+ .await
+ .unwrap()
+ .is_none());
+ assert!(scoped
+ .documents()
+ .get("cron.jobs", "missing")
+ .await
+ .unwrap()
+ .is_none());
}
}