The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): improve legacy scope inventory reading
Steven Enamakel77e0d5d2bbdd
diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rsindex 430cb31c83..7687ecf1a9 100644--- a/crates/openhuman-core/src/storage/import/scope_move.rs+++ b/crates/openhuman-core/src/storage/import/scope_move.rs@@ -5,6 +5,7 @@//! explicitly and decode its rows here before writing them through typed v2//! storage ports. This reader never changes the source file.+use std::collections::HashSet;use std::path::Path;use crate::storage::rusqlite::{self, Connection, OpenFlags};@@ -62,6 +63,9 @@ pub struct LegacyBlob {/// A complete, immutable snapshot of the v1 rows needed for a scope move.#[derive(Debug, Clone, Default, PartialEq)]pub struct LegacyScopeInventory {+ /// Scope identities found in the source, including admitted scopes with+ /// no current rows.+ pub scopes: Vec<String>,/// Live document rows.pub documents: Vec<LegacyDocument>,/// Streams and their entries.@@ -84,12 +88,34 @@ impl LegacyScopeInventory {).with_context(|| format!("open legacy storage database {} read-only", path.display()))?;let mut inventory = Self::default();+ let mut scopes = std::collections::BTreeSet::new();++ // Admission rows are the authoritative inventory for v1 scopes that+ // happen to have no documents, streams, or blobs yet.+ if table_exists(&conn, "_tsd_scope_admissions")? {+ let mut statement = conn.prepare("SELECT scope FROM _tsd_scope_admissions ORDER BY scope")?;+ let rows = statement.query_map([], |row| row.get::<_, String>(0))?;+ for row in rows { scopes.insert(row?); }+ }if table_exists(&conn, "docs")? {- let mut statement = conn.prepare(- "SELECT scope, coll, id, schema, created_ms, updated_ms, expires_ms, deleted, body+ let columns = table_columns(&conn, "docs")?;+ for required in ["scope", "coll", "id", "body"] {+ if !columns.contains(required) {+ bail!("legacy docs table is missing required column `{required}`");+ }+ }+ let select = |name: &str, fallback: &str| {+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }+ };+ let sql = format!(+ "SELECT scope, coll, id, {}, {}, {}, {}, {}, bodyFROM docs ORDER BY scope, coll, id",- )?;+ select("schema", "0"), select("created_ms", "0"),+ select("updated_ms", "0"), select("expires_ms", "NULL"),+ select("deleted", "0"),+ );+ let mut statement = conn.prepare(&sql)?;let rows = statement.query_map([], |row| {Ok((row.get::<_, String>(0)?,@@ -115,6 +141,7 @@ impl LegacyScopeInventory {deleted,body,) = row?;+ scopes.insert(scope.clone());if deleted {continue;}@@ -135,10 +162,20 @@ impl LegacyScopeInventory {let mut heads = std::collections::BTreeMap::new();if table_exists(&conn, "stream_heads")? {- let mut statement = conn.prepare(- "SELECT scope, name, base, len, sealed, sealed_ms- FROM stream_heads ORDER BY scope, name",- )?;+ let columns = table_columns(&conn, "stream_heads")?;+ for required in ["scope", "name", "base", "len"] {+ if !columns.contains(required) {+ bail!("legacy stream_heads table is missing required column `{required}`");+ }+ }+ let select = |name: &str, fallback: &str| {+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }+ };+ let sql = format!(+ "SELECT scope, name, base, len, {}, {} FROM stream_heads ORDER BY scope, name",+ select("sealed", "0"), select("sealed_ms", "NULL"),+ );+ let mut statement = conn.prepare(&sql)?;let rows = statement.query_map([], |row| {Ok((row.get::<_, String>(0)?,@@ -151,14 +188,22 @@ impl LegacyScopeInventory {})?;for row in rows {let (scope, name, base, len, sealed, sealed_ms) = row?;+ scopes.insert(scope.clone());heads.insert((scope, name), (base, len, sealed, sealed_ms));}}let mut stream_entries: std::collections::BTreeMap<(String, String), Vec<(u64, Value)>> =std::collections::BTreeMap::new();if table_exists(&conn, "streams")? {- let mut statement = conn- .prepare("SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq")?;+ let columns = table_columns(&conn, "streams")?;+ for required in ["scope", "name", "seq", "body"] {+ if !columns.contains(required) {+ bail!("legacy streams table is missing required column `{required}`");+ }+ }+ let mut statement = conn.prepare(+ "SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq",+ )?;let rows = statement.query_map([], |row| {Ok((row.get::<_, String>(0)?,@@ -169,6 +214,7 @@ impl LegacyScopeInventory {})?;for row in rows {let (scope, name, seq, body) = row?;+ scopes.insert(scope.clone());stream_entries.entry((scope, name)).or_default().push((seq,serde_json::from_str(&body).context("decode legacy stream entry")?,@@ -205,9 +251,20 @@ impl LegacyScopeInventory {}if table_exists(&conn, "blobs")? {- let mut statement = conn.prepare(- "SELECT scope, key, content_type, expires_ms, bytes FROM blobs ORDER BY scope, key",- )?;+ let columns = table_columns(&conn, "blobs")?;+ for required in ["scope", "key", "bytes"] {+ if !columns.contains(required) {+ bail!("legacy blobs table is missing required column `{required}`");+ }+ }+ let select = |name: &str, fallback: &str| {+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }+ };+ let sql = format!(+ "SELECT scope, key, {}, {}, bytes FROM blobs ORDER BY scope, key",+ select("content_type", "NULL"), select("expires_ms", "NULL"),+ );+ let mut statement = conn.prepare(&sql)?;let rows = statement.query_map([], |row| {Ok((row.get::<_, String>(0)?,@@ -219,6 +276,7 @@ impl LegacyScopeInventory {})?;for row in rows {let (scope, key, content_type, expires_ms, bytes) = row?;+ scopes.insert(scope.clone());inventory.blobs.push(LegacyBlob {scope,key,@@ -228,6 +286,7 @@ impl LegacyScopeInventory {});}}+ inventory.scopes = scopes.into_iter().collect();Ok(inventory)}}@@ -240,6 +299,12 @@ fn table_exists(conn: &Connection, name: &str) -> rusqlite::Result<bool> {)}+fn table_columns(conn: &Connection, table: &str) -> rusqlite::Result<HashSet<String>> {+ let mut statement = conn.prepare(&format!("PRAGMA table_info(\"{}\")", table.replace('"', "\"\"")))?;+ let rows = statement.query_map([], |row| row.get::<_, String>(1))?;+ rows.collect()+}+fn validate_stream(scope: &str,name: &str,diff --git a/crates/openhuman-core/src/storage/import/scope_move_tests.rs b/crates/openhuman-core/src/storage/import/scope_move_tests.rsindex 4a33cbd20e..a1f7b1bf96 100644--- a/crates/openhuman-core/src/storage/import/scope_move_tests.rs+++ b/crates/openhuman-core/src/storage/import/scope_move_tests.rs@@ -8,7 +8,9 @@ fn inventory_reads_all_legacy_families_without_changing_the_database() {let path = dir.path().join("legacy.db");let conn = Connection::open(&path).unwrap();conn.execute_batch(- "CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,+ "CREATE TABLE _tsd_scope_admissions(scope TEXT PRIMARY KEY, generation INTEGER, operation TEXT);+ INSERT INTO _tsd_scope_admissions VALUES('empty-agent',1,'write');+ CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,created_ms INTEGER, updated_ms INTEGER, expires_ms INTEGER, deleted INTEGER, body TEXT,PRIMARY KEY(scope, coll, id));INSERT INTO docs VALUES('alice~agent','notes','n1',1,4,100,200,900,0,'{\"x\":1}');@@ -32,6 +34,8 @@ fn inventory_reads_all_legacy_families_without_changing_the_database() {assert_eq!(sha256(&path), before);assert_eq!(inventory.documents.len(), 1);+ assert!(inventory.scopes.contains(&"alice~agent".to_owned()));+ assert!(inventory.scopes.contains(&"empty-agent".to_owned()));assert_eq!(inventory.documents[0].meta.schema, 4);assert_eq!(inventory.documents[0].meta.created_ms, 100);assert_eq!(inventory.documents[0].meta.updated_ms, 200);diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rsindex f4bd71fc1b..e9ee86832e 100644--- a/crates/openhuman-core/src/storage/scope_move.rs+++ b/crates/openhuman-core/src/storage/scope_move.rs@@ -169,16 +169,12 @@ pub async fn copy_inventory(use tinystoragedrivers::CollectionSpec;let mut report = MoveReport::default();- let mut source_scopes = HashSet::new();- for row in &inventory.documents {- source_scopes.insert(row.scope.as_str());- }- for row in &inventory.streams {- source_scopes.insert(row.scope.as_str());- }- for row in &inventory.blobs {- source_scopes.insert(row.scope.as_str());- }+ let mut source_scopes: Vec<&str> = inventory.scopes.iter().map(String::as_str).collect();+ for row in &inventory.documents { source_scopes.push(&row.scope); }+ for row in &inventory.streams { source_scopes.push(&row.scope); }+ for row in &inventory.blobs { source_scopes.push(&row.scope); }+ source_scopes.sort_unstable();+ source_scopes.dedup();for source in source_scopes {let target_scope = match map_scope(source, known) {Mapping::V2 => continue,@@ -289,6 +285,22 @@ pub async fn copy_inventory(Ok(report)}+/// Reads an explicit SQLite source and copies its known legacy scopes.+///+/// The caller should run the synchronous inventory read on a blocking worker+/// when this function is invoked from an async runtime. MongoDB and other+/// non-SQLite legacy sources need their own read-only inventory adapter; they+/// are not enumerated from the v2 target backend.+#[cfg(feature = "legacy-import")]+pub async fn run_from_source(+ backend: &Arc<dyn StorageBackend>,+ source_db: &std::path::Path,+ known: &Known,+) -> anyhow::Result<MoveReport> {+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(source_db)?;+ copy_inventory(backend, known, &inventory).await+}+/// Legacy compatibility entry point retained to produce an actionable error./// v2 backends cannot enumerate old scopes; callers must supply the old SQLite/// path to [`crate::storage::import::scope_move::LegacyScopeInventory::read`]