Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): improve legacy scope inventory reading

Steven Enamakel77e0d5d2bbdd
diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rs
index 430cb31c83..7687ecf1a9 100644
--- a/crates/openhuman-core/src/storage/import/scope_move.rs
+++ b/crates/openhuman-core/src/storage/import/scope_move.rs
@@ -5,6 +5,7 @@
//! explicitly and decode its rows here before writing them through typed v2
//! storage ports. This reader never changes the source file.
+use std::collections::HashSet;
use std::path::Path;
use crate::storage::rusqlite::{self, Connection, OpenFlags};
@@ -62,6 +63,9 @@ pub struct LegacyBlob {
/// A complete, immutable snapshot of the v1 rows needed for a scope move.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct LegacyScopeInventory {
+ /// Scope identities found in the source, including admitted scopes with
+ /// no current rows.
+ pub scopes: Vec<String>,
/// Live document rows.
pub documents: Vec<LegacyDocument>,
/// Streams and their entries.
@@ -84,12 +88,34 @@ impl LegacyScopeInventory {
)
.with_context(|| format!("open legacy storage database {} read-only", path.display()))?;
let mut inventory = Self::default();
+ let mut scopes = std::collections::BTreeSet::new();
+
+ // Admission rows are the authoritative inventory for v1 scopes that
+ // happen to have no documents, streams, or blobs yet.
+ if table_exists(&conn, "_tsd_scope_admissions")? {
+ let mut statement = conn.prepare("SELECT scope FROM _tsd_scope_admissions ORDER BY scope")?;
+ let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
+ for row in rows { scopes.insert(row?); }
+ }
if table_exists(&conn, "docs")? {
- let mut statement = conn.prepare(
- "SELECT scope, coll, id, schema, created_ms, updated_ms, expires_ms, deleted, body
+ let columns = table_columns(&conn, "docs")?;
+ for required in ["scope", "coll", "id", "body"] {
+ if !columns.contains(required) {
+ bail!("legacy docs table is missing required column `{required}`");
+ }
+ }
+ let select = |name: &str, fallback: &str| {
+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }
+ };
+ let sql = format!(
+ "SELECT scope, coll, id, {}, {}, {}, {}, {}, body
FROM docs ORDER BY scope, coll, id",
- )?;
+ select("schema", "0"), select("created_ms", "0"),
+ select("updated_ms", "0"), select("expires_ms", "NULL"),
+ select("deleted", "0"),
+ );
+ let mut statement = conn.prepare(&sql)?;
let rows = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
@@ -115,6 +141,7 @@ impl LegacyScopeInventory {
deleted,
body,
) = row?;
+ scopes.insert(scope.clone());
if deleted {
continue;
}
@@ -135,10 +162,20 @@ impl LegacyScopeInventory {
let mut heads = std::collections::BTreeMap::new();
if table_exists(&conn, "stream_heads")? {
- let mut statement = conn.prepare(
- "SELECT scope, name, base, len, sealed, sealed_ms
- FROM stream_heads ORDER BY scope, name",
- )?;
+ let columns = table_columns(&conn, "stream_heads")?;
+ for required in ["scope", "name", "base", "len"] {
+ if !columns.contains(required) {
+ bail!("legacy stream_heads table is missing required column `{required}`");
+ }
+ }
+ let select = |name: &str, fallback: &str| {
+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }
+ };
+ let sql = format!(
+ "SELECT scope, name, base, len, {}, {} FROM stream_heads ORDER BY scope, name",
+ select("sealed", "0"), select("sealed_ms", "NULL"),
+ );
+ let mut statement = conn.prepare(&sql)?;
let rows = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
@@ -151,14 +188,22 @@ impl LegacyScopeInventory {
})?;
for row in rows {
let (scope, name, base, len, sealed, sealed_ms) = row?;
+ scopes.insert(scope.clone());
heads.insert((scope, name), (base, len, sealed, sealed_ms));
}
}
let mut stream_entries: std::collections::BTreeMap<(String, String), Vec<(u64, Value)>> =
std::collections::BTreeMap::new();
if table_exists(&conn, "streams")? {
- let mut statement = conn
- .prepare("SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq")?;
+ let columns = table_columns(&conn, "streams")?;
+ for required in ["scope", "name", "seq", "body"] {
+ if !columns.contains(required) {
+ bail!("legacy streams table is missing required column `{required}`");
+ }
+ }
+ let mut statement = conn.prepare(
+ "SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq",
+ )?;
let rows = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
@@ -169,6 +214,7 @@ impl LegacyScopeInventory {
})?;
for row in rows {
let (scope, name, seq, body) = row?;
+ scopes.insert(scope.clone());
stream_entries.entry((scope, name)).or_default().push((
seq,
serde_json::from_str(&body).context("decode legacy stream entry")?,
@@ -205,9 +251,20 @@ impl LegacyScopeInventory {
}
if table_exists(&conn, "blobs")? {
- let mut statement = conn.prepare(
- "SELECT scope, key, content_type, expires_ms, bytes FROM blobs ORDER BY scope, key",
- )?;
+ let columns = table_columns(&conn, "blobs")?;
+ for required in ["scope", "key", "bytes"] {
+ if !columns.contains(required) {
+ bail!("legacy blobs table is missing required column `{required}`");
+ }
+ }
+ let select = |name: &str, fallback: &str| {
+ if columns.contains(name) { format!("\"{name}\"") } else { fallback.to_owned() }
+ };
+ let sql = format!(
+ "SELECT scope, key, {}, {}, bytes FROM blobs ORDER BY scope, key",
+ select("content_type", "NULL"), select("expires_ms", "NULL"),
+ );
+ let mut statement = conn.prepare(&sql)?;
let rows = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
@@ -219,6 +276,7 @@ impl LegacyScopeInventory {
})?;
for row in rows {
let (scope, key, content_type, expires_ms, bytes) = row?;
+ scopes.insert(scope.clone());
inventory.blobs.push(LegacyBlob {
scope,
key,
@@ -228,6 +286,7 @@ impl LegacyScopeInventory {
});
}
}
+ inventory.scopes = scopes.into_iter().collect();
Ok(inventory)
}
}
@@ -240,6 +299,12 @@ fn table_exists(conn: &Connection, name: &str) -> rusqlite::Result<bool> {
)
}
+fn table_columns(conn: &Connection, table: &str) -> rusqlite::Result<HashSet<String>> {
+ let mut statement = conn.prepare(&format!("PRAGMA table_info(\"{}\")", table.replace('"', "\"\"")))?;
+ let rows = statement.query_map([], |row| row.get::<_, String>(1))?;
+ rows.collect()
+}
+
fn validate_stream(
scope: &str,
name: &str,
diff --git a/crates/openhuman-core/src/storage/import/scope_move_tests.rs b/crates/openhuman-core/src/storage/import/scope_move_tests.rs
index 4a33cbd20e..a1f7b1bf96 100644
--- a/crates/openhuman-core/src/storage/import/scope_move_tests.rs
+++ b/crates/openhuman-core/src/storage/import/scope_move_tests.rs
@@ -8,7 +8,9 @@ fn inventory_reads_all_legacy_families_without_changing_the_database() {
let path = dir.path().join("legacy.db");
let conn = Connection::open(&path).unwrap();
conn.execute_batch(
- "CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,
+ "CREATE TABLE _tsd_scope_admissions(scope TEXT PRIMARY KEY, generation INTEGER, operation TEXT);
+ INSERT INTO _tsd_scope_admissions VALUES('empty-agent',1,'write');
+ CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,
created_ms INTEGER, updated_ms INTEGER, expires_ms INTEGER, deleted INTEGER, body TEXT,
PRIMARY KEY(scope, coll, id));
INSERT INTO docs VALUES('alice~agent','notes','n1',1,4,100,200,900,0,'{\"x\":1}');
@@ -32,6 +34,8 @@ fn inventory_reads_all_legacy_families_without_changing_the_database() {
assert_eq!(sha256(&path), before);
assert_eq!(inventory.documents.len(), 1);
+ assert!(inventory.scopes.contains(&"alice~agent".to_owned()));
+ assert!(inventory.scopes.contains(&"empty-agent".to_owned()));
assert_eq!(inventory.documents[0].meta.schema, 4);
assert_eq!(inventory.documents[0].meta.created_ms, 100);
assert_eq!(inventory.documents[0].meta.updated_ms, 200);
diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rs
index f4bd71fc1b..e9ee86832e 100644
--- a/crates/openhuman-core/src/storage/scope_move.rs
+++ b/crates/openhuman-core/src/storage/scope_move.rs
@@ -169,16 +169,12 @@ pub async fn copy_inventory(
use tinystoragedrivers::CollectionSpec;
let mut report = MoveReport::default();
- let mut source_scopes = HashSet::new();
- for row in &inventory.documents {
- source_scopes.insert(row.scope.as_str());
- }
- for row in &inventory.streams {
- source_scopes.insert(row.scope.as_str());
- }
- for row in &inventory.blobs {
- source_scopes.insert(row.scope.as_str());
- }
+ let mut source_scopes: Vec<&str> = inventory.scopes.iter().map(String::as_str).collect();
+ for row in &inventory.documents { source_scopes.push(&row.scope); }
+ for row in &inventory.streams { source_scopes.push(&row.scope); }
+ for row in &inventory.blobs { source_scopes.push(&row.scope); }
+ source_scopes.sort_unstable();
+ source_scopes.dedup();
for source in source_scopes {
let target_scope = match map_scope(source, known) {
Mapping::V2 => continue,
@@ -289,6 +285,22 @@ pub async fn copy_inventory(
Ok(report)
}
+/// Reads an explicit SQLite source and copies its known legacy scopes.
+///
+/// The caller should run the synchronous inventory read on a blocking worker
+/// when this function is invoked from an async runtime. MongoDB and other
+/// non-SQLite legacy sources need their own read-only inventory adapter; they
+/// are not enumerated from the v2 target backend.
+#[cfg(feature = "legacy-import")]
+pub async fn run_from_source(
+ backend: &Arc<dyn StorageBackend>,
+ source_db: &std::path::Path,
+ known: &Known,
+) -> anyhow::Result<MoveReport> {
+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(source_db)?;
+ copy_inventory(backend, known, &inventory).await
+}
+
/// Legacy compatibility entry point retained to produce an actionable error.
/// v2 backends cannot enumerate old scopes; callers must supply the old SQLite
/// path to [`crate::storage::import::scope_move::LegacyScopeInventory::read`]