Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(storage): implement scope move inventory copy

Steven Enamakel43027bb8b4cb
diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rs
index 9c112c05c7..ce15f86ddb 100644
--- a/crates/openhuman-core/src/storage/import/mod.rs
+++ b/crates/openhuman-core/src/storage/import/mod.rs
@@ -52,6 +52,8 @@
pub(crate) mod layout_db;
#[cfg(feature = "legacy-import")]
pub(crate) mod layout_status;
+#[cfg(feature = "legacy-import")]
+pub(crate) mod scope_move;
pub mod live;
mod schemas;
#[cfg(feature = "legacy-import")]
diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rs
new file mode 100644
index 0000000000..430cb31c83
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/scope_move.rs
@@ -0,0 +1,266 @@
+//! Read-only inventory of records in the v1 shared SQLite storage database.
+//!
+//! Scope enumeration was a v1 driver capability. The v2 backend intentionally
+//! has no equivalent, so a host migration must identify its legacy database
+//! explicitly and decode its rows here before writing them through typed v2
+//! storage ports. This reader never changes the source file.
+
+use std::path::Path;
+
+use crate::storage::rusqlite::{self, Connection, OpenFlags};
+use anyhow::{Context, Result, bail};
+use serde_json::Value;
+use tinystoragedrivers::Meta;
+
+/// One live document row from the v1 storage database.
+#[derive(Debug, Clone, PartialEq)]
+pub struct LegacyDocument {
+ /// The old scope string, which may not parse as a v2 [`Scope`].
+ pub scope: String,
+ /// Collection name.
+ pub collection: String,
+ /// Document id.
+ pub id: String,
+ /// Decoded JSON body.
+ pub body: Value,
+ /// Persisted driver metadata.
+ pub meta: Meta,
+}
+
+/// One stream and its retained entries from the v1 database.
+#[derive(Debug, Clone, PartialEq)]
+pub struct LegacyStream {
+ /// The old scope string.
+ pub scope: String,
+ /// Stream name.
+ pub name: String,
+ /// First retained sequence number.
+ pub base: u64,
+ /// Whether the stream was sealed.
+ pub sealed: bool,
+ /// Seal time when available.
+ pub sealed_ms: Option<i64>,
+ /// Ordered entries, each paired with its original sequence number.
+ pub entries: Vec<(u64, Value)>,
+}
+
+/// One blob row from the v1 database.
+#[derive(Debug, Clone, PartialEq)]
+pub struct LegacyBlob {
+ /// The old scope string.
+ pub scope: String,
+ /// Blob key.
+ pub key: String,
+ /// Content type when recorded.
+ pub content_type: Option<String>,
+ /// Absolute expiry when recorded.
+ pub expires_ms: Option<i64>,
+ /// Original bytes.
+ pub bytes: Vec<u8>,
+}
+
+/// A complete, immutable snapshot of the v1 rows needed for a scope move.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct LegacyScopeInventory {
+ /// Live document rows.
+ pub documents: Vec<LegacyDocument>,
+ /// Streams and their entries.
+ pub streams: Vec<LegacyStream>,
+ /// Blobs.
+ pub blobs: Vec<LegacyBlob>,
+}
+
+impl LegacyScopeInventory {
+ /// Reads the legacy database without creating, migrating, or mutating it.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error for an unreadable database, malformed JSON, duplicate
+ /// or inconsistent stream sequences, or a missing required v1 table.
+ pub fn read(path: &Path) -> Result<Self> {
+ let conn = Connection::open_with_flags(
+ path,
+ OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
+ )
+ .with_context(|| format!("open legacy storage database {} read-only", path.display()))?;
+ let mut inventory = Self::default();
+
+ if table_exists(&conn, "docs")? {
+ let mut statement = conn.prepare(
+ "SELECT scope, coll, id, schema, created_ms, updated_ms, expires_ms, deleted, body
+ FROM docs ORDER BY scope, coll, id",
+ )?;
+ let rows = statement.query_map([], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, String>(2)?,
+ row.get::<_, u32>(3)?,
+ row.get::<_, i64>(4)?,
+ row.get::<_, i64>(5)?,
+ row.get::<_, Option<i64>>(6)?,
+ row.get::<_, bool>(7)?,
+ row.get::<_, String>(8)?,
+ ))
+ })?;
+ for row in rows {
+ let (
+ scope,
+ collection,
+ id,
+ schema,
+ created_ms,
+ updated_ms,
+ expires_ms,
+ deleted,
+ body,
+ ) = row?;
+ if deleted {
+ continue;
+ }
+ inventory.documents.push(LegacyDocument {
+ scope,
+ collection,
+ id,
+ body: serde_json::from_str(&body).context("decode legacy document body")?,
+ meta: Meta {
+ schema,
+ created_ms,
+ updated_ms,
+ expires_ms,
+ },
+ });
+ }
+ }
+
+ let mut heads = std::collections::BTreeMap::new();
+ if table_exists(&conn, "stream_heads")? {
+ let mut statement = conn.prepare(
+ "SELECT scope, name, base, len, sealed, sealed_ms
+ FROM stream_heads ORDER BY scope, name",
+ )?;
+ let rows = statement.query_map([], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, u64>(2)?,
+ row.get::<_, u64>(3)?,
+ row.get::<_, bool>(4)?,
+ row.get::<_, Option<i64>>(5)?,
+ ))
+ })?;
+ for row in rows {
+ let (scope, name, base, len, sealed, sealed_ms) = row?;
+ heads.insert((scope, name), (base, len, sealed, sealed_ms));
+ }
+ }
+ let mut stream_entries: std::collections::BTreeMap<(String, String), Vec<(u64, Value)>> =
+ std::collections::BTreeMap::new();
+ if table_exists(&conn, "streams")? {
+ let mut statement = conn
+ .prepare("SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq")?;
+ let rows = statement.query_map([], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, u64>(2)?,
+ row.get::<_, String>(3)?,
+ ))
+ })?;
+ for row in rows {
+ let (scope, name, seq, body) = row?;
+ stream_entries.entry((scope, name)).or_default().push((
+ seq,
+ serde_json::from_str(&body).context("decode legacy stream entry")?,
+ ));
+ }
+ }
+ for (key, entries) in stream_entries {
+ let (base, len, sealed, sealed_ms) = heads.remove(&key).unwrap_or((
+ entries.first().map_or(0, |(seq, _)| *seq),
+ entries.len() as u64,
+ false,
+ None,
+ ));
+ validate_stream(&key.0, &key.1, base, len, &entries)?;
+ inventory.streams.push(LegacyStream {
+ scope: key.0,
+ name: key.1,
+ base,
+ sealed,
+ sealed_ms,
+ entries,
+ });
+ }
+ for ((scope, name), (base, len, sealed, sealed_ms)) in heads {
+ validate_stream(&scope, &name, base, len, &[])?;
+ inventory.streams.push(LegacyStream {
+ scope,
+ name,
+ base,
+ sealed,
+ sealed_ms,
+ entries: Vec::new(),
+ });
+ }
+
+ if table_exists(&conn, "blobs")? {
+ let mut statement = conn.prepare(
+ "SELECT scope, key, content_type, expires_ms, bytes FROM blobs ORDER BY scope, key",
+ )?;
+ let rows = statement.query_map([], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, Option<String>>(2)?,
+ row.get::<_, Option<i64>>(3)?,
+ row.get::<_, Vec<u8>>(4)?,
+ ))
+ })?;
+ for row in rows {
+ let (scope, key, content_type, expires_ms, bytes) = row?;
+ inventory.blobs.push(LegacyBlob {
+ scope,
+ key,
+ content_type,
+ expires_ms,
+ bytes,
+ });
+ }
+ }
+ Ok(inventory)
+ }
+}
+
+fn table_exists(conn: &Connection, name: &str) -> rusqlite::Result<bool> {
+ conn.query_row(
+ "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
+ [name],
+ |row| row.get(0),
+ )
+}
+
+fn validate_stream(
+ scope: &str,
+ name: &str,
+ base: u64,
+ len: u64,
+ entries: &[(u64, Value)],
+) -> Result<()> {
+ let expected_len =
+ u64::try_from(entries.len()).context("legacy stream has too many entries")?;
+ if expected_len != len {
+ bail!("legacy stream {scope}/{name} has head length {len} but {expected_len} rows");
+ }
+ for (offset, (actual, _)) in entries.iter().enumerate() {
+ let expected = base + offset as u64;
+ if *actual != expected {
+ bail!("legacy stream {scope}/{name} is not contiguous at sequence {expected}");
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+#[path = "scope_move_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/storage/import/scope_move_tests.rs b/crates/openhuman-core/src/storage/import/scope_move_tests.rs
new file mode 100644
index 0000000000..4a33cbd20e
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/scope_move_tests.rs
@@ -0,0 +1,70 @@
+use super::*;
+use crate::storage::rusqlite::Connection;
+use std::io::Read;
+
+#[test]
+fn inventory_reads_all_legacy_families_without_changing_the_database() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("legacy.db");
+ let conn = Connection::open(&path).unwrap();
+ conn.execute_batch(
+ "CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,
+ created_ms INTEGER, updated_ms INTEGER, expires_ms INTEGER, deleted INTEGER, body TEXT,
+ PRIMARY KEY(scope, coll, id));
+ INSERT INTO docs VALUES('alice~agent','notes','n1',1,4,100,200,900,0,'{\"x\":1}');
+ INSERT INTO docs VALUES('alice~agent','notes','gone',1,4,100,200,NULL,1,'{}');
+ CREATE TABLE stream_heads(scope TEXT, name TEXT, base INTEGER, len INTEGER,
+ sealed INTEGER, sealed_ms INTEGER, last_append_ms INTEGER, PRIMARY KEY(scope,name));
+ INSERT INTO stream_heads VALUES('alice~agent','events',3,2,1,700,600);
+ CREATE TABLE streams(scope TEXT, name TEXT, seq INTEGER, body TEXT,
+ PRIMARY KEY(scope,name,seq));
+ INSERT INTO streams VALUES('alice~agent','events',3,'{\"n\":1}');
+ INSERT INTO streams VALUES('alice~agent','events',4,'{\"n\":2}');
+ CREATE TABLE blobs(scope TEXT, key TEXT, content_type TEXT, size INTEGER, sha256 BLOB,
+ expires_ms INTEGER, bytes BLOB, PRIMARY KEY(scope,key));
+ INSERT INTO blobs VALUES('alice~agent','b1','application/octet-stream',3,x'010203',NULL,x'010203');",
+ )
+ .unwrap();
+ drop(conn);
+ let before = sha256(&path);
+
+ let inventory = LegacyScopeInventory::read(&path).unwrap();
+
+ assert_eq!(sha256(&path), before);
+ assert_eq!(inventory.documents.len(), 1);
+ assert_eq!(inventory.documents[0].meta.schema, 4);
+ assert_eq!(inventory.documents[0].meta.created_ms, 100);
+ assert_eq!(inventory.documents[0].meta.updated_ms, 200);
+ assert_eq!(inventory.documents[0].meta.expires_ms, Some(900));
+ assert_eq!(inventory.streams.len(), 1);
+ assert_eq!(inventory.streams[0].base, 3);
+ assert!(inventory.streams[0].sealed);
+ assert_eq!(inventory.streams[0].sealed_ms, Some(700));
+ assert_eq!(inventory.streams[0].entries[1].0, 4);
+ assert_eq!(inventory.blobs[0].bytes, [1, 2, 3]);
+}
+
+#[test]
+fn inventory_rejects_stream_gaps_instead_of_silently_resequencing() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("legacy.db");
+ let conn = Connection::open(&path).unwrap();
+ conn.execute_batch(
+ "CREATE TABLE streams(scope TEXT, name TEXT, seq INTEGER, body TEXT);
+ INSERT INTO streams VALUES('agent','events',0,'{}');
+ INSERT INTO streams VALUES('agent','events',2,'{}');",
+ )
+ .unwrap();
+ drop(conn);
+
+ let error = LegacyScopeInventory::read(&path).unwrap_err();
+ assert!(error.to_string().contains("not contiguous"));
+}
+
+fn sha256(path: &Path) -> Vec<u8> {
+ let mut file = std::fs::File::open(path).unwrap();
+ let mut bytes = Vec::new();
+ file.read_to_end(&mut bytes).unwrap();
+ use sha2::Digest;
+ sha2::Sha256::digest(bytes).to_vec()
+}
diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rs
index b14e708ca7..f4bd71fc1b 100644
--- a/crates/openhuman-core/src/storage/scope_move.rs
+++ b/crates/openhuman-core/src/storage/scope_move.rs
@@ -8,35 +8,18 @@
//! `local`, `u:<id>` and a principal under either (`/agent:<id>`,
//! `/module:<id>`), see [`super::scope`].
//!
-//! Owner migration steps run inside one scope, so they cannot move a record
-//! between scopes. The host does it first, here: for each v1 scope found by
-//! `StorageBackend::scopes`, copy every document (with its schema, creation
-//! time and expiry), stream (and its sealed state) and blob into the v2 scope
-//! it maps to, then remove the v1 copy. The owners' steps then rewrite names
-//! and shapes inside the new scope.
-//!
-//! * **Resumable.** A document is written only when absent and removed from
-//! the source after its collection is copied, a stream continues from the
-//! target's length, a blob is skipped when the target already holds it, and
-//! the source scope is deleted last. A crash re-runs the same copy to the same
-//! result. Completion is recorded in the target scope's `tsd.migrations`
-//! (owner `host`, step `scope_move:<source scope>`).
-//! * **Merging.** Several v1 scopes can map to one v2 scope (the
-//! `module:<id>@<base>` of each acting agent all become the tenant's
-//! `module:<id>`). A record already present in the target wins.
-//! * **Hashed scopes.** A `sha256:` / `profile-sha256:` / `module-sha256:` v1
-//! scope cannot be inverted. It maps only when a known agent, profile or
-//! module id hashes to it (the recorded agents, the profile registry and the
-//! module registry); any other is left in place and logged.
+//! A v2 backend does not enumerate legacy scopes. The host therefore reads an
+//! explicit SQLite source through [`crate::storage::import::scope_move`] and
+//! copies its typed inventory into mapped v2 scopes. The source is retained
+//! unchanged for operator verification; unknown scope names are reported and
+//! unsupported fidelity cases fail explicitly.
use std::collections::HashSet;
use std::sync::Arc;
use sha2::Digest;
-use tinystoragedrivers::schema::Migrations;
use tinystoragedrivers::{
- Capability, DocumentStore, DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Scope,
- ScopedStorage, StorageBackend, StorageError, WriteOptions,
+ DocumentStore, ErrorKind, Precondition, Scope, StorageBackend, WriteOptions,
};
use super::scope::{scope_for_agent, scope_for_profile, tenant_scope};
@@ -44,9 +27,6 @@ use super::scope::{scope_for_agent, scope_for_profile, tenant_scope};
/// The `tsd.migrations` owner the move records under.
pub const MIGRATION_OWNER: &str = "host";
-const PAGE: usize = 200;
-const STREAM_PAGE: usize = 500;
-
/// The ids a hashed v1 scope can be matched against.
#[derive(Debug, Default, Clone)]
pub struct Known {
@@ -167,268 +147,160 @@ pub struct MoveReport {
pub blobs: u64,
}
-/// Reads the ids hashed scopes are matched against.
-pub async fn known_ids(backend: &Arc<dyn StorageBackend>) -> Known {
- let mut known = Known::default();
- if let Ok(local) = backend.for_scope(&Scope::local()) {
- if let Ok(found) = ids_of(local.documents(), super::agents::AGENTS).await {
- known.agents = found;
- }
- }
- if let Ok(cluster) = backend.for_scope(&Scope::cluster()) {
- if let Ok(found) = ids_of(cluster.documents(), "profiles").await {
- known.profiles = found;
- }
- }
- #[cfg(feature = "modules")]
- {
- known.modules = crate::modules::registry::ALL
- .iter()
- .map(|record| record.id.to_string())
- .collect();
- }
- known
-}
-
-async fn ids_of(
- docs: &Arc<dyn DocumentStore>,
- collection: &str,
-) -> Result<Vec<String>, StorageError> {
- // Absent collections read as empty; only names that exist are listed.
- match docs.collections(collection).await {
- Ok(found) if !found.iter().any(|name| name == collection) => return Ok(Vec::new()),
- Ok(_) | Err(_) => {}
- }
- Ok(docs
- .query_all(collection, &Query::all())
- .await?
- .into_iter()
- .map(|doc| doc.id)
- .collect())
-}
-
-/// Moves every v1 scope of `backend` to its v2 scope. See the module docs.
+/// Copies the rows in a source inventory to their mapped v2 scopes.
+///
+/// The source is deliberately not deleted. Physical inventory readers are
+/// read-only, and source cleanup belongs to the operator after verifying the
+/// target. Unknown scope names are returned for explicit operator handling.
///
/// # Errors
///
-/// The first storage error. Nothing is lost: re-running resumes.
-pub async fn run(backend: &Arc<dyn StorageBackend>) -> Result<MoveReport, StorageError> {
+/// Fails closed on non-zero stream bases, expiring blobs (the blob port cannot
+/// preserve an absolute expiry), malformed source data, or a target write
+/// error. Re-running is idempotent for documents/blobs and verifies stream
+/// prefixes before appending missing entries.
+#[cfg(feature = "legacy-import")]
+pub async fn copy_inventory(
+ backend: &Arc<dyn StorageBackend>,
+ known: &Known,
+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,
+) -> anyhow::Result<MoveReport> {
+ use anyhow::{Context, bail};
+ use tinystoragedrivers::CollectionSpec;
+
let mut report = MoveReport::default();
- if !backend
- .capabilities()
- .contains(Capability::ScopeEnumeration)
- {
- tracing::debug!(
- target: "openhuman::storage",
- driver = backend.driver(),
- "[storage][scope-move] backend cannot enumerate scopes; nothing to move"
- );
- return Ok(report);
+ let mut source_scopes = HashSet::new();
+ for row in &inventory.documents {
+ source_scopes.insert(row.scope.as_str());
}
- let sources = backend.scopes("").await?;
- let candidates: Vec<&Scope> = sources
- .iter()
- .filter(|scope| scope.parsed().is_err())
- .collect();
- if candidates.is_empty() {
- return Ok(report);
+ for row in &inventory.streams {
+ source_scopes.insert(row.scope.as_str());
}
- let known = known_ids(backend).await;
- for src in candidates {
- match map_scope(src.as_str(), &known) {
- Mapping::V2 => {}
+ for row in &inventory.blobs {
+ source_scopes.insert(row.scope.as_str());
+ }
+ for source in source_scopes {
+ let target_scope = match map_scope(source, known) {
+ Mapping::V2 => continue,
Mapping::Unknown => {
- tracing::warn!(
- target: "openhuman::storage",
- scope = src.as_str(),
- "[storage][scope-move] no known id maps this v1 scope; left in place"
- );
- report.unknown.push(src.as_str().to_owned());
+ report.unknown.push(source.to_owned());
+ continue;
}
- Mapping::Move(target) => {
- if move_scope(backend, src, &target, &mut report).await? {
- report.moved += 1;
- } else {
- report.already_done += 1;
- }
+ Mapping::Move(scope) => scope,
+ };
+ let target = backend.for_scope(&target_scope)?;
+ let docs = target.documents();
+ for row in inventory.documents.iter().filter(|row| row.scope == source) {
+ docs.ensure_collection(&CollectionSpec::new(&row.collection))
+ .await?;
+ let mut options = WriteOptions::new();
+ if row.meta.schema != 0 {
+ options.schema = row.meta.schema;
+ }
+ if row.meta.created_ms != 0 {
+ options.created_ms = Some(row.meta.created_ms);
+ }
+ options.expires_ms = row.meta.expires_ms;
+ match docs
+ .put_with(
+ &row.collection,
+ &row.id,
+ row.body.clone(),
+ Precondition::Absent,
+ options,
+ )
+ .await
+ {
+ Ok(_) => report.documents += 1,
+ Err(error)
+ if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => {}
+ Err(error) => return Err(error.into()),
}
}
- }
- tracing::info!(
- target: "openhuman::storage",
- moved = report.moved,
- documents = report.documents,
- entries = report.entries,
- blobs = report.blobs,
- unknown = report.unknown.len(),
- "[storage][scope-move] finished"
- );
- Ok(report)
-}
-
-fn step_id(src: &Scope) -> String {
- format!("scope_move:{}", src.as_str())
-}
-
-/// Moves one scope. `false` when it was already recorded as moved.
-async fn move_scope(
- backend: &Arc<dyn StorageBackend>,
- src: &Scope,
- target: &Scope,
- report: &mut MoveReport,
-) -> Result<bool, StorageError> {
- let to = backend.for_scope(target)?;
- let migrations = Migrations::new(&to);
- let step = step_id(src);
- to.documents()
- .ensure_collection(&tinystoragedrivers::CollectionSpec::new(
- tinystoragedrivers::MIGRATIONS_COLLECTION,
- ))
- .await?;
- if migrations.done(MIGRATION_OWNER, &step).await? {
- // The copy finished earlier; make sure no source records remain.
- let from = backend.for_scope(src)?;
- purge(&from).await?;
- return Ok(false);
- }
- tracing::info!(
- target: "openhuman::storage",
- from = src.as_str(),
- to = target.as_str(),
- "[storage][scope-move] moving a v1 scope"
- );
- let from = backend.for_scope(src)?;
- copy_documents(&from, &to, report).await?;
- copy_streams(&from, &to, report).await?;
- copy_blobs(&from, &to, report).await?;
- migrations.mark_done(MIGRATION_OWNER, &step).await?;
- purge(&from).await?;
- Ok(true)
-}
-
-async fn copy_documents(
- from: &ScopedStorage,
- to: &ScopedStorage,
- report: &mut MoveReport,
-) -> Result<(), StorageError> {
- let source = from.documents();
- let target = to.documents();
- let metadata = source.capabilities().contains(Capability::RecordMetadata)
- && target.capabilities().contains(Capability::RecordMetadata);
- for collection in source.collections("").await? {
- target
- .ensure_collection(&tinystoragedrivers::CollectionSpec::new(&collection))
- .await?;
- let mut cursor = None;
- loop {
- let mut query = Query::filter(Filter::default())
- .sort(tinystoragedrivers::Sort::asc(tinystoragedrivers::ID_FIELD))
- .limit(PAGE);
- if let Some(after) = cursor.take() {
- query = query.after(after);
+ let streams = target.streams();
+ for row in inventory.streams.iter().filter(|row| row.scope == source) {
+ if row.base != 0 {
+ bail!(
+ "cannot preserve non-zero base {} for legacy stream {source}/{} with the v2 append port",
+ row.base,
+ row.name
+ );
}
- let page = source.query(&collection, &query).await?;
- for doc in page.items {
- let options = if metadata {
- let mut options = WriteOptions::new().created_ms(doc.meta.created_ms);
- options.schema = doc.meta.schema;
- options.expires_ms = doc.meta.expires_ms;
- options
- } else {
- WriteOptions::new()
- };
- match target
- .put_with(&collection, &doc.id, doc.doc, Precondition::Absent, options)
- .await
+ let existing = streams.len(&row.name).await?;
+ if existing > row.entries.len() as u64 {
+ bail!(
+ "target stream {source}/{} is longer than the legacy source",
+ row.name
+ );
+ }
+ if existing > 0 {
+ let prefix = streams.read_window(&row.name, 0, existing as usize).await?;
+ if prefix.len() != existing as usize
+ || prefix
+ .iter()
+ .zip(&row.entries)
+ .any(|(old, (seq, value))| old.offset != *seq || old.value != *value)
{
- Ok(_) => report.documents += 1,
- Err(error)
- if matches!(
- error.kind(),
- ErrorKind::Conflict | ErrorKind::AlreadyExists
- ) => {}
- Err(error) => return Err(error),
+ bail!(
+ "target stream {source}/{} conflicts with the legacy prefix",
+ row.name
+ );
}
}
- match page.next {
- Some(next) => cursor = Some(next),
- None => break,
+ let tail = &row.entries[existing as usize..];
+ if !tail.is_empty() {
+ streams
+ .append_batch(
+ &row.name,
+ tail.iter().map(|(_, value)| value.clone()).collect(),
+ )
+ .await?;
+ report.entries += tail.len() as u64;
}
- }
- }
- Ok(())
-}
-
-async fn copy_streams(
- from: &ScopedStorage,
- to: &ScopedStorage,
- report: &mut MoveReport,
-) -> Result<(), StorageError> {
- let source = from.streams();
- let target = to.streams();
- for name in source.streams("").await? {
- let info = source.info(&name).await?;
- let base = info.as_ref().map_or(0, |info| info.base);
- let mut offset = base + target.len(&name).await?;
- loop {
- let window = source.read_window(&name, offset, STREAM_PAGE).await?;
- if window.is_empty() {
- break;
+ if row.sealed {
+ match streams.seal(&row.name).await {
+ Ok(_) => {}
+ Err(error) if matches!(error.kind(), ErrorKind::Unsupported(_)) => bail!(
+ "target cannot preserve sealed state for legacy stream {source}/{}",
+ row.name
+ ),
+ Err(error) => return Err(error.into()),
+ }
}
- offset = window.last().map_or(offset, |entry| entry.offset + 1);
- let count = window.len() as u64;
- target
- .append_batch(&name, window.into_iter().map(|entry| entry.value).collect())
- .await?;
- report.entries += count;
}
- if info.is_some_and(|info| info.sealed_ms.is_some()) {
- match target.seal(&name).await {
- Ok(_) => {}
- Err(error) if matches!(error.kind(), ErrorKind::Unsupported(_)) => {}
- Err(error) => return Err(error),
+ let blobs = target.blobs();
+ for row in inventory.blobs.iter().filter(|row| row.scope == source) {
+ if row.expires_ms.is_some() {
+ bail!(
+ "target blob port cannot preserve absolute expiry for legacy blob {source}/{}",
+ row.key
+ );
+ }
+ if blobs.head(&row.key).await?.is_none() {
+ blobs
+ .put(&row.key, row.bytes.clone(), row.content_type.as_deref())
+ .await
+ .with_context(|| format!("copy legacy blob {source}/{}", row.key))?;
+ report.blobs += 1;
}
}
+ report.moved += 1;
}
- Ok(())
-}
-
-async fn copy_blobs(
- from: &ScopedStorage,
- to: &ScopedStorage,
- report: &mut MoveReport,
-) -> Result<(), StorageError> {
- let source = from.blobs();
- let target = to.blobs();
- for meta in source.list("").await? {
- if target.head(&meta.key).await?.is_none() {
- let Some(blob) = source.get(&meta.key).await? else {
- continue;
- };
- target
- .put(&meta.key, blob.bytes, meta.content_type.as_deref())
- .await?;
- report.blobs += 1;
- }
- }
- Ok(())
+ Ok(report)
}
-/// Removes every record of a moved scope.
-async fn purge(from: &ScopedStorage) -> Result<(), StorageError> {
- let docs = from.documents();
- for collection in docs.collections("").await? {
- docs.drop_collection(&collection).await?;
- }
- let streams = from.streams();
- for name in streams.streams("").await? {
- streams.delete_stream(&name).await?;
- }
- let blobs = from.blobs();
- for meta in blobs.list("").await? {
- blobs.delete(&meta.key).await?;
- }
- Ok(())
+/// Legacy compatibility entry point retained to produce an actionable error.
+/// v2 backends cannot enumerate old scopes; callers must supply the old SQLite
+/// path to [`crate::storage::import::scope_move::LegacyScopeInventory::read`]
+/// and then pass that inventory to [`copy_inventory`].
+#[deprecated(note = "use an explicit read-only legacy scope inventory")]
+pub async fn run(
+ _backend: &Arc<dyn StorageBackend>,
+) -> Result<MoveReport, tinystoragedrivers::StorageError> {
+ Err(tinystoragedrivers::StorageError::unsupported(
+ tinystoragedrivers::Capability::ScopeEnumeration,
+ "v2 backends do not enumerate legacy scopes; supply an explicit legacy database inventory",
+ ))
}
/// The scopes of `scopes` that are not in the v2 grammar (for tests and the
diff --git a/crates/openhuman-core/src/storage/scope_move_tests.rs b/crates/openhuman-core/src/storage/scope_move_tests.rs
index bc6bf79966..feee3cb604 100644
--- a/crates/openhuman-core/src/storage/scope_move_tests.rs
+++ b/crates/openhuman-core/src/storage/scope_move_tests.rs
@@ -1,6 +1,8 @@
use super::*;
+#[cfg(feature = "legacy-import")]
use serde_json::json;
-use tinystoragedrivers::MemoryStorage;
+#[cfg(feature = "legacy-import")]
+use tinystoragedrivers::{MemoryStorage, Meta, Precondition};
fn known() -> Known {
Known {
@@ -56,29 +58,37 @@ fn v2_scopes_are_left_alone() {
}
#[tokio::test]
+#[cfg(feature = "legacy-import")]
async fn a_v1_scope_moves_its_documents_streams_and_blobs() {
let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());
- let old = backend
- .for_scope(&Scope::new("alice~asha").unwrap())
- .unwrap();
- old.documents()
- .put("notes", "n1", json!({"a": 1}), Precondition::None)
- .await
- .unwrap();
- old.streams()
- .append("journal", json!({"k": 1}))
- .await
- .unwrap();
- old.streams()
- .append("journal", json!({"k": 2}))
+ let source = "alice~asha";
+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory {
+ documents: vec![crate::storage::import::scope_move::LegacyDocument {
+ scope: source.into(),
+ collection: "notes".into(),
+ id: "n1".into(),
+ body: json!({"a": 1}),
+ meta: Meta::default(),
+ }],
+ streams: vec![crate::storage::import::scope_move::LegacyStream {
+ scope: source.into(),
+ name: "journal".into(),
+ base: 0,
+ sealed: false,
+ sealed_ms: None,
+ entries: vec![(0, json!({"k":1})), (1, json!({"k":2}))],
+ }],
+ blobs: vec![crate::storage::import::scope_move::LegacyBlob {
+ scope: source.into(),
+ key: "b/1".into(),
+ content_type: Some("text/plain".into()),
+ expires_ms: None,
+ bytes: b"hi".to_vec(),
+ }],
+ };
+ let report = copy_inventory(&backend, &known(), &inventory)
.await
.unwrap();
- old.blobs()
- .put("b/1", b"hi".to_vec(), Some("text/plain"))
- .await
- .unwrap();
-
- let report = run(&backend).await.unwrap();
assert_eq!(report.moved, 1);
assert!(report.documents >= 1);
assert_eq!(report.entries, 2);
@@ -99,17 +109,19 @@ async fn a_v1_scope_moves_its_documents_streams_and_blobs() {
assert_eq!(new.streams().len("journal").await.unwrap(), 2);
assert_eq!(new.blobs().get("b/1").await.unwrap().unwrap().bytes, b"hi");
- // The v1 scope is gone and a second run is a no-op.
- let old = backend
- .for_scope(&Scope::new("alice~asha").unwrap())
+ // The explicit source inventory remains available for operator review.
+ assert_eq!(inventory.documents.len(), 1);
+ assert_eq!(inventory.streams[0].entries.len(), 2);
+ // Repeating the typed copy preserves rows without appending stream entries.
+ let again = copy_inventory(&backend, &known(), &inventory)
+ .await
.unwrap();
- assert!(old.documents().get("notes", "n1").await.unwrap().is_none());
- assert_eq!(old.streams().len("journal").await.unwrap(), 0);
- let again = run(&backend).await.unwrap();
- assert_eq!(again.moved, 0);
+ assert_eq!(again.entries, 0);
+ assert_eq!(new.streams().len("journal").await.unwrap(), 2);
}
#[tokio::test]
+#[cfg(feature = "legacy-import")]
async fn a_record_already_in_the_target_wins_a_merge() {
let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());
let target = backend.for_scope(&scope_for_agent("asha")).unwrap();
@@ -118,12 +130,19 @@ async fn a_record_already_in_the_target_wins_a_merge() {
.put("notes", "n", json!({"from": "v2"}), Precondition::None)
.await
.unwrap();
- let old = backend.for_scope(&Scope::new("asha").unwrap()).unwrap();
- old.documents()
- .put("notes", "n", json!({"from": "v1"}), Precondition::None)
+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory {
+ documents: vec![crate::storage::import::scope_move::LegacyDocument {
+ scope: "asha".into(),
+ collection: "notes".into(),
+ id: "n".into(),
+ body: json!({"from": "v1"}),
+ meta: Meta::default(),
+ }],
+ ..Default::default()
+ };
+ copy_inventory(&backend, &known(), &inventory)
.await
.unwrap();
- run(&backend).await.unwrap();
assert_eq!(
target
.documents()