The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(storage): implement scope move inventory copy
Steven Enamakel43027bb8b4cb
diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rsindex 9c112c05c7..ce15f86ddb 100644--- a/crates/openhuman-core/src/storage/import/mod.rs+++ b/crates/openhuman-core/src/storage/import/mod.rs@@ -52,6 +52,8 @@pub(crate) mod layout_db;#[cfg(feature = "legacy-import")]pub(crate) mod layout_status;+#[cfg(feature = "legacy-import")]+pub(crate) mod scope_move;pub mod live;mod schemas;#[cfg(feature = "legacy-import")]diff --git a/crates/openhuman-core/src/storage/import/scope_move.rs b/crates/openhuman-core/src/storage/import/scope_move.rsnew file mode 100644index 0000000000..430cb31c83--- /dev/null+++ b/crates/openhuman-core/src/storage/import/scope_move.rs@@ -0,0 +1,266 @@+//! Read-only inventory of records in the v1 shared SQLite storage database.+//!+//! Scope enumeration was a v1 driver capability. The v2 backend intentionally+//! has no equivalent, so a host migration must identify its legacy database+//! explicitly and decode its rows here before writing them through typed v2+//! storage ports. This reader never changes the source file.++use std::path::Path;++use crate::storage::rusqlite::{self, Connection, OpenFlags};+use anyhow::{Context, Result, bail};+use serde_json::Value;+use tinystoragedrivers::Meta;++/// One live document row from the v1 storage database.+#[derive(Debug, Clone, PartialEq)]+pub struct LegacyDocument {+ /// The old scope string, which may not parse as a v2 [`Scope`].+ pub scope: String,+ /// Collection name.+ pub collection: String,+ /// Document id.+ pub id: String,+ /// Decoded JSON body.+ pub body: Value,+ /// Persisted driver metadata.+ pub meta: Meta,+}++/// One stream and its retained entries from the v1 database.+#[derive(Debug, Clone, PartialEq)]+pub struct LegacyStream {+ /// The old scope string.+ pub scope: String,+ /// Stream name.+ pub name: String,+ /// First retained sequence number.+ pub base: u64,+ /// Whether the stream was sealed.+ pub sealed: bool,+ /// Seal time when available.+ pub sealed_ms: Option<i64>,+ /// Ordered entries, each paired with its original sequence number.+ pub entries: Vec<(u64, Value)>,+}++/// One blob row from the v1 database.+#[derive(Debug, Clone, PartialEq)]+pub struct LegacyBlob {+ /// The old scope string.+ pub scope: String,+ /// Blob key.+ pub key: String,+ /// Content type when recorded.+ pub content_type: Option<String>,+ /// Absolute expiry when recorded.+ pub expires_ms: Option<i64>,+ /// Original bytes.+ pub bytes: Vec<u8>,+}++/// A complete, immutable snapshot of the v1 rows needed for a scope move.+#[derive(Debug, Clone, Default, PartialEq)]+pub struct LegacyScopeInventory {+ /// Live document rows.+ pub documents: Vec<LegacyDocument>,+ /// Streams and their entries.+ pub streams: Vec<LegacyStream>,+ /// Blobs.+ pub blobs: Vec<LegacyBlob>,+}++impl LegacyScopeInventory {+ /// Reads the legacy database without creating, migrating, or mutating it.+ ///+ /// # Errors+ ///+ /// Returns an error for an unreadable database, malformed JSON, duplicate+ /// or inconsistent stream sequences, or a missing required v1 table.+ pub fn read(path: &Path) -> Result<Self> {+ let conn = Connection::open_with_flags(+ path,+ OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,+ )+ .with_context(|| format!("open legacy storage database {} read-only", path.display()))?;+ let mut inventory = Self::default();++ if table_exists(&conn, "docs")? {+ let mut statement = conn.prepare(+ "SELECT scope, coll, id, schema, created_ms, updated_ms, expires_ms, deleted, body+ FROM docs ORDER BY scope, coll, id",+ )?;+ let rows = statement.query_map([], |row| {+ Ok((+ row.get::<_, String>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, String>(2)?,+ row.get::<_, u32>(3)?,+ row.get::<_, i64>(4)?,+ row.get::<_, i64>(5)?,+ row.get::<_, Option<i64>>(6)?,+ row.get::<_, bool>(7)?,+ row.get::<_, String>(8)?,+ ))+ })?;+ for row in rows {+ let (+ scope,+ collection,+ id,+ schema,+ created_ms,+ updated_ms,+ expires_ms,+ deleted,+ body,+ ) = row?;+ if deleted {+ continue;+ }+ inventory.documents.push(LegacyDocument {+ scope,+ collection,+ id,+ body: serde_json::from_str(&body).context("decode legacy document body")?,+ meta: Meta {+ schema,+ created_ms,+ updated_ms,+ expires_ms,+ },+ });+ }+ }++ let mut heads = std::collections::BTreeMap::new();+ if table_exists(&conn, "stream_heads")? {+ let mut statement = conn.prepare(+ "SELECT scope, name, base, len, sealed, sealed_ms+ FROM stream_heads ORDER BY scope, name",+ )?;+ let rows = statement.query_map([], |row| {+ Ok((+ row.get::<_, String>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, u64>(2)?,+ row.get::<_, u64>(3)?,+ row.get::<_, bool>(4)?,+ row.get::<_, Option<i64>>(5)?,+ ))+ })?;+ for row in rows {+ let (scope, name, base, len, sealed, sealed_ms) = row?;+ heads.insert((scope, name), (base, len, sealed, sealed_ms));+ }+ }+ let mut stream_entries: std::collections::BTreeMap<(String, String), Vec<(u64, Value)>> =+ std::collections::BTreeMap::new();+ if table_exists(&conn, "streams")? {+ let mut statement = conn+ .prepare("SELECT scope, name, seq, body FROM streams ORDER BY scope, name, seq")?;+ let rows = statement.query_map([], |row| {+ Ok((+ row.get::<_, String>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, u64>(2)?,+ row.get::<_, String>(3)?,+ ))+ })?;+ for row in rows {+ let (scope, name, seq, body) = row?;+ stream_entries.entry((scope, name)).or_default().push((+ seq,+ serde_json::from_str(&body).context("decode legacy stream entry")?,+ ));+ }+ }+ for (key, entries) in stream_entries {+ let (base, len, sealed, sealed_ms) = heads.remove(&key).unwrap_or((+ entries.first().map_or(0, |(seq, _)| *seq),+ entries.len() as u64,+ false,+ None,+ ));+ validate_stream(&key.0, &key.1, base, len, &entries)?;+ inventory.streams.push(LegacyStream {+ scope: key.0,+ name: key.1,+ base,+ sealed,+ sealed_ms,+ entries,+ });+ }+ for ((scope, name), (base, len, sealed, sealed_ms)) in heads {+ validate_stream(&scope, &name, base, len, &[])?;+ inventory.streams.push(LegacyStream {+ scope,+ name,+ base,+ sealed,+ sealed_ms,+ entries: Vec::new(),+ });+ }++ if table_exists(&conn, "blobs")? {+ let mut statement = conn.prepare(+ "SELECT scope, key, content_type, expires_ms, bytes FROM blobs ORDER BY scope, key",+ )?;+ let rows = statement.query_map([], |row| {+ Ok((+ row.get::<_, String>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, Option<String>>(2)?,+ row.get::<_, Option<i64>>(3)?,+ row.get::<_, Vec<u8>>(4)?,+ ))+ })?;+ for row in rows {+ let (scope, key, content_type, expires_ms, bytes) = row?;+ inventory.blobs.push(LegacyBlob {+ scope,+ key,+ content_type,+ expires_ms,+ bytes,+ });+ }+ }+ Ok(inventory)+ }+}++fn table_exists(conn: &Connection, name: &str) -> rusqlite::Result<bool> {+ conn.query_row(+ "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",+ [name],+ |row| row.get(0),+ )+}++fn validate_stream(+ scope: &str,+ name: &str,+ base: u64,+ len: u64,+ entries: &[(u64, Value)],+) -> Result<()> {+ let expected_len =+ u64::try_from(entries.len()).context("legacy stream has too many entries")?;+ if expected_len != len {+ bail!("legacy stream {scope}/{name} has head length {len} but {expected_len} rows");+ }+ for (offset, (actual, _)) in entries.iter().enumerate() {+ let expected = base + offset as u64;+ if *actual != expected {+ bail!("legacy stream {scope}/{name} is not contiguous at sequence {expected}");+ }+ }+ Ok(())+}++#[cfg(test)]+#[path = "scope_move_tests.rs"]+mod tests;diff --git a/crates/openhuman-core/src/storage/import/scope_move_tests.rs b/crates/openhuman-core/src/storage/import/scope_move_tests.rsnew file mode 100644index 0000000000..4a33cbd20e--- /dev/null+++ b/crates/openhuman-core/src/storage/import/scope_move_tests.rs@@ -0,0 +1,70 @@+use super::*;+use crate::storage::rusqlite::Connection;+use std::io::Read;++#[test]+fn inventory_reads_all_legacy_families_without_changing_the_database() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("legacy.db");+ let conn = Connection::open(&path).unwrap();+ conn.execute_batch(+ "CREATE TABLE docs(scope TEXT, coll TEXT, id TEXT, version INTEGER, schema INTEGER,+ created_ms INTEGER, updated_ms INTEGER, expires_ms INTEGER, deleted INTEGER, body TEXT,+ PRIMARY KEY(scope, coll, id));+ INSERT INTO docs VALUES('alice~agent','notes','n1',1,4,100,200,900,0,'{\"x\":1}');+ INSERT INTO docs VALUES('alice~agent','notes','gone',1,4,100,200,NULL,1,'{}');+ CREATE TABLE stream_heads(scope TEXT, name TEXT, base INTEGER, len INTEGER,+ sealed INTEGER, sealed_ms INTEGER, last_append_ms INTEGER, PRIMARY KEY(scope,name));+ INSERT INTO stream_heads VALUES('alice~agent','events',3,2,1,700,600);+ CREATE TABLE streams(scope TEXT, name TEXT, seq INTEGER, body TEXT,+ PRIMARY KEY(scope,name,seq));+ INSERT INTO streams VALUES('alice~agent','events',3,'{\"n\":1}');+ INSERT INTO streams VALUES('alice~agent','events',4,'{\"n\":2}');+ CREATE TABLE blobs(scope TEXT, key TEXT, content_type TEXT, size INTEGER, sha256 BLOB,+ expires_ms INTEGER, bytes BLOB, PRIMARY KEY(scope,key));+ INSERT INTO blobs VALUES('alice~agent','b1','application/octet-stream',3,x'010203',NULL,x'010203');",+ )+ .unwrap();+ drop(conn);+ let before = sha256(&path);++ let inventory = LegacyScopeInventory::read(&path).unwrap();++ assert_eq!(sha256(&path), before);+ assert_eq!(inventory.documents.len(), 1);+ assert_eq!(inventory.documents[0].meta.schema, 4);+ assert_eq!(inventory.documents[0].meta.created_ms, 100);+ assert_eq!(inventory.documents[0].meta.updated_ms, 200);+ assert_eq!(inventory.documents[0].meta.expires_ms, Some(900));+ assert_eq!(inventory.streams.len(), 1);+ assert_eq!(inventory.streams[0].base, 3);+ assert!(inventory.streams[0].sealed);+ assert_eq!(inventory.streams[0].sealed_ms, Some(700));+ assert_eq!(inventory.streams[0].entries[1].0, 4);+ assert_eq!(inventory.blobs[0].bytes, [1, 2, 3]);+}++#[test]+fn inventory_rejects_stream_gaps_instead_of_silently_resequencing() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("legacy.db");+ let conn = Connection::open(&path).unwrap();+ conn.execute_batch(+ "CREATE TABLE streams(scope TEXT, name TEXT, seq INTEGER, body TEXT);+ INSERT INTO streams VALUES('agent','events',0,'{}');+ INSERT INTO streams VALUES('agent','events',2,'{}');",+ )+ .unwrap();+ drop(conn);++ let error = LegacyScopeInventory::read(&path).unwrap_err();+ assert!(error.to_string().contains("not contiguous"));+}++fn sha256(path: &Path) -> Vec<u8> {+ let mut file = std::fs::File::open(path).unwrap();+ let mut bytes = Vec::new();+ file.read_to_end(&mut bytes).unwrap();+ use sha2::Digest;+ sha2::Sha256::digest(bytes).to_vec()+}diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rsindex b14e708ca7..f4bd71fc1b 100644--- a/crates/openhuman-core/src/storage/scope_move.rs+++ b/crates/openhuman-core/src/storage/scope_move.rs@@ -8,35 +8,18 @@//! `local`, `u:<id>` and a principal under either (`/agent:<id>`,//! `/module:<id>`), see [`super::scope`].//!-//! Owner migration steps run inside one scope, so they cannot move a record-//! between scopes. The host does it first, here: for each v1 scope found by-//! `StorageBackend::scopes`, copy every document (with its schema, creation-//! time and expiry), stream (and its sealed state) and blob into the v2 scope-//! it maps to, then remove the v1 copy. The owners' steps then rewrite names-//! and shapes inside the new scope.-//!-//! * **Resumable.** A document is written only when absent and removed from-//! the source after its collection is copied, a stream continues from the-//! target's length, a blob is skipped when the target already holds it, and-//! the source scope is deleted last. A crash re-runs the same copy to the same-//! result. Completion is recorded in the target scope's `tsd.migrations`-//! (owner `host`, step `scope_move:<source scope>`).-//! * **Merging.** Several v1 scopes can map to one v2 scope (the-//! `module:<id>@<base>` of each acting agent all become the tenant's-//! `module:<id>`). A record already present in the target wins.-//! * **Hashed scopes.** A `sha256:` / `profile-sha256:` / `module-sha256:` v1-//! scope cannot be inverted. It maps only when a known agent, profile or-//! module id hashes to it (the recorded agents, the profile registry and the-//! module registry); any other is left in place and logged.+//! A v2 backend does not enumerate legacy scopes. The host therefore reads an+//! explicit SQLite source through [`crate::storage::import::scope_move`] and+//! copies its typed inventory into mapped v2 scopes. The source is retained+//! unchanged for operator verification; unknown scope names are reported and+//! unsupported fidelity cases fail explicitly.use std::collections::HashSet;use std::sync::Arc;use sha2::Digest;-use tinystoragedrivers::schema::Migrations;use tinystoragedrivers::{- Capability, DocumentStore, DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Scope,- ScopedStorage, StorageBackend, StorageError, WriteOptions,+ DocumentStore, ErrorKind, Precondition, Scope, StorageBackend, WriteOptions,};use super::scope::{scope_for_agent, scope_for_profile, tenant_scope};@@ -44,9 +27,6 @@ use super::scope::{scope_for_agent, scope_for_profile, tenant_scope};/// The `tsd.migrations` owner the move records under.pub const MIGRATION_OWNER: &str = "host";-const PAGE: usize = 200;-const STREAM_PAGE: usize = 500;-/// The ids a hashed v1 scope can be matched against.#[derive(Debug, Default, Clone)]pub struct Known {@@ -167,268 +147,160 @@ pub struct MoveReport {pub blobs: u64,}-/// Reads the ids hashed scopes are matched against.-pub async fn known_ids(backend: &Arc<dyn StorageBackend>) -> Known {- let mut known = Known::default();- if let Ok(local) = backend.for_scope(&Scope::local()) {- if let Ok(found) = ids_of(local.documents(), super::agents::AGENTS).await {- known.agents = found;- }- }- if let Ok(cluster) = backend.for_scope(&Scope::cluster()) {- if let Ok(found) = ids_of(cluster.documents(), "profiles").await {- known.profiles = found;- }- }- #[cfg(feature = "modules")]- {- known.modules = crate::modules::registry::ALL- .iter()- .map(|record| record.id.to_string())- .collect();- }- known-}--async fn ids_of(- docs: &Arc<dyn DocumentStore>,- collection: &str,-) -> Result<Vec<String>, StorageError> {- // Absent collections read as empty; only names that exist are listed.- match docs.collections(collection).await {- Ok(found) if !found.iter().any(|name| name == collection) => return Ok(Vec::new()),- Ok(_) | Err(_) => {}- }- Ok(docs- .query_all(collection, &Query::all())- .await?- .into_iter()- .map(|doc| doc.id)- .collect())-}--/// Moves every v1 scope of `backend` to its v2 scope. See the module docs.+/// Copies the rows in a source inventory to their mapped v2 scopes.+///+/// The source is deliberately not deleted. Physical inventory readers are+/// read-only, and source cleanup belongs to the operator after verifying the+/// target. Unknown scope names are returned for explicit operator handling.////// # Errors///-/// The first storage error. Nothing is lost: re-running resumes.-pub async fn run(backend: &Arc<dyn StorageBackend>) -> Result<MoveReport, StorageError> {+/// Fails closed on non-zero stream bases, expiring blobs (the blob port cannot+/// preserve an absolute expiry), malformed source data, or a target write+/// error. Re-running is idempotent for documents/blobs and verifies stream+/// prefixes before appending missing entries.+#[cfg(feature = "legacy-import")]+pub async fn copy_inventory(+ backend: &Arc<dyn StorageBackend>,+ known: &Known,+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+) -> anyhow::Result<MoveReport> {+ use anyhow::{Context, bail};+ use tinystoragedrivers::CollectionSpec;+let mut report = MoveReport::default();- if !backend- .capabilities()- .contains(Capability::ScopeEnumeration)- {- tracing::debug!(- target: "openhuman::storage",- driver = backend.driver(),- "[storage][scope-move] backend cannot enumerate scopes; nothing to move"- );- return Ok(report);+ let mut source_scopes = HashSet::new();+ for row in &inventory.documents {+ source_scopes.insert(row.scope.as_str());}- let sources = backend.scopes("").await?;- let candidates: Vec<&Scope> = sources- .iter()- .filter(|scope| scope.parsed().is_err())- .collect();- if candidates.is_empty() {- return Ok(report);+ for row in &inventory.streams {+ source_scopes.insert(row.scope.as_str());}- let known = known_ids(backend).await;- for src in candidates {- match map_scope(src.as_str(), &known) {- Mapping::V2 => {}+ for row in &inventory.blobs {+ source_scopes.insert(row.scope.as_str());+ }+ for source in source_scopes {+ let target_scope = match map_scope(source, known) {+ Mapping::V2 => continue,Mapping::Unknown => {- tracing::warn!(- target: "openhuman::storage",- scope = src.as_str(),- "[storage][scope-move] no known id maps this v1 scope; left in place"- );- report.unknown.push(src.as_str().to_owned());+ report.unknown.push(source.to_owned());+ continue;}- Mapping::Move(target) => {- if move_scope(backend, src, &target, &mut report).await? {- report.moved += 1;- } else {- report.already_done += 1;- }+ Mapping::Move(scope) => scope,+ };+ let target = backend.for_scope(&target_scope)?;+ let docs = target.documents();+ for row in inventory.documents.iter().filter(|row| row.scope == source) {+ docs.ensure_collection(&CollectionSpec::new(&row.collection))+ .await?;+ let mut options = WriteOptions::new();+ if row.meta.schema != 0 {+ options.schema = row.meta.schema;+ }+ if row.meta.created_ms != 0 {+ options.created_ms = Some(row.meta.created_ms);+ }+ options.expires_ms = row.meta.expires_ms;+ match docs+ .put_with(+ &row.collection,+ &row.id,+ row.body.clone(),+ Precondition::Absent,+ options,+ )+ .await+ {+ Ok(_) => report.documents += 1,+ Err(error)+ if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => {}+ Err(error) => return Err(error.into()),}}- }- tracing::info!(- target: "openhuman::storage",- moved = report.moved,- documents = report.documents,- entries = report.entries,- blobs = report.blobs,- unknown = report.unknown.len(),- "[storage][scope-move] finished"- );- Ok(report)-}--fn step_id(src: &Scope) -> String {- format!("scope_move:{}", src.as_str())-}--/// Moves one scope. `false` when it was already recorded as moved.-async fn move_scope(- backend: &Arc<dyn StorageBackend>,- src: &Scope,- target: &Scope,- report: &mut MoveReport,-) -> Result<bool, StorageError> {- let to = backend.for_scope(target)?;- let migrations = Migrations::new(&to);- let step = step_id(src);- to.documents()- .ensure_collection(&tinystoragedrivers::CollectionSpec::new(- tinystoragedrivers::MIGRATIONS_COLLECTION,- ))- .await?;- if migrations.done(MIGRATION_OWNER, &step).await? {- // The copy finished earlier; make sure no source records remain.- let from = backend.for_scope(src)?;- purge(&from).await?;- return Ok(false);- }- tracing::info!(- target: "openhuman::storage",- from = src.as_str(),- to = target.as_str(),- "[storage][scope-move] moving a v1 scope"- );- let from = backend.for_scope(src)?;- copy_documents(&from, &to, report).await?;- copy_streams(&from, &to, report).await?;- copy_blobs(&from, &to, report).await?;- migrations.mark_done(MIGRATION_OWNER, &step).await?;- purge(&from).await?;- Ok(true)-}--async fn copy_documents(- from: &ScopedStorage,- to: &ScopedStorage,- report: &mut MoveReport,-) -> Result<(), StorageError> {- let source = from.documents();- let target = to.documents();- let metadata = source.capabilities().contains(Capability::RecordMetadata)- && target.capabilities().contains(Capability::RecordMetadata);- for collection in source.collections("").await? {- target- .ensure_collection(&tinystoragedrivers::CollectionSpec::new(&collection))- .await?;- let mut cursor = None;- loop {- let mut query = Query::filter(Filter::default())- .sort(tinystoragedrivers::Sort::asc(tinystoragedrivers::ID_FIELD))- .limit(PAGE);- if let Some(after) = cursor.take() {- query = query.after(after);+ let streams = target.streams();+ for row in inventory.streams.iter().filter(|row| row.scope == source) {+ if row.base != 0 {+ bail!(+ "cannot preserve non-zero base {} for legacy stream {source}/{} with the v2 append port",+ row.base,+ row.name+ );}- let page = source.query(&collection, &query).await?;- for doc in page.items {- let options = if metadata {- let mut options = WriteOptions::new().created_ms(doc.meta.created_ms);- options.schema = doc.meta.schema;- options.expires_ms = doc.meta.expires_ms;- options- } else {- WriteOptions::new()- };- match target- .put_with(&collection, &doc.id, doc.doc, Precondition::Absent, options)- .await+ let existing = streams.len(&row.name).await?;+ if existing > row.entries.len() as u64 {+ bail!(+ "target stream {source}/{} is longer than the legacy source",+ row.name+ );+ }+ if existing > 0 {+ let prefix = streams.read_window(&row.name, 0, existing as usize).await?;+ if prefix.len() != existing as usize+ || prefix+ .iter()+ .zip(&row.entries)+ .any(|(old, (seq, value))| old.offset != *seq || old.value != *value){- Ok(_) => report.documents += 1,- Err(error)- if matches!(- error.kind(),- ErrorKind::Conflict | ErrorKind::AlreadyExists- ) => {}- Err(error) => return Err(error),+ bail!(+ "target stream {source}/{} conflicts with the legacy prefix",+ row.name+ );}}- match page.next {- Some(next) => cursor = Some(next),- None => break,+ let tail = &row.entries[existing as usize..];+ if !tail.is_empty() {+ streams+ .append_batch(+ &row.name,+ tail.iter().map(|(_, value)| value.clone()).collect(),+ )+ .await?;+ report.entries += tail.len() as u64;}- }- }- Ok(())-}--async fn copy_streams(- from: &ScopedStorage,- to: &ScopedStorage,- report: &mut MoveReport,-) -> Result<(), StorageError> {- let source = from.streams();- let target = to.streams();- for name in source.streams("").await? {- let info = source.info(&name).await?;- let base = info.as_ref().map_or(0, |info| info.base);- let mut offset = base + target.len(&name).await?;- loop {- let window = source.read_window(&name, offset, STREAM_PAGE).await?;- if window.is_empty() {- break;+ if row.sealed {+ match streams.seal(&row.name).await {+ Ok(_) => {}+ Err(error) if matches!(error.kind(), ErrorKind::Unsupported(_)) => bail!(+ "target cannot preserve sealed state for legacy stream {source}/{}",+ row.name+ ),+ Err(error) => return Err(error.into()),+ }}- offset = window.last().map_or(offset, |entry| entry.offset + 1);- let count = window.len() as u64;- target- .append_batch(&name, window.into_iter().map(|entry| entry.value).collect())- .await?;- report.entries += count;}- if info.is_some_and(|info| info.sealed_ms.is_some()) {- match target.seal(&name).await {- Ok(_) => {}- Err(error) if matches!(error.kind(), ErrorKind::Unsupported(_)) => {}- Err(error) => return Err(error),+ let blobs = target.blobs();+ for row in inventory.blobs.iter().filter(|row| row.scope == source) {+ if row.expires_ms.is_some() {+ bail!(+ "target blob port cannot preserve absolute expiry for legacy blob {source}/{}",+ row.key+ );+ }+ if blobs.head(&row.key).await?.is_none() {+ blobs+ .put(&row.key, row.bytes.clone(), row.content_type.as_deref())+ .await+ .with_context(|| format!("copy legacy blob {source}/{}", row.key))?;+ report.blobs += 1;}}+ report.moved += 1;}- Ok(())-}--async fn copy_blobs(- from: &ScopedStorage,- to: &ScopedStorage,- report: &mut MoveReport,-) -> Result<(), StorageError> {- let source = from.blobs();- let target = to.blobs();- for meta in source.list("").await? {- if target.head(&meta.key).await?.is_none() {- let Some(blob) = source.get(&meta.key).await? else {- continue;- };- target- .put(&meta.key, blob.bytes, meta.content_type.as_deref())- .await?;- report.blobs += 1;- }- }- Ok(())+ Ok(report)}-/// Removes every record of a moved scope.-async fn purge(from: &ScopedStorage) -> Result<(), StorageError> {- let docs = from.documents();- for collection in docs.collections("").await? {- docs.drop_collection(&collection).await?;- }- let streams = from.streams();- for name in streams.streams("").await? {- streams.delete_stream(&name).await?;- }- let blobs = from.blobs();- for meta in blobs.list("").await? {- blobs.delete(&meta.key).await?;- }- Ok(())+/// Legacy compatibility entry point retained to produce an actionable error.+/// v2 backends cannot enumerate old scopes; callers must supply the old SQLite+/// path to [`crate::storage::import::scope_move::LegacyScopeInventory::read`]+/// and then pass that inventory to [`copy_inventory`].+#[deprecated(note = "use an explicit read-only legacy scope inventory")]+pub async fn run(+ _backend: &Arc<dyn StorageBackend>,+) -> Result<MoveReport, tinystoragedrivers::StorageError> {+ Err(tinystoragedrivers::StorageError::unsupported(+ tinystoragedrivers::Capability::ScopeEnumeration,+ "v2 backends do not enumerate legacy scopes; supply an explicit legacy database inventory",+ ))}/// The scopes of `scopes` that are not in the v2 grammar (for tests and thediff --git a/crates/openhuman-core/src/storage/scope_move_tests.rs b/crates/openhuman-core/src/storage/scope_move_tests.rsindex bc6bf79966..feee3cb604 100644--- a/crates/openhuman-core/src/storage/scope_move_tests.rs+++ b/crates/openhuman-core/src/storage/scope_move_tests.rs@@ -1,6 +1,8 @@use super::*;+#[cfg(feature = "legacy-import")]use serde_json::json;-use tinystoragedrivers::MemoryStorage;+#[cfg(feature = "legacy-import")]+use tinystoragedrivers::{MemoryStorage, Meta, Precondition};fn known() -> Known {Known {@@ -56,29 +58,37 @@ fn v2_scopes_are_left_alone() {}#[tokio::test]+#[cfg(feature = "legacy-import")]async fn a_v1_scope_moves_its_documents_streams_and_blobs() {let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());- let old = backend- .for_scope(&Scope::new("alice~asha").unwrap())- .unwrap();- old.documents()- .put("notes", "n1", json!({"a": 1}), Precondition::None)- .await- .unwrap();- old.streams()- .append("journal", json!({"k": 1}))- .await- .unwrap();- old.streams()- .append("journal", json!({"k": 2}))+ let source = "alice~asha";+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory {+ documents: vec![crate::storage::import::scope_move::LegacyDocument {+ scope: source.into(),+ collection: "notes".into(),+ id: "n1".into(),+ body: json!({"a": 1}),+ meta: Meta::default(),+ }],+ streams: vec![crate::storage::import::scope_move::LegacyStream {+ scope: source.into(),+ name: "journal".into(),+ base: 0,+ sealed: false,+ sealed_ms: None,+ entries: vec![(0, json!({"k":1})), (1, json!({"k":2}))],+ }],+ blobs: vec![crate::storage::import::scope_move::LegacyBlob {+ scope: source.into(),+ key: "b/1".into(),+ content_type: Some("text/plain".into()),+ expires_ms: None,+ bytes: b"hi".to_vec(),+ }],+ };+ let report = copy_inventory(&backend, &known(), &inventory).await.unwrap();- old.blobs()- .put("b/1", b"hi".to_vec(), Some("text/plain"))- .await- .unwrap();-- let report = run(&backend).await.unwrap();assert_eq!(report.moved, 1);assert!(report.documents >= 1);assert_eq!(report.entries, 2);@@ -99,17 +109,19 @@ async fn a_v1_scope_moves_its_documents_streams_and_blobs() {assert_eq!(new.streams().len("journal").await.unwrap(), 2);assert_eq!(new.blobs().get("b/1").await.unwrap().unwrap().bytes, b"hi");- // The v1 scope is gone and a second run is a no-op.- let old = backend- .for_scope(&Scope::new("alice~asha").unwrap())+ // The explicit source inventory remains available for operator review.+ assert_eq!(inventory.documents.len(), 1);+ assert_eq!(inventory.streams[0].entries.len(), 2);+ // Repeating the typed copy preserves rows without appending stream entries.+ let again = copy_inventory(&backend, &known(), &inventory)+ .await.unwrap();- assert!(old.documents().get("notes", "n1").await.unwrap().is_none());- assert_eq!(old.streams().len("journal").await.unwrap(), 0);- let again = run(&backend).await.unwrap();- assert_eq!(again.moved, 0);+ assert_eq!(again.entries, 0);+ assert_eq!(new.streams().len("journal").await.unwrap(), 2);}#[tokio::test]+#[cfg(feature = "legacy-import")]async fn a_record_already_in_the_target_wins_a_merge() {let backend: Arc<dyn StorageBackend> = Arc::new(MemoryStorage::new());let target = backend.for_scope(&scope_for_agent("asha")).unwrap();@@ -118,12 +130,19 @@ async fn a_record_already_in_the_target_wins_a_merge() {.put("notes", "n", json!({"from": "v2"}), Precondition::None).await.unwrap();- let old = backend.for_scope(&Scope::new("asha").unwrap()).unwrap();- old.documents()- .put("notes", "n", json!({"from": "v1"}), Precondition::None)+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory {+ documents: vec![crate::storage::import::scope_move::LegacyDocument {+ scope: "asha".into(),+ collection: "notes".into(),+ id: "n".into(),+ body: json!({"from": "v1"}),+ meta: Meta::default(),+ }],+ ..Default::default()+ };+ copy_inventory(&backend, &known(), &inventory).await.unwrap();- run(&backend).await.unwrap();assert_eq!(target.documents()