The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(storage): implement legacy scope import functionality
Steven Enamakel33f40b4f7b08
diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rsindex 365d41a70e..0908a337c0 100644--- a/crates/openhuman-core/src/storage/scope_move.rs+++ b/crates/openhuman-core/src/storage/scope_move.rs@@ -150,6 +150,343 @@ pub struct MoveReport {pub updated_times_not_restored: u64,}+const IMPORT_DOMAIN: &str = "scope_move";+const IMPORT_OWNER: &str = "storage";++fn marker_collections() -> Vec<CollectionSpec> {+ crate::storage::import_state::collection_specs()+}++/// Selects the historical v1 scope database from the configured profile or+/// workspace layout and imports its inventory through the normal boot/rebind+/// importer pipeline. The destination backend is never used to discover+/// source scopes.+///+/// A profile-layout source is `<profile>/workspace/storage/storage.db`, as+/// selected by [`crate::storage::layout::legacy::v1_db`]. An unprofiled v1+/// workspace uses the default backend's directory database at+/// `<workspace>/storage/storage.db`.+#[cfg(feature = "legacy-import")]+pub fn run_from_config(+ backend: &Arc<dyn StorageBackend>,+ config: &crate::config::Config,+) -> crate::storage::import::Report {+ let result = (|| {+ let source = legacy_scope_source(config);+ if !source.is_file() {+ if let Some(url) = crate::storage::explicit_url(config) {+ let parsed = crate::storage::StorageUrl::parse(&url)?;+ if parsed.driver() != "sqlite" {+ anyhow::bail!(+ "legacy scope inventory source driver `{}` is unsupported; only the configured v1 SQLite source can be read",+ parsed.driver()+ );+ }+ }+ return Ok(crate::storage::import::Outcome::NoSource);+ }++ let known = known_scope_ids(config)?;+ let path = source.clone();+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(&path)+ .with_context(|| format!("read legacy scope inventory {}", path.display()))?;+ import_inventory(backend, &source, &known, &inventory)+ })();++ crate::storage::import::Report {+ domain: IMPORT_DOMAIN,+ outcome: result,+ }+}++#[cfg(not(feature = "legacy-import"))]+pub fn run_from_config(+ _backend: &Arc<dyn StorageBackend>,+ _config: &crate::config::Config,+) -> crate::storage::import::Report {+ crate::storage::import::Report {+ domain: IMPORT_DOMAIN,+ outcome: Err(anyhow::anyhow!(+ "legacy scope inventory import requires the `legacy-import` feature"+ )),+ }+}++fn legacy_scope_source(config: &crate::config::Config) -> PathBuf {+ if let Some(target) = crate::storage::layout::Target::from_config(config) {+ return crate::storage::layout::legacy::v1_db(&target);+ }+ config+ .workspace_dir+ .join(crate::storage::config::DEFAULT_DIR)+ .join("storage.db")+}++#[cfg(feature = "legacy-import")]+fn known_scope_ids(config: &crate::config::Config) -> anyhow::Result<Known> {+ let mut profiles = Vec::new();+ if let Some(target) = crate::storage::layout::Target::from_config(config) {+ profiles.push(target.user_id);+ }+ if let Some(host) = crate::profiles::host::host() {+ let hosted = crate::storage::block_on_anyhow(async move {+ host.known_profiles()+ .await+ .map_err(anyhow::Error::msg)+ })?;+ profiles.extend(hosted);+ }+ profiles.sort();+ profiles.dedup();++ let agents = crate::agent_registry::host_definition::AgentDefinitionRegistry::current()+ .map(|registry| {+ registry+ .list()+ .into_iter()+ .map(|definition| definition.id.clone())+ .collect()+ })+ .unwrap_or_default();+ let modules = crate::modules::registry::ALL+ .iter()+ .map(|module| module.id.to_owned())+ .collect();+ Ok(Known {+ agents,+ profiles,+ modules,+ })+}++#[cfg(feature = "legacy-import")]+fn import_inventory(+ backend: &Arc<dyn StorageBackend>,+ source_path: &Path,+ known: &Known,+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+) -> anyhow::Result<crate::storage::import::Outcome> {+ let mut report = MoveReport::default();+ let mut scopes = inventory.scopes.clone();+ scopes.extend(inventory.documents.iter().map(|row| row.scope.clone()));+ scopes.extend(inventory.streams.iter().map(|row| row.scope.clone()));+ scopes.extend(inventory.blobs.iter().map(|row| row.scope.clone()));+ scopes.sort();+ scopes.dedup();+ if scopes.is_empty() {+ return Ok(crate::storage::import::Outcome::Imported {+ found: 0,+ written: 0,+ });+ }++ let mut all_already_done = true;+ let mut in_progress = 0usize;+ let mut found = 0usize;+ let mut written = 0usize;+ for source_scope in scopes {+ let target_scope = match map_scope(&source_scope, known) {+ Mapping::Move(scope) => scope,+ Mapping::V2 => Scope::new(&source_scope)?,+ Mapping::Unknown => {+ report.unknown.push(source_scope);+ all_already_done = false;+ continue;+ }+ };+ let scope_backend = backend_for_target_scope(backend, &target_scope, known)?;+ let marker = crate::storage::documents::Repo::on(+ &scope_backend,+ &target_scope,+ "storage::scope_move",+ marker_collections,+ )?;+ let step = scope_move_step(source_path, &source_scope);+ if crate::storage::import_state::is_done(&marker, IMPORT_OWNER, &step)? {+ report.already_done += 1;+ let scope_found = inventory_scope_counts(inventory, &source_scope);+ found += scope_found;+ continue;+ }++ all_already_done = false;+ let scope_inventory = inventory_for_scope(inventory, &source_scope);+ let scope_known = known.clone();+ let scope_backend_for_claim = Arc::clone(&scope_backend);+ let marker_for_claim = marker.clone();+ let result = crate::storage::import_state::with_claim(+ &marker,+ IMPORT_OWNER,+ &step,+ move |condition| {+ let claimed_marker = marker_for_claim.conditioned(condition)?;+ if crate::storage::import_state::is_done(+ &claimed_marker,+ IMPORT_OWNER,+ &step,+ )? {+ return Ok(None);+ }+ let claim_bound = crate::storage::import_state::backend_with_condition(+ scope_backend_for_claim,+ condition.clone(),+ );+ let copied = crate::storage::block_on_anyhow(copy_inventory(+ &claim_bound,+ &scope_known,+ &scope_inventory,+ ))?;+ crate::storage::import_state::set_done(&claimed_marker, IMPORT_OWNER, &step)?;+ Ok(Some(copied))+ },+ )?;++ match result {+ None => in_progress += 1,+ Some(None) => {+ report.already_done += 1;+ found += inventory_scope_counts(inventory, &source_scope);+ }+ Some(Some(scope_report)) => {+ let scope_found = inventory_scope_counts(inventory, &source_scope);+ found += scope_found;+ written += usize::try_from(+ scope_report.documents + scope_report.entries + scope_report.blobs,+ )+ .unwrap_or(usize::MAX);+ report.moved += scope_report.moved;+ report.documents += scope_report.documents;+ report.entries += scope_report.entries;+ report.blobs += scope_report.blobs;+ report.updated_times_not_restored += scope_report.updated_times_not_restored;+ }+ }+ }++ if !report.unknown.is_empty() {+ return Ok(crate::storage::import::Outcome::Incomplete {+ found,+ written,+ skipped: report.unknown.len(),+ });+ }+ if in_progress > 0 {+ if all_already_done {+ return Ok(crate::storage::import::Outcome::InProgress);+ }+ return Ok(crate::storage::import::Outcome::Incomplete {+ found,+ written,+ skipped: in_progress,+ });+ }+ if all_already_done {+ Ok(crate::storage::import::Outcome::AlreadyDone)+ } else {+ Ok(crate::storage::import::Outcome::Imported { found, written })+ }+}++#[cfg(feature = "legacy-import")]+fn backend_for_target_scope(+ backend: &Arc<dyn StorageBackend>,+ target_scope: &Scope,+ known: &Known,+) -> anyhow::Result<Arc<dyn StorageBackend>> {+ if target_scope.is_local() || target_scope.is_cluster() {+ return Ok(Arc::clone(backend));+ }+ let parsed = target_scope.parsed()?;+ let crate::storage::Tenant::User(_) = parsed.tenant else {+ anyhow::bail!("scope {} is not an admitted tenant scope", target_scope.as_str());+ };+ let Some(host) = crate::profiles::host::host() else {+ let config_profile = crate::storage::layout::Target::from_config(&crate::config::Config::default());+ let _ = config_profile;+ anyhow::bail!(+ "cannot copy profile scope {} without its admitted profile host",+ target_scope.as_str()+ );+ };+ let original_id = known+ .profiles+ .iter()+ .find(|id| crate::storage::scope_for_profile(id).tenant_scope() == target_scope.tenant_scope())+ .ok_or_else(|| anyhow::anyhow!("no known profile id maps to scope {}", target_scope.as_str()))?;+ let profile_id = crate::profiles::ProfileId::parse(original_id)+ .map_err(anyhow::Error::msg)?;+ let profile = host.get(&profile_id).ok_or_else(|| {+ anyhow::anyhow!(+ "profile {original_id} is not currently admitted on this host; refusing scope copy"+ )+ })?;+ host.storage_backend_for(&profile)?+ .ok_or_else(|| anyhow::anyhow!("profile {original_id} has no captured storage backend"))+}++#[cfg(feature = "legacy-import")]+fn scope_move_step(source_path: &Path, source_scope: &str) -> String {+ use sha2::Digest;+ let source_hash = sha2::Sha256::digest(source_path.as_os_str().as_encoded_bytes());+ let scope_hash = sha2::Sha256::digest(source_scope.as_bytes());+ format!(+ "scope_move/{}/{}",+ hex::encode(&source_hash[..8]),+ hex::encode(&scope_hash[..8])+ )+}++#[cfg(feature = "legacy-import")]+fn inventory_for_scope(+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+ source_scope: &str,+) -> crate::storage::import::scope_move::LegacyScopeInventory {+ crate::storage::import::scope_move::LegacyScopeInventory {+ scopes: vec![source_scope.to_owned()],+ documents: inventory+ .documents+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ streams: inventory+ .streams+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ blobs: inventory+ .blobs+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ }+}++#[cfg(feature = "legacy-import")]+fn inventory_scope_counts(+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+ source_scope: &str,+) -> usize {+ inventory+ .documents+ .iter()+ .filter(|row| row.scope == source_scope)+ .count()+ + inventory+ .streams+ .iter()+ .filter(|row| row.scope == source_scope)+ .map(|row| row.entries.len())+ .sum::<usize>()+ + inventory+ .blobs+ .iter()+ .filter(|row| row.scope == source_scope)+ .count()+}+/// Copies the rows in a source inventory to their mapped v2 scopes.////// The source is deliberately not deleted. Physical inventory readers are