Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(storage): implement legacy scope import functionality

Steven Enamakel33f40b4f7b08
diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rs
index 365d41a70e..0908a337c0 100644
--- a/crates/openhuman-core/src/storage/scope_move.rs
+++ b/crates/openhuman-core/src/storage/scope_move.rs
@@ -150,6 +150,343 @@ pub struct MoveReport {
pub updated_times_not_restored: u64,
}
+const IMPORT_DOMAIN: &str = "scope_move";
+const IMPORT_OWNER: &str = "storage";
+
+fn marker_collections() -> Vec<CollectionSpec> {
+ crate::storage::import_state::collection_specs()
+}
+
+/// Selects the historical v1 scope database from the configured profile or
+/// workspace layout and imports its inventory through the normal boot/rebind
+/// importer pipeline. The destination backend is never used to discover
+/// source scopes.
+///
+/// A profile-layout source is `<profile>/workspace/storage/storage.db`, as
+/// selected by [`crate::storage::layout::legacy::v1_db`]. An unprofiled v1
+/// workspace uses the default backend's directory database at
+/// `<workspace>/storage/storage.db`.
+#[cfg(feature = "legacy-import")]
+pub fn run_from_config(
+ backend: &Arc<dyn StorageBackend>,
+ config: &crate::config::Config,
+) -> crate::storage::import::Report {
+ let result = (|| {
+ let source = legacy_scope_source(config);
+ if !source.is_file() {
+ if let Some(url) = crate::storage::explicit_url(config) {
+ let parsed = crate::storage::StorageUrl::parse(&url)?;
+ if parsed.driver() != "sqlite" {
+ anyhow::bail!(
+ "legacy scope inventory source driver `{}` is unsupported; only the configured v1 SQLite source can be read",
+ parsed.driver()
+ );
+ }
+ }
+ return Ok(crate::storage::import::Outcome::NoSource);
+ }
+
+ let known = known_scope_ids(config)?;
+ let path = source.clone();
+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(&path)
+ .with_context(|| format!("read legacy scope inventory {}", path.display()))?;
+ import_inventory(backend, &source, &known, &inventory)
+ })();
+
+ crate::storage::import::Report {
+ domain: IMPORT_DOMAIN,
+ outcome: result,
+ }
+}
+
+#[cfg(not(feature = "legacy-import"))]
+pub fn run_from_config(
+ _backend: &Arc<dyn StorageBackend>,
+ _config: &crate::config::Config,
+) -> crate::storage::import::Report {
+ crate::storage::import::Report {
+ domain: IMPORT_DOMAIN,
+ outcome: Err(anyhow::anyhow!(
+ "legacy scope inventory import requires the `legacy-import` feature"
+ )),
+ }
+}
+
+fn legacy_scope_source(config: &crate::config::Config) -> PathBuf {
+ if let Some(target) = crate::storage::layout::Target::from_config(config) {
+ return crate::storage::layout::legacy::v1_db(&target);
+ }
+ config
+ .workspace_dir
+ .join(crate::storage::config::DEFAULT_DIR)
+ .join("storage.db")
+}
+
+#[cfg(feature = "legacy-import")]
+fn known_scope_ids(config: &crate::config::Config) -> anyhow::Result<Known> {
+ let mut profiles = Vec::new();
+ if let Some(target) = crate::storage::layout::Target::from_config(config) {
+ profiles.push(target.user_id);
+ }
+ if let Some(host) = crate::profiles::host::host() {
+ let hosted = crate::storage::block_on_anyhow(async move {
+ host.known_profiles()
+ .await
+ .map_err(anyhow::Error::msg)
+ })?;
+ profiles.extend(hosted);
+ }
+ profiles.sort();
+ profiles.dedup();
+
+ let agents = crate::agent_registry::host_definition::AgentDefinitionRegistry::current()
+ .map(|registry| {
+ registry
+ .list()
+ .into_iter()
+ .map(|definition| definition.id.clone())
+ .collect()
+ })
+ .unwrap_or_default();
+ let modules = crate::modules::registry::ALL
+ .iter()
+ .map(|module| module.id.to_owned())
+ .collect();
+ Ok(Known {
+ agents,
+ profiles,
+ modules,
+ })
+}
+
+#[cfg(feature = "legacy-import")]
+fn import_inventory(
+ backend: &Arc<dyn StorageBackend>,
+ source_path: &Path,
+ known: &Known,
+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,
+) -> anyhow::Result<crate::storage::import::Outcome> {
+ let mut report = MoveReport::default();
+ let mut scopes = inventory.scopes.clone();
+ scopes.extend(inventory.documents.iter().map(|row| row.scope.clone()));
+ scopes.extend(inventory.streams.iter().map(|row| row.scope.clone()));
+ scopes.extend(inventory.blobs.iter().map(|row| row.scope.clone()));
+ scopes.sort();
+ scopes.dedup();
+ if scopes.is_empty() {
+ return Ok(crate::storage::import::Outcome::Imported {
+ found: 0,
+ written: 0,
+ });
+ }
+
+ let mut all_already_done = true;
+ let mut in_progress = 0usize;
+ let mut found = 0usize;
+ let mut written = 0usize;
+ for source_scope in scopes {
+ let target_scope = match map_scope(&source_scope, known) {
+ Mapping::Move(scope) => scope,
+ Mapping::V2 => Scope::new(&source_scope)?,
+ Mapping::Unknown => {
+ report.unknown.push(source_scope);
+ all_already_done = false;
+ continue;
+ }
+ };
+ let scope_backend = backend_for_target_scope(backend, &target_scope, known)?;
+ let marker = crate::storage::documents::Repo::on(
+ &scope_backend,
+ &target_scope,
+ "storage::scope_move",
+ marker_collections,
+ )?;
+ let step = scope_move_step(source_path, &source_scope);
+ if crate::storage::import_state::is_done(&marker, IMPORT_OWNER, &step)? {
+ report.already_done += 1;
+ let scope_found = inventory_scope_counts(inventory, &source_scope);
+ found += scope_found;
+ continue;
+ }
+
+ all_already_done = false;
+ let scope_inventory = inventory_for_scope(inventory, &source_scope);
+ let scope_known = known.clone();
+ let scope_backend_for_claim = Arc::clone(&scope_backend);
+ let marker_for_claim = marker.clone();
+ let result = crate::storage::import_state::with_claim(
+ &marker,
+ IMPORT_OWNER,
+ &step,
+ move |condition| {
+ let claimed_marker = marker_for_claim.conditioned(condition)?;
+ if crate::storage::import_state::is_done(
+ &claimed_marker,
+ IMPORT_OWNER,
+ &step,
+ )? {
+ return Ok(None);
+ }
+ let claim_bound = crate::storage::import_state::backend_with_condition(
+ scope_backend_for_claim,
+ condition.clone(),
+ );
+ let copied = crate::storage::block_on_anyhow(copy_inventory(
+ &claim_bound,
+ &scope_known,
+ &scope_inventory,
+ ))?;
+ crate::storage::import_state::set_done(&claimed_marker, IMPORT_OWNER, &step)?;
+ Ok(Some(copied))
+ },
+ )?;
+
+ match result {
+ None => in_progress += 1,
+ Some(None) => {
+ report.already_done += 1;
+ found += inventory_scope_counts(inventory, &source_scope);
+ }
+ Some(Some(scope_report)) => {
+ let scope_found = inventory_scope_counts(inventory, &source_scope);
+ found += scope_found;
+ written += usize::try_from(
+ scope_report.documents + scope_report.entries + scope_report.blobs,
+ )
+ .unwrap_or(usize::MAX);
+ report.moved += scope_report.moved;
+ report.documents += scope_report.documents;
+ report.entries += scope_report.entries;
+ report.blobs += scope_report.blobs;
+ report.updated_times_not_restored += scope_report.updated_times_not_restored;
+ }
+ }
+ }
+
+ if !report.unknown.is_empty() {
+ return Ok(crate::storage::import::Outcome::Incomplete {
+ found,
+ written,
+ skipped: report.unknown.len(),
+ });
+ }
+ if in_progress > 0 {
+ if all_already_done {
+ return Ok(crate::storage::import::Outcome::InProgress);
+ }
+ return Ok(crate::storage::import::Outcome::Incomplete {
+ found,
+ written,
+ skipped: in_progress,
+ });
+ }
+ if all_already_done {
+ Ok(crate::storage::import::Outcome::AlreadyDone)
+ } else {
+ Ok(crate::storage::import::Outcome::Imported { found, written })
+ }
+}
+
+#[cfg(feature = "legacy-import")]
+fn backend_for_target_scope(
+ backend: &Arc<dyn StorageBackend>,
+ target_scope: &Scope,
+ known: &Known,
+) -> anyhow::Result<Arc<dyn StorageBackend>> {
+ if target_scope.is_local() || target_scope.is_cluster() {
+ return Ok(Arc::clone(backend));
+ }
+ let parsed = target_scope.parsed()?;
+ let crate::storage::Tenant::User(_) = parsed.tenant else {
+ anyhow::bail!("scope {} is not an admitted tenant scope", target_scope.as_str());
+ };
+ let Some(host) = crate::profiles::host::host() else {
+ let config_profile = crate::storage::layout::Target::from_config(&crate::config::Config::default());
+ let _ = config_profile;
+ anyhow::bail!(
+ "cannot copy profile scope {} without its admitted profile host",
+ target_scope.as_str()
+ );
+ };
+ let original_id = known
+ .profiles
+ .iter()
+ .find(|id| crate::storage::scope_for_profile(id).tenant_scope() == target_scope.tenant_scope())
+ .ok_or_else(|| anyhow::anyhow!("no known profile id maps to scope {}", target_scope.as_str()))?;
+ let profile_id = crate::profiles::ProfileId::parse(original_id)
+ .map_err(anyhow::Error::msg)?;
+ let profile = host.get(&profile_id).ok_or_else(|| {
+ anyhow::anyhow!(
+ "profile {original_id} is not currently admitted on this host; refusing scope copy"
+ )
+ })?;
+ host.storage_backend_for(&profile)?
+ .ok_or_else(|| anyhow::anyhow!("profile {original_id} has no captured storage backend"))
+}
+
+#[cfg(feature = "legacy-import")]
+fn scope_move_step(source_path: &Path, source_scope: &str) -> String {
+ use sha2::Digest;
+ let source_hash = sha2::Sha256::digest(source_path.as_os_str().as_encoded_bytes());
+ let scope_hash = sha2::Sha256::digest(source_scope.as_bytes());
+ format!(
+ "scope_move/{}/{}",
+ hex::encode(&source_hash[..8]),
+ hex::encode(&scope_hash[..8])
+ )
+}
+
+#[cfg(feature = "legacy-import")]
+fn inventory_for_scope(
+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,
+ source_scope: &str,
+) -> crate::storage::import::scope_move::LegacyScopeInventory {
+ crate::storage::import::scope_move::LegacyScopeInventory {
+ scopes: vec![source_scope.to_owned()],
+ documents: inventory
+ .documents
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .cloned()
+ .collect(),
+ streams: inventory
+ .streams
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .cloned()
+ .collect(),
+ blobs: inventory
+ .blobs
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .cloned()
+ .collect(),
+ }
+}
+
+#[cfg(feature = "legacy-import")]
+fn inventory_scope_counts(
+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,
+ source_scope: &str,
+) -> usize {
+ inventory
+ .documents
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .count()
+ + inventory
+ .streams
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .map(|row| row.entries.len())
+ .sum::<usize>()
+ + inventory
+ .blobs
+ .iter()
+ .filter(|row| row.scope == source_scope)
+ .count()
+}
+
/// Copies the rows in a source inventory to their mapped v2 scopes.
///
/// The source is deliberately not deleted. Physical inventory readers are