The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): extract legacy scope import boot logic into its own module
Steven Enamakelf23846dbbbf0
diff --git a/crates/openhuman-core/src/storage/scope_move.rs b/crates/openhuman-core/src/storage/scope_move.rsindex fc1287ac8d..4da0c29de0 100644--- a/crates/openhuman-core/src/storage/scope_move.rs+++ b/crates/openhuman-core/src/storage/scope_move.rs@@ -152,376 +152,10 @@ pub struct MoveReport {pub updated_times_not_restored: u64,}-const IMPORT_DOMAIN: &str = "scope_move";-const IMPORT_OWNER: &str = "storage";--fn marker_collections() -> Vec<CollectionSpec> {- crate::storage::import_state::collection_specs()-}--/// Selects the historical v1 scope database from the configured profile or-/// workspace layout and imports its inventory through the normal boot/rebind-/// importer pipeline. The destination backend is never used to discover-/// source scopes.-///-/// A profile-layout source is `<profile>/workspace/storage/storage.db`, as-/// selected by [`crate::storage::layout::legacy::v1_db`]. An unprofiled v1-/// workspace uses the default backend's directory database at-/// `<workspace>/storage/storage.db`.-#[cfg(feature = "legacy-import")]-pub fn run_from_config(- backend: &Arc<dyn StorageBackend>,- config: &crate::config::Config,-) -> crate::storage::import::Report {- let result = (|| {- let source = legacy_scope_source(config);- if !source.is_file() {- if let Some(url) = crate::storage::explicit_url(config) {- let parsed = crate::storage::StorageUrl::parse(&url)?;- if parsed.driver() != "sqlite" {- anyhow::bail!(- "legacy scope inventory source driver `{}` is unsupported; only the configured v1 SQLite source can be read",- parsed.driver()- );- }- }- return Ok(crate::storage::import::Outcome::NoSource);- }-- let known = known_scope_ids(config)?;- let path = source.clone();- let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(&path)- .with_context(|| format!("read legacy scope inventory {}", path.display()))?;- import_inventory(backend, config, &source, &known, &inventory)- })();-- crate::storage::import::Report {- domain: IMPORT_DOMAIN,- outcome: result,- }-}--#[cfg(not(feature = "legacy-import"))]-pub fn run_from_config(- _backend: &Arc<dyn StorageBackend>,- _config: &crate::config::Config,-) -> crate::storage::import::Report {- crate::storage::import::Report {- domain: IMPORT_DOMAIN,- outcome: Err(anyhow::anyhow!(- "legacy scope inventory import requires the `legacy-import` feature"- )),- }-}--#[cfg(feature = "legacy-import")]-fn legacy_scope_source(config: &crate::config::Config) -> PathBuf {- if let Some(target) = crate::storage::layout::Target::from_config(config) {- return crate::storage::layout::legacy::v1_db(&target);- }- config- .workspace_dir- .join(crate::storage::config::DEFAULT_DIR)- .join("storage.db")-}--#[cfg(feature = "legacy-import")]-fn known_scope_ids(config: &crate::config::Config) -> anyhow::Result<Known> {- let mut profiles = Vec::new();- if let Some(target) = crate::storage::layout::Target::from_config(config) {- profiles.push(target.user_id);- }- if let Some(host) = crate::profiles::host::host() {- let hosted = crate::storage::block_on_anyhow(async move {- host.known_profiles().await.map_err(anyhow::Error::msg)- })?;- profiles.extend(hosted);- }- profiles.sort();- profiles.dedup();-- let agents = crate::agent_registry::host_definition::AgentDefinitionRegistry::current()- .map(|registry| {- registry- .list()- .into_iter()- .map(|definition| definition.id.clone())- .collect()- })- .unwrap_or_default();- let modules = crate::modules::registry::ALL- .iter()- .map(|module| module.id.to_owned())- .collect();- Ok(Known {- agents,- profiles,- modules,- })-}--#[cfg(feature = "legacy-import")]-fn import_inventory(- backend: &Arc<dyn StorageBackend>,- config: &crate::config::Config,- source_path: &Path,- known: &Known,- inventory: &crate::storage::import::scope_move::LegacyScopeInventory,-) -> anyhow::Result<crate::storage::import::Outcome> {- let mut report = MoveReport::default();- let mut scopes = inventory.scopes.clone();- scopes.extend(inventory.documents.iter().map(|row| row.scope.clone()));- scopes.extend(inventory.streams.iter().map(|row| row.scope.clone()));- scopes.extend(inventory.blobs.iter().map(|row| row.scope.clone()));- scopes.sort();- scopes.dedup();- if scopes.is_empty() {- return Ok(crate::storage::import::Outcome::Imported {- found: 0,- written: 0,- });- }-- let mut all_already_done = true;- let mut in_progress = 0usize;- let mut found = 0usize;- let mut written = 0usize;- for source_scope in scopes {- let target_scope = match map_scope(&source_scope, known) {- Mapping::Move(scope) => scope,- Mapping::V2 => Scope::new(&source_scope)?,- Mapping::Unknown => {- report.unknown.push(source_scope);- all_already_done = false;- continue;- }- };- let (scope_backend, profile_guard) =- backend_for_target_scope(backend, &target_scope, known, config)?;- let marker = crate::storage::documents::Repo::on(- &scope_backend,- &target_scope,- "storage::scope_move",- marker_collections,- )?;- let step = scope_move_step(source_path, &source_scope);- if crate::storage::import_state::is_done(&marker, IMPORT_OWNER, &step)? {- report.already_done += 1;- let scope_found = inventory_scope_counts(inventory, &source_scope);- found += scope_found;- continue;- }-- all_already_done = false;- let scope_inventory = inventory_for_scope(inventory, &source_scope);- let scope_known = known.clone();- let scope_backend_for_claim = Arc::clone(&scope_backend);- let marker_for_claim = marker.clone();- let step_for_claim = step.clone();- let result = crate::storage::import_state::with_claim(- &marker,- IMPORT_OWNER,- &step,- move |condition| {- // Keep the admitted profile handle alive until all copied- // documents, streams, blobs, and the completion marker finish.- let _profile_guard = profile_guard;- let claimed_marker = marker_for_claim.conditioned(condition)?;- if crate::storage::import_state::is_done(- &claimed_marker,- IMPORT_OWNER,- &step_for_claim,- )? {- return Ok(None);- }- let claim_bound = crate::storage::import_state::backend_with_condition(- scope_backend_for_claim,- condition.clone(),- );- let copied = crate::storage::block_on_anyhow(async move {- copy_inventory(&claim_bound, &scope_known, &scope_inventory).await- })?;- crate::storage::import_state::set_done(- &claimed_marker,- IMPORT_OWNER,- &step_for_claim,- )?;- Ok(Some(copied))- },- )?;-- match result {- None => in_progress += 1,- Some(None) => {- report.already_done += 1;- found += inventory_scope_counts(inventory, &source_scope);- }- Some(Some(scope_report)) => {- let scope_found = inventory_scope_counts(inventory, &source_scope);- found += scope_found;- written += usize::try_from(- scope_report.documents + scope_report.entries + scope_report.blobs,- )- .unwrap_or(usize::MAX);- report.moved += scope_report.moved;- report.documents += scope_report.documents;- report.entries += scope_report.entries;- report.blobs += scope_report.blobs;- report.updated_times_not_restored += scope_report.updated_times_not_restored;- }- }- }-- if !report.unknown.is_empty() {- return Ok(crate::storage::import::Outcome::Incomplete {- found,- written,- skipped: report.unknown.len(),- });- }- if in_progress > 0 {- if all_already_done {- return Ok(crate::storage::import::Outcome::InProgress);- }- return Ok(crate::storage::import::Outcome::Incomplete {- found,- written,- skipped: in_progress,- });- }- if all_already_done {- Ok(crate::storage::import::Outcome::AlreadyDone)- } else {- Ok(crate::storage::import::Outcome::Imported { found, written })- }-}--#[cfg(feature = "legacy-import")]-fn backend_for_target_scope(- backend: &Arc<dyn StorageBackend>,- target_scope: &Scope,- known: &Known,- config: &crate::config::Config,-) -> anyhow::Result<(- Arc<dyn StorageBackend>,- Option<Arc<crate::profiles::host::Profile>>,-)> {- if target_scope.is_local() || target_scope.is_cluster() {- return Ok((Arc::clone(backend), None));- }- let parsed = target_scope.parsed()?;- let crate::storage::Tenant::User(_) = parsed.tenant else {- anyhow::bail!(- "scope {} is not an admitted tenant scope",- target_scope.as_str()- );- };- let original_id = known- .profiles- .iter()- .find(|id| {- crate::storage::scope_for_profile(id).tenant_scope() == target_scope.tenant_scope()- })- .ok_or_else(|| {- anyhow::anyhow!(- "no known profile id maps to scope {}",- target_scope.as_str()- )- })?;- let Some(host) = crate::profiles::host::host() else {- if let Some(target) = crate::storage::layout::Target::from_config(config) {- let expected_url = format!("sqlite:{}", target.storage_db().display());- let installed = crate::storage::installed();- let backend_is_installed = installed- .as_ref()- .is_some_and(|installed| Arc::ptr_eq(installed, backend));- if target.user_id == *original_id- && crate::storage::configured_url(config).as_deref() == Some(&expected_url)- && crate::storage::installed_url().as_deref() == Some(&expected_url)- && backend_is_installed- {- return Ok((Arc::clone(backend), None));- }- }- anyhow::bail!(- "cannot copy profile scope {} without its admitted profile fence or dedicated profile database",- target_scope.as_str()- );- };- let profile_id = crate::profiles::ProfileId::parse(original_id).map_err(anyhow::Error::msg)?;- let profile = host.get(&profile_id).ok_or_else(|| {- anyhow::anyhow!(- "profile {original_id} is not currently admitted on this host; refusing scope copy"- )- })?;- let scoped = host- .storage_backend_for(&profile)?- .ok_or_else(|| anyhow::anyhow!("profile {original_id} has no captured storage backend"))?;- Ok((scoped, Some(profile)))-}--#[cfg(feature = "legacy-import")]-fn scope_move_step(source_path: &Path, source_scope: &str) -> String {- use sha2::Digest;- let source_hash = sha2::Sha256::digest(source_path.as_os_str().as_encoded_bytes());- let scope_hash = sha2::Sha256::digest(source_scope.as_bytes());- format!(- "scope_move/{}/{}",- hex::encode(&source_hash[..8]),- hex::encode(&scope_hash[..8])- )-}--#[cfg(feature = "legacy-import")]-fn inventory_for_scope(- inventory: &crate::storage::import::scope_move::LegacyScopeInventory,- source_scope: &str,-) -> crate::storage::import::scope_move::LegacyScopeInventory {- crate::storage::import::scope_move::LegacyScopeInventory {- scopes: vec![source_scope.to_owned()],- documents: inventory- .documents- .iter()- .filter(|row| row.scope == source_scope)- .cloned()- .collect(),- streams: inventory- .streams- .iter()- .filter(|row| row.scope == source_scope)- .cloned()- .collect(),- blobs: inventory- .blobs- .iter()- .filter(|row| row.scope == source_scope)- .cloned()- .collect(),- }-}--#[cfg(feature = "legacy-import")]-fn inventory_scope_counts(- inventory: &crate::storage::import::scope_move::LegacyScopeInventory,- source_scope: &str,-) -> usize {- inventory- .documents- .iter()- .filter(|row| row.scope == source_scope)- .count()- + inventory- .streams- .iter()- .filter(|row| row.scope == source_scope)- .map(|row| row.entries.len())- .sum::<usize>()- + inventory- .blobs- .iter()- .filter(|row| row.scope == source_scope)- .count()-}+[path = "scope_move_boot.rs"]+mod boot;+pub use boot::run_from_config;+pub(crate) use boot::IMPORT_DOMAIN;/// Copies the rows in a source inventory to their mapped v2 scopes.///diff --git a/crates/openhuman-core/src/storage/scope_move_boot.rs b/crates/openhuman-core/src/storage/scope_move_boot.rsnew file mode 100644index 0000000000..c1665b48f0--- /dev/null+++ b/crates/openhuman-core/src/storage/scope_move_boot.rs@@ -0,0 +1,379 @@+use std::path::{Path, PathBuf};+use std::sync::Arc;++use anyhow::Context;+use tinystoragedrivers_types::CollectionSpec;++use super::*;++const IMPORT_DOMAIN: &str = "scope_move";+const IMPORT_OWNER: &str = "storage";++fn marker_collections() -> Vec<CollectionSpec> {+ crate::storage::import_state::collection_specs()+}++/// Selects the historical v1 scope database from the configured profile or+/// workspace layout and imports its inventory through the normal boot/rebind+/// importer pipeline. The destination backend is never used to discover+/// source scopes.+///+/// A profile-layout source is `<profile>/workspace/storage/storage.db`, as+/// selected by [`crate::storage::layout::legacy::v1_db`]. An unprofiled v1+/// workspace uses the default backend's directory database at+/// `<workspace>/storage/storage.db`.+#[cfg(feature = "legacy-import")]+pub fn run_from_config(+ backend: &Arc<dyn StorageBackend>,+ config: &crate::config::Config,+) -> crate::storage::import::Report {+ let result = (|| {+ let source = legacy_scope_source(config);+ if !source.is_file() {+ if let Some(url) = crate::storage::explicit_url(config) {+ let parsed = crate::storage::StorageUrl::parse(&url)?;+ if parsed.driver() != "sqlite" {+ anyhow::bail!(+ "legacy scope inventory source driver `{}` is unsupported; only the configured v1 SQLite source can be read",+ parsed.driver()+ );+ }+ }+ return Ok(crate::storage::import::Outcome::NoSource);+ }++ let known = known_scope_ids(config)?;+ let path = source.clone();+ let inventory = crate::storage::import::scope_move::LegacyScopeInventory::read(&path)+ .with_context(|| format!("read legacy scope inventory {}", path.display()))?;+ import_inventory(backend, config, &source, &known, &inventory)+ })();++ crate::storage::import::Report {+ domain: IMPORT_DOMAIN,+ outcome: result,+ }+}++#[cfg(not(feature = "legacy-import"))]+pub fn run_from_config(+ _backend: &Arc<dyn StorageBackend>,+ _config: &crate::config::Config,+) -> crate::storage::import::Report {+ crate::storage::import::Report {+ domain: IMPORT_DOMAIN,+ outcome: Err(anyhow::anyhow!(+ "legacy scope inventory import requires the `legacy-import` feature"+ )),+ }+}++#[cfg(feature = "legacy-import")]+fn legacy_scope_source(config: &crate::config::Config) -> PathBuf {+ if let Some(target) = crate::storage::layout::Target::from_config(config) {+ return crate::storage::layout::legacy::v1_db(&target);+ }+ config+ .workspace_dir+ .join(crate::storage::config::DEFAULT_DIR)+ .join("storage.db")+}++#[cfg(feature = "legacy-import")]+fn known_scope_ids(config: &crate::config::Config) -> anyhow::Result<Known> {+ let mut profiles = Vec::new();+ if let Some(target) = crate::storage::layout::Target::from_config(config) {+ profiles.push(target.user_id);+ }+ if let Some(host) = crate::profiles::host::host() {+ let hosted = crate::storage::block_on_anyhow(async move {+ host.known_profiles().await.map_err(anyhow::Error::msg)+ })?;+ profiles.extend(hosted);+ }+ profiles.sort();+ profiles.dedup();++ let agents = crate::agent_registry::host_definition::AgentDefinitionRegistry::current()+ .map(|registry| {+ registry+ .list()+ .into_iter()+ .map(|definition| definition.id.clone())+ .collect()+ })+ .unwrap_or_default();+ let modules = crate::modules::registry::ALL+ .iter()+ .map(|module| module.id.to_owned())+ .collect();+ Ok(Known {+ agents,+ profiles,+ modules,+ })+}++#[cfg(feature = "legacy-import")]+fn import_inventory(+ backend: &Arc<dyn StorageBackend>,+ config: &crate::config::Config,+ source_path: &Path,+ known: &Known,+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+) -> anyhow::Result<crate::storage::import::Outcome> {+ let mut report = MoveReport::default();+ let mut scopes = inventory.scopes.clone();+ scopes.extend(inventory.documents.iter().map(|row| row.scope.clone()));+ scopes.extend(inventory.streams.iter().map(|row| row.scope.clone()));+ scopes.extend(inventory.blobs.iter().map(|row| row.scope.clone()));+ scopes.sort();+ scopes.dedup();+ if scopes.is_empty() {+ return Ok(crate::storage::import::Outcome::Imported {+ found: 0,+ written: 0,+ });+ }++ let mut all_already_done = true;+ let mut in_progress = 0usize;+ let mut found = 0usize;+ let mut written = 0usize;+ for source_scope in scopes {+ let target_scope = match map_scope(&source_scope, known) {+ Mapping::Move(scope) => scope,+ Mapping::V2 => Scope::new(&source_scope)?,+ Mapping::Unknown => {+ report.unknown.push(source_scope);+ all_already_done = false;+ continue;+ }+ };+ let (scope_backend, profile_guard) =+ backend_for_target_scope(backend, &target_scope, known, config)?;+ let marker = crate::storage::documents::Repo::on(+ &scope_backend,+ &target_scope,+ "storage::scope_move",+ marker_collections,+ )?;+ let step = scope_move_step(source_path, &source_scope);+ if crate::storage::import_state::is_done(&marker, IMPORT_OWNER, &step)? {+ report.already_done += 1;+ let scope_found = inventory_scope_counts(inventory, &source_scope);+ found += scope_found;+ continue;+ }++ all_already_done = false;+ let scope_inventory = inventory_for_scope(inventory, &source_scope);+ let scope_known = known.clone();+ let scope_backend_for_claim = Arc::clone(&scope_backend);+ let marker_for_claim = marker.clone();+ let step_for_claim = step.clone();+ let result = crate::storage::import_state::with_claim(+ &marker,+ IMPORT_OWNER,+ &step,+ move |condition| {+ // Keep the admitted profile handle alive until all copied+ // documents, streams, blobs, and the completion marker finish.+ let _profile_guard = profile_guard;+ let claimed_marker = marker_for_claim.conditioned(condition)?;+ if crate::storage::import_state::is_done(+ &claimed_marker,+ IMPORT_OWNER,+ &step_for_claim,+ )? {+ return Ok(None);+ }+ let claim_bound = crate::storage::import_state::backend_with_condition(+ scope_backend_for_claim,+ condition.clone(),+ );+ let copied = crate::storage::block_on_anyhow(async move {+ copy_inventory(&claim_bound, &scope_known, &scope_inventory).await+ })?;+ crate::storage::import_state::set_done(+ &claimed_marker,+ IMPORT_OWNER,+ &step_for_claim,+ )?;+ Ok(Some(copied))+ },+ )?;++ match result {+ None => in_progress += 1,+ Some(None) => {+ report.already_done += 1;+ found += inventory_scope_counts(inventory, &source_scope);+ }+ Some(Some(scope_report)) => {+ let scope_found = inventory_scope_counts(inventory, &source_scope);+ found += scope_found;+ written += usize::try_from(+ scope_report.documents + scope_report.entries + scope_report.blobs,+ )+ .unwrap_or(usize::MAX);+ report.moved += scope_report.moved;+ report.documents += scope_report.documents;+ report.entries += scope_report.entries;+ report.blobs += scope_report.blobs;+ report.updated_times_not_restored += scope_report.updated_times_not_restored;+ }+ }+ }++ if !report.unknown.is_empty() {+ return Ok(crate::storage::import::Outcome::Incomplete {+ found,+ written,+ skipped: report.unknown.len(),+ });+ }+ if in_progress > 0 {+ if all_already_done {+ return Ok(crate::storage::import::Outcome::InProgress);+ }+ return Ok(crate::storage::import::Outcome::Incomplete {+ found,+ written,+ skipped: in_progress,+ });+ }+ if all_already_done {+ Ok(crate::storage::import::Outcome::AlreadyDone)+ } else {+ Ok(crate::storage::import::Outcome::Imported { found, written })+ }+}++#[cfg(feature = "legacy-import")]+fn backend_for_target_scope(+ backend: &Arc<dyn StorageBackend>,+ target_scope: &Scope,+ known: &Known,+ config: &crate::config::Config,+) -> anyhow::Result<(+ Arc<dyn StorageBackend>,+ Option<Arc<crate::profiles::host::Profile>>,+)> {+ if target_scope.is_local() || target_scope.is_cluster() {+ return Ok((Arc::clone(backend), None));+ }+ let parsed = target_scope.parsed()?;+ let crate::storage::Tenant::User(_) = parsed.tenant else {+ anyhow::bail!(+ "scope {} is not an admitted tenant scope",+ target_scope.as_str()+ );+ };+ let original_id = known+ .profiles+ .iter()+ .find(|id| {+ crate::storage::scope_for_profile(id).tenant_scope() == target_scope.tenant_scope()+ })+ .ok_or_else(|| {+ anyhow::anyhow!(+ "no known profile id maps to scope {}",+ target_scope.as_str()+ )+ })?;+ let Some(host) = crate::profiles::host::host() else {+ if let Some(target) = crate::storage::layout::Target::from_config(config) {+ let expected_url = format!("sqlite:{}", target.storage_db().display());+ let installed = crate::storage::installed();+ let backend_is_installed = installed+ .as_ref()+ .is_some_and(|installed| Arc::ptr_eq(installed, backend));+ if target.user_id == *original_id+ && crate::storage::configured_url(config).as_deref() == Some(&expected_url)+ && crate::storage::installed_url().as_deref() == Some(&expected_url)+ && backend_is_installed+ {+ return Ok((Arc::clone(backend), None));+ }+ }+ anyhow::bail!(+ "cannot copy profile scope {} without its admitted profile fence or dedicated profile database",+ target_scope.as_str()+ );+ };+ let profile_id = crate::profiles::ProfileId::parse(original_id).map_err(anyhow::Error::msg)?;+ let profile = host.get(&profile_id).ok_or_else(|| {+ anyhow::anyhow!(+ "profile {original_id} is not currently admitted on this host; refusing scope copy"+ )+ })?;+ let scoped = host+ .storage_backend_for(&profile)?+ .ok_or_else(|| anyhow::anyhow!("profile {original_id} has no captured storage backend"))?;+ Ok((scoped, Some(profile)))+}++#[cfg(feature = "legacy-import")]+fn scope_move_step(source_path: &Path, source_scope: &str) -> String {+ use sha2::Digest;+ let source_hash = sha2::Sha256::digest(source_path.as_os_str().as_encoded_bytes());+ let scope_hash = sha2::Sha256::digest(source_scope.as_bytes());+ format!(+ "scope_move/{}/{}",+ hex::encode(&source_hash[..8]),+ hex::encode(&scope_hash[..8])+ )+}++#[cfg(feature = "legacy-import")]+fn inventory_for_scope(+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+ source_scope: &str,+) -> crate::storage::import::scope_move::LegacyScopeInventory {+ crate::storage::import::scope_move::LegacyScopeInventory {+ scopes: vec![source_scope.to_owned()],+ documents: inventory+ .documents+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ streams: inventory+ .streams+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ blobs: inventory+ .blobs+ .iter()+ .filter(|row| row.scope == source_scope)+ .cloned()+ .collect(),+ }+}++#[cfg(feature = "legacy-import")]+fn inventory_scope_counts(+ inventory: &crate::storage::import::scope_move::LegacyScopeInventory,+ source_scope: &str,+) -> usize {+ inventory+ .documents+ .iter()+ .filter(|row| row.scope == source_scope)+ .count()+ + inventory+ .streams+ .iter()+ .filter(|row| row.scope == source_scope)+ .map(|row| row.entries.len())+ .sum::<usize>()+ + inventory+ .blobs+ .iter()+ .filter(|row| row.scope == source_scope)+ .count()+}+