The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): update storage import logic for legacy support
Steven Enamakel89075314a9e1
diff --git a/crates/openhuman-core/src/desktop/notifications/mod.rs b/crates/openhuman-core/src/desktop/notifications/mod.rsindex 505f42bb57..c75e7e1d4a 100644--- a/crates/openhuman-core/src/desktop/notifications/mod.rs+++ b/crates/openhuman-core/src/desktop/notifications/mod.rs@@ -28,14 +28,16 @@ pub mod schemas;mod storage_schema;pub mod store;mod store_documents;+#[cfg(feature = "legacy-import")]mod store_import;pub(crate) use storage_schema::{collection_specs, upgrade_records};+#[cfg(feature = "legacy-import")]pub(crate) use store_documents::import_source;pub mod types;pub use bus::{- publish_core_notification, register_notification_bridge_subscriber,- subscribe_core_notifications, NotificationBridgeSubscriber,+ NotificationBridgeSubscriber, publish_core_notification,+ register_notification_bridge_subscriber, subscribe_core_notifications,};pub use schemas::{all_controller_schemas as all_notifications_controller_schemas,diff --git a/crates/openhuman-core/src/desktop/notifications/store_documents.rs b/crates/openhuman-core/src/desktop/notifications/store_documents.rsindex 4b038e409a..ba0bac6791 100644--- a/crates/openhuman-core/src/desktop/notifications/store_documents.rs+++ b/crates/openhuman-core/src/desktop/notifications/store_documents.rs@@ -31,7 +31,7 @@ use std::sync::Arc;use anyhow::{Context, Result};use chrono::{DateTime, Duration, Utc};-use serde_json::{json, Map, Value};+use serde_json::{Map, Value, json};use sha2::{Digest, Sha256};use tinystoragedrivers::{CollectionSpec, ErrorKind, Filter, Precondition, Query, Sort, Versioned};@@ -40,10 +40,10 @@ use super::types::{NotificationStatus,};use crate::config::Config;-use crate::storage::documents::{compare_and_swap, text, update_or_insert, Repo};+use crate::storage::documents::{Repo, compare_and_swap, text, update_or_insert};use crate::storage::{DocumentStore, DocumentStoreExt, StorageError};-pub(super) use super::storage_schema::{core_id, CORE, DEDUP, NOTIFICATIONS, SETTINGS};+pub(super) use super::storage_schema::{CORE, DEDUP, NOTIFICATIONS, SETTINGS, core_id};use super::storage_schema::{legacy_collections, specs};use crate::storage::schema_support::key_of;@@ -62,6 +62,7 @@ const DEDUP_WINDOW_SECS: i64 = 60;/// that may serve several workspaces gets them re-filed under the real/// workspace's key, exactly as [`current`] does for new records/// ([`workspace_key_for`]).+#[cfg(feature = "legacy-import")]pub(crate) fn import_source() -> crate::storage::import::Source {crate::storage::import::Source {domain: "notifications",diff --git a/crates/openhuman-core/src/integrations/task_sources/mod.rs b/crates/openhuman-core/src/integrations/task_sources/mod.rsindex 821b47b215..c16bf6abf9 100644--- a/crates/openhuman-core/src/integrations/task_sources/mod.rs+++ b/crates/openhuman-core/src/integrations/task_sources/mod.rs@@ -25,8 +25,10 @@ mod schemas;mod storage_schema;pub mod store;mod store_documents;+#[cfg(feature = "legacy-import")]mod store_import;pub(crate) use storage_schema::{collection_specs, upgrade_records};+#[cfg(feature = "legacy-import")]pub(crate) use store_documents::import_source;pub mod tools;pub mod types;diff --git a/crates/openhuman-core/src/integrations/task_sources/store_documents.rs b/crates/openhuman-core/src/integrations/task_sources/store_documents.rsindex c904972c86..7111bf606e 100644--- a/crates/openhuman-core/src/integrations/task_sources/store_documents.rs+++ b/crates/openhuman-core/src/integrations/task_sources/store_documents.rs@@ -19,26 +19,27 @@//! fields (`created_ms`, `ingested_ms`) carry the ordering. Removing a source//! removes its ledger entries, which the SQL schema did with a cascade.-use anyhow::{anyhow, Context, Result};+use anyhow::{Context, Result, anyhow};use chrono::{DateTime, Utc};-use serde_json::{json, Map, Value};+use serde_json::{Map, Value, json};use tinystoragedrivers::{CollectionSpec, Filter, Precondition, Query, Sort, Versioned};-use super::store::{apply_patch, content_hash, IngestedTaskRef};+use super::store::{IngestedTaskRef, apply_patch, content_hash};use super::types::{FetchReason, FilterSpec, ProviderSlug, SourceTarget, TaskSource, TaskSourcePatch,};use crate::config::Config;use crate::integrations::composio::contract::NormalizedTask;-use crate::storage::documents::{compare_and_swap, text, Repo};+use crate::storage::documents::{Repo, compare_and_swap, text};use crate::storage::{DocumentStoreExt, StorageError};-pub(super) use super::storage_schema::{ingested_id, INGESTED, SOURCES};+pub(super) use super::storage_schema::{INGESTED, SOURCES, ingested_id};use super::storage_schema::{legacy_collections, specs};use crate::storage::schema_support::key_of;const DOMAIN: &str = "task_sources::store";/// The legacy `sources.db`, as the storage importer reads it.+#[cfg(feature = "legacy-import")]pub(crate) fn import_source() -> crate::storage::import::Source {crate::storage::import::Source {domain: "task_sources",@@ -226,11 +227,7 @@ impl Docs {.await?;docs.delete(SOURCES, &key, Precondition::None).await})?;- if removed {- Ok(())- } else {- Err(not_found(id))- }+ if removed { Ok(()) } else { Err(not_found(id)) }}pub(super) fn record_fetch(diff --git a/crates/openhuman-core/src/security/approval/mod.rs b/crates/openhuman-core/src/security/approval/mod.rsindex 8607d34442..033cd89355 100644--- a/crates/openhuman-core/src/security/approval/mod.rs+++ b/crates/openhuman-core/src/security/approval/mod.rs@@ -23,14 +23,17 @@ pub mod schemas;mod storage_schema;pub mod store;mod store_documents;+#[cfg(feature = "legacy-import")]mod store_import;pub(crate) use storage_schema::{collection_specs, upgrade_records};+#[cfg(feature = "legacy-import")]pub(crate) use store_documents::import_source;pub mod types;pub use gate::{- is_detached_request, is_unanswered_approval_reason, parse_approval_reply, ApprovalChatContext,- ApprovalError, ApprovalGate, ApprovalTurnContext, FlowRunContext, APPROVAL_UNANSWERED_PHRASE,+ APPROVAL_UNANSWERED_PHRASE, ApprovalChatContext, ApprovalError, ApprovalGate,+ ApprovalTurnContext, FlowRunContext, is_detached_request, is_unanswered_approval_reason,+ parse_approval_reply,};pub use redact::{redact_args, summarize_action};pub use schemas::all_controller_schemas as all_approval_controller_schemas;diff --git a/crates/openhuman-core/src/security/approval/store.rs b/crates/openhuman-core/src/security/approval/store.rsindex 3546a43dfa..e04c813f3f 100644--- a/crates/openhuman-core/src/security/approval/store.rs+++ b/crates/openhuman-core/src/security/approval/store.rs@@ -46,6 +46,7 @@ pub use store_flow_trust::{record_flow_preauthorization,};+#[cfg(feature = "legacy-import")]pub use super::store_import::PRE_MIGRATION_SESSION_ID;/// Insert a pending approval row. `session_id` is the per-launch UUIDdiff --git a/crates/openhuman-core/src/security/approval/store_documents.rs b/crates/openhuman-core/src/security/approval/store_documents.rsindex a86306a589..9f8ec2b4f0 100644--- a/crates/openhuman-core/src/security/approval/store_documents.rs+++ b/crates/openhuman-core/src/security/approval/store_documents.rs@@ -24,20 +24,20 @@use std::sync::Arc;-use anyhow::{anyhow, Context, Result};+use anyhow::{Context, Result, anyhow};use chrono::{DateTime, Utc};-use serde_json::{json, Value};+use serde_json::{Value, json};use crate::config::Config;use crate::storage::documents::compare_and_swap;use crate::storage::local;-use crate::storage::{block_on, DocumentStore, DocumentStoreExt, Storage, StorageError};+use crate::storage::{DocumentStore, DocumentStoreExt, Storage, StorageError, block_on};use tinystoragedrivers::{CollectionSpec, ErrorKind, Filter, Key, Precondition, Query, Sort, Versioned,};+pub(super) use super::storage_schema::{APPROVALS, FLOW_TRUST, trust_id};use super::storage_schema::{legacy_collections, specs};-pub(super) use super::storage_schema::{trust_id, APPROVALS, FLOW_TRUST};use super::types::{ApprovalAuditEntry, ApprovalDecision, ApprovalSourceContext, ExecutionOutcome, PendingApproval,@@ -74,6 +74,7 @@ pub(super) fn for_agent(config: &Config, agent: &str) -> Result<Docs> {}/// The legacy `approval.db`, as the storage importer reads it.+#[cfg(feature = "legacy-import")]pub(crate) fn import_source() -> crate::storage::import::Source {crate::storage::import::Source {domain: "approval",diff --git a/crates/openhuman-core/src/security/devices/mod.rs b/crates/openhuman-core/src/security/devices/mod.rsindex d199ce6698..15bff29267 100644--- a/crates/openhuman-core/src/security/devices/mod.rs+++ b/crates/openhuman-core/src/security/devices/mod.rs@@ -11,8 +11,10 @@ pub mod schemas;mod storage_schema;pub mod store;mod store_documents;+#[cfg(feature = "legacy-import")]mod store_import;pub(crate) use storage_schema::{collection_specs, upgrade_records};+#[cfg(feature = "legacy-import")]pub(crate) use store_documents::import_source;pub mod tunnel_client;pub mod types;diff --git a/crates/openhuman-core/src/security/devices/store_documents.rs b/crates/openhuman-core/src/security/devices/store_documents.rsindex 7dd277068f..10edceddaa 100644--- a/crates/openhuman-core/src/security/devices/store_documents.rs+++ b/crates/openhuman-core/src/security/devices/store_documents.rs@@ -13,20 +13,21 @@use anyhow::Result;use chrono::Utc;-use serde_json::{json, Value};+use serde_json::{Value, json};use tinystoragedrivers::{CollectionSpec, Filter, Precondition, Query, Sort, Versioned};use crate::config::Config;use crate::security::devices::types::PairedDevice;-use crate::storage::documents::{compare_and_swap, text, Repo};use crate::storage::DocumentStoreExt;+use crate::storage::documents::{Repo, compare_and_swap, text};-use super::storage_schema::{legacy_collections, spec, DEVICES};+use super::storage_schema::{DEVICES, legacy_collections, spec};use crate::storage::schema_support::{key_of, ms_rfc3339};const DOMAIN: &str = "devices::store";/// The legacy `devices.db`, as the storage importer reads it.+#[cfg(feature = "legacy-import")]pub(crate) fn import_source() -> crate::storage::import::Source {crate::storage::import::Source {domain: "devices",diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rsindex 51abeef76b..c821f847de 100644--- a/crates/openhuman-core/src/storage/import/mod.rs+++ b/crates/openhuman-core/src/storage/import/mod.rs@@ -67,20 +67,20 @@ use std::sync::Arc;use anyhow::Result;#[cfg(feature = "legacy-import")]-use anyhow::{anyhow, Context};+use anyhow::{Context, anyhow};#[cfg(feature = "legacy-import")]use sha2::Digest;use tinystoragedrivers::CollectionSpec;#[cfg(feature = "legacy-import")]use tinystoragedrivers::{ErrorKind, Precondition};+use super::StorageBackend;+#[cfg(feature = "legacy-import")]+use super::block_on;#[cfg(feature = "legacy-import")]use super::documents::Repo;#[cfg(feature = "legacy-import")]use super::rusqlite;-#[cfg(feature = "legacy-import")]-use super::block_on;-use super::StorageBackend;use crate::config::Config;#[cfg(feature = "legacy-import")]use rusqlite::Connection;@@ -352,8 +352,8 @@ pub fn run_with(/// declare. A record in a collection no step reads passes through unchanged.#[cfg(feature = "legacy-import")]fn upgrade(owner: &str, records: Vec<Record>) -> Result<(Vec<Record>, Vec<CollectionSpec>)> {- use tinystoragedrivers::schema::{Item, SourceRecord, StepBody};use tinystoragedrivers::Meta;+ use tinystoragedrivers::schema::{Item, SourceRecord, StepBody};let schema = super::schema::owner_schemas().into_iter().find(|schema| schema.owner.as_str() == owner)diff --git a/crates/openhuman-core/src/storage/import/schemas.rs b/crates/openhuman-core/src/storage/import/schemas.rsindex 75ed798888..1439aac173 100644--- a/crates/openhuman-core/src/storage/import/schemas.rs+++ b/crates/openhuman-core/src/storage/import/schemas.rs@@ -3,10 +3,10 @@use crate::core::runtime::DispatchCtx;use serde::Deserialize;-use serde_json::{json, Map, Value};+use serde_json::{Map, Value, json};-use crate::core::all::{ControllerFuture, RegisteredController};use crate::core::Outcome;+use crate::core::all::{ControllerFuture, RegisteredController};use crate::core::{ControllerSchema, FieldSchema, TypeSchema};#[derive(Debug, Deserialize, Default)]@@ -102,51 +102,57 @@ fn handle_session_import_run(_ctx: DispatchCtx, params: Map<String, Value>) -> C).into_cli_compatible_json();}- let configured = crate::config::Config::load_or_init()- .await- .map_err(|e| format!("failed to load config: {e}"))?;- let configured_workspace = configured.workspace_dir.clone();- let config = crate::config::Config {- workspace_dir: workspace.clone(),- ..configured- };- // A workspace override that differs from the configured one must not- // write into the installed backend (bound to the configured workspace).- let backend = if !overridden || workspace == configured_workspace {- match crate::storage::installed() {- Some(backend) => backend,- None => crate::storage::local::backend_for(&workspace)- .map_err(|e| format!("open the storage backend: {e:#}"))?,- }- } else {- crate::storage::open(&crate::storage::config::default_url(&workspace))+ #[cfg(not(feature = "legacy-import"))]+ return Err("legacy session import requires the `legacy-import` feature".to_owned());++ #[cfg(feature = "legacy-import")]+ {+ let configured = crate::config::Config::load_or_init().await- .map_err(|e| format!("open the storage backend: {e:#}"))?- };- let reports = tokio::task::spawn_blocking(move || {- crate::storage::import::sessions::run_with(&backend, &config)- })- .await- .map_err(|e| format!("session import task failed: {e}"))?;- let imports: Vec<Value> = reports- .iter()- .map(|report| match &report.outcome {- Ok(outcome) => {- json!({ "domain": report.domain, "outcome": format!("{outcome:?}") })+ .map_err(|e| format!("failed to load config: {e}"))?;+ let configured_workspace = configured.workspace_dir.clone();+ let config = crate::config::Config {+ workspace_dir: workspace.clone(),+ ..configured+ };+ // A workspace override that differs from the configured one must not+ // write into the installed backend (bound to the configured workspace).+ let backend = if !overridden || workspace == configured_workspace {+ match crate::storage::installed() {+ Some(backend) => backend,+ None => crate::storage::local::backend_for(&workspace)+ .map_err(|e| format!("open the storage backend: {e:#}"))?,}- Err(error) => json!({ "domain": report.domain, "error": format!("{error:#}") }),+ } else {+ crate::storage::open(&crate::storage::config::default_url(&workspace))+ .await+ .map_err(|e| format!("open the storage backend: {e:#}"))?+ };+ let reports = tokio::task::spawn_blocking(move || {+ crate::storage::import::sessions::run_with(&backend, &config)})- .collect();- let logs = reports- .iter()- .filter_map(|report| report.outcome.as_ref().err())- .map(|error| format!("{error:#}"))- .collect();- Outcome::new(+ .await+ .map_err(|e| format!("session import task failed: {e}"))?;+ let imports: Vec<Value> = reports+ .iter()+ .map(|report| match &report.outcome {+ Ok(outcome) => {+ json!({ "domain": report.domain, "outcome": format!("{outcome:?}") })+ }+ Err(error) => json!({ "domain": report.domain, "error": format!("{error:#}") }),+ })+ .collect();+ let logs = reports+ .iter()+ .filter_map(|report| report.outcome.as_ref().err())+ .map(|error| format!("{error:#}"))+ .collect();+ Outcome::new(json!({ "dry_run": false, "workspace": workspace, "sources": sources, "imports": imports }),logs,).into_cli_compatible_json()+ }})}diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rsindex 6e5d1584a6..ec26e5b068 100644--- a/crates/openhuman-core/src/storage/mod.rs+++ b/crates/openhuman-core/src/storage/mod.rs@@ -44,18 +44,19 @@ pub mod rebind;pub mod schema;pub mod schema_support;pub mod scope;+#[cfg(feature = "legacy-import")]pub mod scope_move;pub mod secrets;pub mod sessions;pub mod state_kv;pub use scope::{- scope_for_agent, scope_for_module, scope_for_profile, tenant_scope, ParsedScope, Principal,- Scope, Tenant,+ ParsedScope, Principal, Scope, Tenant, scope_for_agent, scope_for_module, scope_for_profile,+ tenant_scope,};use std::future::Future;-use std::sync::{mpsc, Arc, LazyLock, Mutex, OnceLock, RwLock};+use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock, mpsc};/// SQLite bindings re-exported by the storage driver crate, for the two/// consumers that open a SQLite file that is not a storage backend: the@@ -180,8 +181,8 @@ impl Blocking {}}-use crate::config::schema::storage::redact_url;use crate::config::Config;+use crate::config::schema::storage::redact_url;/// The environment variable that overrides `[storage] url`.pub const STORAGE_URL_VAR: &str = "OPENHUMAN_STORAGE_URL";diff --git a/crates/openhuman-core/src/storage/mod_tests.rs b/crates/openhuman-core/src/storage/mod_tests.rsindex 6f44f3643b..a621f717a2 100644--- a/crates/openhuman-core/src/storage/mod_tests.rs+++ b/crates/openhuman-core/src/storage/mod_tests.rs@@ -91,12 +91,14 @@ fn install_installed_and_clear() {assert!(slot.installed().is_none());let backend: Arc<dyn StorageBackend> = Arc::new(tinystoragedrivers::MemoryStorage::new());assert!(slot.install(Arc::clone(&backend)).is_none());- assert!(slot- .installed()- .is_some_and(|got| Arc::ptr_eq(&got, &backend)));- assert!(slot- .install(Arc::new(tinystoragedrivers::MemoryStorage::new()))- .is_some());+ assert!(+ slot.installed()+ .is_some_and(|got| Arc::ptr_eq(&got, &backend))+ );+ assert!(+ slot.install(Arc::new(tinystoragedrivers::MemoryStorage::new()))+ .is_some()+ );assert!(slot.clear());assert!(!slot.clear());}@@ -104,9 +106,11 @@ fn install_installed_and_clear() {#[test]fn agent_scopes_sit_under_the_local_tenant() {assert_eq!(scope_for_agent("agent-7").as_str(), "local/agent:agent-7");- assert!(scope_for_agent("has a space")- .as_str()- .starts_with("local/agent:~"));+ assert!(+ scope_for_agent("has a space")+ .as_str()+ .starts_with("local/agent:~")+ );}#[test]@@ -144,9 +148,11 @@ fn a_profile_scopes_storage_over_its_agents() {let odd = scope_for_profile("has a space");assert!(odd.as_str().starts_with("u:~"), "{odd}");assert_ne!(odd, scope_for_profile("has a space"));- assert!(scope_for_profile(&"x".repeat(400))- .as_str()- .starts_with("u:~"));+ assert!(+ scope_for_profile(&"x".repeat(400))+ .as_str()+ .starts_with("u:~")+ );}#[test]diff --git a/crates/openhuman-core/src/storage/rebind_tests.rs b/crates/openhuman-core/src/storage/rebind_tests.rsindex a0c267ee96..5bbe92f39d 100644--- a/crates/openhuman-core/src/storage/rebind_tests.rs+++ b/crates/openhuman-core/src/storage/rebind_tests.rs@@ -1,4 +1,5 @@use super::*;+#[cfg(feature = "legacy-import")]use crate::storage::import::Outcome;use crate::storage::{CollectionSpec, Precondition, Scope};use std::sync::Mutex;@@ -98,6 +99,7 @@ fn the_new_workspace_is_a_different_database() {}#[test]+#[cfg(feature = "legacy-import")]fn the_legacy_files_of_the_new_workspace_are_imported() {let first = TempDir::new().unwrap();let second = TempDir::new().unwrap();