Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): update storage import logic for legacy support

Steven Enamakel89075314a9e1
diff --git a/crates/openhuman-core/src/desktop/notifications/mod.rs b/crates/openhuman-core/src/desktop/notifications/mod.rs
index 505f42bb57..c75e7e1d4a 100644
--- a/crates/openhuman-core/src/desktop/notifications/mod.rs
+++ b/crates/openhuman-core/src/desktop/notifications/mod.rs
@@ -28,14 +28,16 @@ pub mod schemas;
mod storage_schema;
pub mod store;
mod store_documents;
+#[cfg(feature = "legacy-import")]
mod store_import;
pub(crate) use storage_schema::{collection_specs, upgrade_records};
+#[cfg(feature = "legacy-import")]
pub(crate) use store_documents::import_source;
pub mod types;
pub use bus::{
- publish_core_notification, register_notification_bridge_subscriber,
- subscribe_core_notifications, NotificationBridgeSubscriber,
+ NotificationBridgeSubscriber, publish_core_notification,
+ register_notification_bridge_subscriber, subscribe_core_notifications,
};
pub use schemas::{
all_controller_schemas as all_notifications_controller_schemas,
diff --git a/crates/openhuman-core/src/desktop/notifications/store_documents.rs b/crates/openhuman-core/src/desktop/notifications/store_documents.rs
index 4b038e409a..ba0bac6791 100644
--- a/crates/openhuman-core/src/desktop/notifications/store_documents.rs
+++ b/crates/openhuman-core/src/desktop/notifications/store_documents.rs
@@ -31,7 +31,7 @@ use std::sync::Arc;
use anyhow::{Context, Result};
use chrono::{DateTime, Duration, Utc};
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value, json};
use sha2::{Digest, Sha256};
use tinystoragedrivers::{CollectionSpec, ErrorKind, Filter, Precondition, Query, Sort, Versioned};
@@ -40,10 +40,10 @@ use super::types::{
NotificationStatus,
};
use crate::config::Config;
-use crate::storage::documents::{compare_and_swap, text, update_or_insert, Repo};
+use crate::storage::documents::{Repo, compare_and_swap, text, update_or_insert};
use crate::storage::{DocumentStore, DocumentStoreExt, StorageError};
-pub(super) use super::storage_schema::{core_id, CORE, DEDUP, NOTIFICATIONS, SETTINGS};
+pub(super) use super::storage_schema::{CORE, DEDUP, NOTIFICATIONS, SETTINGS, core_id};
use super::storage_schema::{legacy_collections, specs};
use crate::storage::schema_support::key_of;
@@ -62,6 +62,7 @@ const DEDUP_WINDOW_SECS: i64 = 60;
/// that may serve several workspaces gets them re-filed under the real
/// workspace's key, exactly as [`current`] does for new records
/// ([`workspace_key_for`]).
+#[cfg(feature = "legacy-import")]
pub(crate) fn import_source() -> crate::storage::import::Source {
crate::storage::import::Source {
domain: "notifications",
diff --git a/crates/openhuman-core/src/integrations/task_sources/mod.rs b/crates/openhuman-core/src/integrations/task_sources/mod.rs
index 821b47b215..c16bf6abf9 100644
--- a/crates/openhuman-core/src/integrations/task_sources/mod.rs
+++ b/crates/openhuman-core/src/integrations/task_sources/mod.rs
@@ -25,8 +25,10 @@ mod schemas;
mod storage_schema;
pub mod store;
mod store_documents;
+#[cfg(feature = "legacy-import")]
mod store_import;
pub(crate) use storage_schema::{collection_specs, upgrade_records};
+#[cfg(feature = "legacy-import")]
pub(crate) use store_documents::import_source;
pub mod tools;
pub mod types;
diff --git a/crates/openhuman-core/src/integrations/task_sources/store_documents.rs b/crates/openhuman-core/src/integrations/task_sources/store_documents.rs
index c904972c86..7111bf606e 100644
--- a/crates/openhuman-core/src/integrations/task_sources/store_documents.rs
+++ b/crates/openhuman-core/src/integrations/task_sources/store_documents.rs
@@ -19,26 +19,27 @@
//! fields (`created_ms`, `ingested_ms`) carry the ordering. Removing a source
//! removes its ledger entries, which the SQL schema did with a cascade.
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value, json};
use tinystoragedrivers::{CollectionSpec, Filter, Precondition, Query, Sort, Versioned};
-use super::store::{apply_patch, content_hash, IngestedTaskRef};
+use super::store::{IngestedTaskRef, apply_patch, content_hash};
use super::types::{
FetchReason, FilterSpec, ProviderSlug, SourceTarget, TaskSource, TaskSourcePatch,
};
use crate::config::Config;
use crate::integrations::composio::contract::NormalizedTask;
-use crate::storage::documents::{compare_and_swap, text, Repo};
+use crate::storage::documents::{Repo, compare_and_swap, text};
use crate::storage::{DocumentStoreExt, StorageError};
-pub(super) use super::storage_schema::{ingested_id, INGESTED, SOURCES};
+pub(super) use super::storage_schema::{INGESTED, SOURCES, ingested_id};
use super::storage_schema::{legacy_collections, specs};
use crate::storage::schema_support::key_of;
const DOMAIN: &str = "task_sources::store";
/// The legacy `sources.db`, as the storage importer reads it.
+#[cfg(feature = "legacy-import")]
pub(crate) fn import_source() -> crate::storage::import::Source {
crate::storage::import::Source {
domain: "task_sources",
@@ -226,11 +227,7 @@ impl Docs {
.await?;
docs.delete(SOURCES, &key, Precondition::None).await
})?;
- if removed {
- Ok(())
- } else {
- Err(not_found(id))
- }
+ if removed { Ok(()) } else { Err(not_found(id)) }
}
pub(super) fn record_fetch(
diff --git a/crates/openhuman-core/src/security/approval/mod.rs b/crates/openhuman-core/src/security/approval/mod.rs
index 8607d34442..033cd89355 100644
--- a/crates/openhuman-core/src/security/approval/mod.rs
+++ b/crates/openhuman-core/src/security/approval/mod.rs
@@ -23,14 +23,17 @@ pub mod schemas;
mod storage_schema;
pub mod store;
mod store_documents;
+#[cfg(feature = "legacy-import")]
mod store_import;
pub(crate) use storage_schema::{collection_specs, upgrade_records};
+#[cfg(feature = "legacy-import")]
pub(crate) use store_documents::import_source;
pub mod types;
pub use gate::{
- is_detached_request, is_unanswered_approval_reason, parse_approval_reply, ApprovalChatContext,
- ApprovalError, ApprovalGate, ApprovalTurnContext, FlowRunContext, APPROVAL_UNANSWERED_PHRASE,
+ APPROVAL_UNANSWERED_PHRASE, ApprovalChatContext, ApprovalError, ApprovalGate,
+ ApprovalTurnContext, FlowRunContext, is_detached_request, is_unanswered_approval_reason,
+ parse_approval_reply,
};
pub use redact::{redact_args, summarize_action};
pub use schemas::all_controller_schemas as all_approval_controller_schemas;
diff --git a/crates/openhuman-core/src/security/approval/store.rs b/crates/openhuman-core/src/security/approval/store.rs
index 3546a43dfa..e04c813f3f 100644
--- a/crates/openhuman-core/src/security/approval/store.rs
+++ b/crates/openhuman-core/src/security/approval/store.rs
@@ -46,6 +46,7 @@ pub use store_flow_trust::{
record_flow_preauthorization,
};
+#[cfg(feature = "legacy-import")]
pub use super::store_import::PRE_MIGRATION_SESSION_ID;
/// Insert a pending approval row. `session_id` is the per-launch UUID
diff --git a/crates/openhuman-core/src/security/approval/store_documents.rs b/crates/openhuman-core/src/security/approval/store_documents.rs
index a86306a589..9f8ec2b4f0 100644
--- a/crates/openhuman-core/src/security/approval/store_documents.rs
+++ b/crates/openhuman-core/src/security/approval/store_documents.rs
@@ -24,20 +24,20 @@
use std::sync::Arc;
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
-use serde_json::{json, Value};
+use serde_json::{Value, json};
use crate::config::Config;
use crate::storage::documents::compare_and_swap;
use crate::storage::local;
-use crate::storage::{block_on, DocumentStore, DocumentStoreExt, Storage, StorageError};
+use crate::storage::{DocumentStore, DocumentStoreExt, Storage, StorageError, block_on};
use tinystoragedrivers::{
CollectionSpec, ErrorKind, Filter, Key, Precondition, Query, Sort, Versioned,
};
+pub(super) use super::storage_schema::{APPROVALS, FLOW_TRUST, trust_id};
use super::storage_schema::{legacy_collections, specs};
-pub(super) use super::storage_schema::{trust_id, APPROVALS, FLOW_TRUST};
use super::types::{
ApprovalAuditEntry, ApprovalDecision, ApprovalSourceContext, ExecutionOutcome, PendingApproval,
@@ -74,6 +74,7 @@ pub(super) fn for_agent(config: &Config, agent: &str) -> Result<Docs> {
}
/// The legacy `approval.db`, as the storage importer reads it.
+#[cfg(feature = "legacy-import")]
pub(crate) fn import_source() -> crate::storage::import::Source {
crate::storage::import::Source {
domain: "approval",
diff --git a/crates/openhuman-core/src/security/devices/mod.rs b/crates/openhuman-core/src/security/devices/mod.rs
index d199ce6698..15bff29267 100644
--- a/crates/openhuman-core/src/security/devices/mod.rs
+++ b/crates/openhuman-core/src/security/devices/mod.rs
@@ -11,8 +11,10 @@ pub mod schemas;
mod storage_schema;
pub mod store;
mod store_documents;
+#[cfg(feature = "legacy-import")]
mod store_import;
pub(crate) use storage_schema::{collection_specs, upgrade_records};
+#[cfg(feature = "legacy-import")]
pub(crate) use store_documents::import_source;
pub mod tunnel_client;
pub mod types;
diff --git a/crates/openhuman-core/src/security/devices/store_documents.rs b/crates/openhuman-core/src/security/devices/store_documents.rs
index 7dd277068f..10edceddaa 100644
--- a/crates/openhuman-core/src/security/devices/store_documents.rs
+++ b/crates/openhuman-core/src/security/devices/store_documents.rs
@@ -13,20 +13,21 @@
use anyhow::Result;
use chrono::Utc;
-use serde_json::{json, Value};
+use serde_json::{Value, json};
use tinystoragedrivers::{CollectionSpec, Filter, Precondition, Query, Sort, Versioned};
use crate::config::Config;
use crate::security::devices::types::PairedDevice;
-use crate::storage::documents::{compare_and_swap, text, Repo};
use crate::storage::DocumentStoreExt;
+use crate::storage::documents::{Repo, compare_and_swap, text};
-use super::storage_schema::{legacy_collections, spec, DEVICES};
+use super::storage_schema::{DEVICES, legacy_collections, spec};
use crate::storage::schema_support::{key_of, ms_rfc3339};
const DOMAIN: &str = "devices::store";
/// The legacy `devices.db`, as the storage importer reads it.
+#[cfg(feature = "legacy-import")]
pub(crate) fn import_source() -> crate::storage::import::Source {
crate::storage::import::Source {
domain: "devices",
diff --git a/crates/openhuman-core/src/storage/import/mod.rs b/crates/openhuman-core/src/storage/import/mod.rs
index 51abeef76b..c821f847de 100644
--- a/crates/openhuman-core/src/storage/import/mod.rs
+++ b/crates/openhuman-core/src/storage/import/mod.rs
@@ -67,20 +67,20 @@ use std::sync::Arc;
use anyhow::Result;
#[cfg(feature = "legacy-import")]
-use anyhow::{anyhow, Context};
+use anyhow::{Context, anyhow};
#[cfg(feature = "legacy-import")]
use sha2::Digest;
use tinystoragedrivers::CollectionSpec;
#[cfg(feature = "legacy-import")]
use tinystoragedrivers::{ErrorKind, Precondition};
+use super::StorageBackend;
+#[cfg(feature = "legacy-import")]
+use super::block_on;
#[cfg(feature = "legacy-import")]
use super::documents::Repo;
#[cfg(feature = "legacy-import")]
use super::rusqlite;
-#[cfg(feature = "legacy-import")]
-use super::block_on;
-use super::StorageBackend;
use crate::config::Config;
#[cfg(feature = "legacy-import")]
use rusqlite::Connection;
@@ -352,8 +352,8 @@ pub fn run_with(
/// declare. A record in a collection no step reads passes through unchanged.
#[cfg(feature = "legacy-import")]
fn upgrade(owner: &str, records: Vec<Record>) -> Result<(Vec<Record>, Vec<CollectionSpec>)> {
- use tinystoragedrivers::schema::{Item, SourceRecord, StepBody};
use tinystoragedrivers::Meta;
+ use tinystoragedrivers::schema::{Item, SourceRecord, StepBody};
let schema = super::schema::owner_schemas()
.into_iter()
.find(|schema| schema.owner.as_str() == owner)
diff --git a/crates/openhuman-core/src/storage/import/schemas.rs b/crates/openhuman-core/src/storage/import/schemas.rs
index 75ed798888..1439aac173 100644
--- a/crates/openhuman-core/src/storage/import/schemas.rs
+++ b/crates/openhuman-core/src/storage/import/schemas.rs
@@ -3,10 +3,10 @@
use crate::core::runtime::DispatchCtx;
use serde::Deserialize;
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value, json};
-use crate::core::all::{ControllerFuture, RegisteredController};
use crate::core::Outcome;
+use crate::core::all::{ControllerFuture, RegisteredController};
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
#[derive(Debug, Deserialize, Default)]
@@ -102,51 +102,57 @@ fn handle_session_import_run(_ctx: DispatchCtx, params: Map<String, Value>) -> C
)
.into_cli_compatible_json();
}
- let configured = crate::config::Config::load_or_init()
- .await
- .map_err(|e| format!("failed to load config: {e}"))?;
- let configured_workspace = configured.workspace_dir.clone();
- let config = crate::config::Config {
- workspace_dir: workspace.clone(),
- ..configured
- };
- // A workspace override that differs from the configured one must not
- // write into the installed backend (bound to the configured workspace).
- let backend = if !overridden || workspace == configured_workspace {
- match crate::storage::installed() {
- Some(backend) => backend,
- None => crate::storage::local::backend_for(&workspace)
- .map_err(|e| format!("open the storage backend: {e:#}"))?,
- }
- } else {
- crate::storage::open(&crate::storage::config::default_url(&workspace))
+ #[cfg(not(feature = "legacy-import"))]
+ return Err("legacy session import requires the `legacy-import` feature".to_owned());
+
+ #[cfg(feature = "legacy-import")]
+ {
+ let configured = crate::config::Config::load_or_init()
.await
- .map_err(|e| format!("open the storage backend: {e:#}"))?
- };
- let reports = tokio::task::spawn_blocking(move || {
- crate::storage::import::sessions::run_with(&backend, &config)
- })
- .await
- .map_err(|e| format!("session import task failed: {e}"))?;
- let imports: Vec<Value> = reports
- .iter()
- .map(|report| match &report.outcome {
- Ok(outcome) => {
- json!({ "domain": report.domain, "outcome": format!("{outcome:?}") })
+ .map_err(|e| format!("failed to load config: {e}"))?;
+ let configured_workspace = configured.workspace_dir.clone();
+ let config = crate::config::Config {
+ workspace_dir: workspace.clone(),
+ ..configured
+ };
+ // A workspace override that differs from the configured one must not
+ // write into the installed backend (bound to the configured workspace).
+ let backend = if !overridden || workspace == configured_workspace {
+ match crate::storage::installed() {
+ Some(backend) => backend,
+ None => crate::storage::local::backend_for(&workspace)
+ .map_err(|e| format!("open the storage backend: {e:#}"))?,
}
- Err(error) => json!({ "domain": report.domain, "error": format!("{error:#}") }),
+ } else {
+ crate::storage::open(&crate::storage::config::default_url(&workspace))
+ .await
+ .map_err(|e| format!("open the storage backend: {e:#}"))?
+ };
+ let reports = tokio::task::spawn_blocking(move || {
+ crate::storage::import::sessions::run_with(&backend, &config)
})
- .collect();
- let logs = reports
- .iter()
- .filter_map(|report| report.outcome.as_ref().err())
- .map(|error| format!("{error:#}"))
- .collect();
- Outcome::new(
+ .await
+ .map_err(|e| format!("session import task failed: {e}"))?;
+ let imports: Vec<Value> = reports
+ .iter()
+ .map(|report| match &report.outcome {
+ Ok(outcome) => {
+ json!({ "domain": report.domain, "outcome": format!("{outcome:?}") })
+ }
+ Err(error) => json!({ "domain": report.domain, "error": format!("{error:#}") }),
+ })
+ .collect();
+ let logs = reports
+ .iter()
+ .filter_map(|report| report.outcome.as_ref().err())
+ .map(|error| format!("{error:#}"))
+ .collect();
+ Outcome::new(
json!({ "dry_run": false, "workspace": workspace, "sources": sources, "imports": imports }),
logs,
)
.into_cli_compatible_json()
+ }
})
}
diff --git a/crates/openhuman-core/src/storage/mod.rs b/crates/openhuman-core/src/storage/mod.rs
index 6e5d1584a6..ec26e5b068 100644
--- a/crates/openhuman-core/src/storage/mod.rs
+++ b/crates/openhuman-core/src/storage/mod.rs
@@ -44,18 +44,19 @@ pub mod rebind;
pub mod schema;
pub mod schema_support;
pub mod scope;
+#[cfg(feature = "legacy-import")]
pub mod scope_move;
pub mod secrets;
pub mod sessions;
pub mod state_kv;
pub use scope::{
- scope_for_agent, scope_for_module, scope_for_profile, tenant_scope, ParsedScope, Principal,
- Scope, Tenant,
+ ParsedScope, Principal, Scope, Tenant, scope_for_agent, scope_for_module, scope_for_profile,
+ tenant_scope,
};
use std::future::Future;
-use std::sync::{mpsc, Arc, LazyLock, Mutex, OnceLock, RwLock};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock, mpsc};
/// SQLite bindings re-exported by the storage driver crate, for the two
/// consumers that open a SQLite file that is not a storage backend: the
@@ -180,8 +181,8 @@ impl Blocking {
}
}
-use crate::config::schema::storage::redact_url;
use crate::config::Config;
+use crate::config::schema::storage::redact_url;
/// The environment variable that overrides `[storage] url`.
pub const STORAGE_URL_VAR: &str = "OPENHUMAN_STORAGE_URL";
diff --git a/crates/openhuman-core/src/storage/mod_tests.rs b/crates/openhuman-core/src/storage/mod_tests.rs
index 6f44f3643b..a621f717a2 100644
--- a/crates/openhuman-core/src/storage/mod_tests.rs
+++ b/crates/openhuman-core/src/storage/mod_tests.rs
@@ -91,12 +91,14 @@ fn install_installed_and_clear() {
assert!(slot.installed().is_none());
let backend: Arc<dyn StorageBackend> = Arc::new(tinystoragedrivers::MemoryStorage::new());
assert!(slot.install(Arc::clone(&backend)).is_none());
- assert!(slot
- .installed()
- .is_some_and(|got| Arc::ptr_eq(&got, &backend)));
- assert!(slot
- .install(Arc::new(tinystoragedrivers::MemoryStorage::new()))
- .is_some());
+ assert!(
+ slot.installed()
+ .is_some_and(|got| Arc::ptr_eq(&got, &backend))
+ );
+ assert!(
+ slot.install(Arc::new(tinystoragedrivers::MemoryStorage::new()))
+ .is_some()
+ );
assert!(slot.clear());
assert!(!slot.clear());
}
@@ -104,9 +106,11 @@ fn install_installed_and_clear() {
#[test]
fn agent_scopes_sit_under_the_local_tenant() {
assert_eq!(scope_for_agent("agent-7").as_str(), "local/agent:agent-7");
- assert!(scope_for_agent("has a space")
- .as_str()
- .starts_with("local/agent:~"));
+ assert!(
+ scope_for_agent("has a space")
+ .as_str()
+ .starts_with("local/agent:~")
+ );
}
#[test]
@@ -144,9 +148,11 @@ fn a_profile_scopes_storage_over_its_agents() {
let odd = scope_for_profile("has a space");
assert!(odd.as_str().starts_with("u:~"), "{odd}");
assert_ne!(odd, scope_for_profile("has a space"));
- assert!(scope_for_profile(&"x".repeat(400))
- .as_str()
- .starts_with("u:~"));
+ assert!(
+ scope_for_profile(&"x".repeat(400))
+ .as_str()
+ .starts_with("u:~")
+ );
}
#[test]
diff --git a/crates/openhuman-core/src/storage/rebind_tests.rs b/crates/openhuman-core/src/storage/rebind_tests.rs
index a0c267ee96..5bbe92f39d 100644
--- a/crates/openhuman-core/src/storage/rebind_tests.rs
+++ b/crates/openhuman-core/src/storage/rebind_tests.rs
@@ -1,4 +1,5 @@
use super::*;
+#[cfg(feature = "legacy-import")]
use crate::storage::import::Outcome;
use crate::storage::{CollectionSpec, Precondition, Scope};
use std::sync::Mutex;
@@ -98,6 +99,7 @@ fn the_new_workspace_is_a_different_database() {
}
#[test]
+#[cfg(feature = "legacy-import")]
fn the_legacy_files_of_the_new_workspace_are_imported() {
let first = TempDir::new().unwrap();
let second = TempDir::new().unwrap();