The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(core): update legacy delivery import logic
Steven Enamakeld70a27ea06b1
diff --git a/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs b/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rsindex 486d393921..f035b75f25 100644--- a/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs+++ b/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs@@ -24,7 +24,8 @@ use tinychannels_bus::module_delivery::{DeliveryQueueEntry, DeliveryTerminalResult, DELIVERY_PENDING_CAPACITY, DELIVERY_RECORD_BYTES,DELIVERY_RESULT_CAPACITY, DELIVERY_RESULT_TTL_SECONDS,};-use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition};+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, StorageBackend};+use crate::storage::{Scope, documents::Repo};const LOG_PREFIX: &str = "[channel-delivery-import]";const MODULE_ID: &str = "tinychannels";@@ -55,6 +56,8 @@ pub(super) enum Outcome {NoBackend,/// The marker says the import already finished.AlreadyDone,+ /// The claim could not be acquired before its wait budget expired.+ InProgress,/// There is no legacy file.NoSource,/// The legacy file was read and its records written.@@ -72,43 +75,92 @@ pub(super) async fn import_legacy_delivery(};let scope = crate::modules::storage_host::current_module_scope(ctx, MODULE_ID).map_err(|error| anyhow::anyhow!("resolve the module scope: {:?}", error.kind()))?;- let scoped = backend- .for_scope(&scope)- .map_err(|error| anyhow::anyhow!("open the module scope: {:?}", error.kind()))?;- import_into(&scoped, workspace).await+ import_into(&backend, &scope, workspace).await}async fn import_into(- scoped: &tinystoragedrivers::ScopedStorage,+ backend: &std::sync::Arc<dyn StorageBackend>,+ scope: &Scope,workspace: &Path,) -> anyhow::Result<Outcome> {- let docs = scoped.documents();let path = workspace.join(LEGACY_FILE);let marker = marker_id(workspace);- if crate::storage::import_state::host_import_done(scoped, MARKER_COLLECTION, &marker)+ let markers = Repo::on(+ backend,+ scope,+ "storage::import",+ crate::storage::import_state::collection_specs,+ )?;+ let initial = backend+ .for_scope(scope)+ .map_err(|error| anyhow::anyhow!("open the module scope: {:?}", error.kind()))?;+ if crate::storage::import_state::host_import_done(&initial, MARKER_COLLECTION, &marker).await.map_err(|error| anyhow::anyhow!("read the import state: {:?}", error.kind()))?{return Ok(Outcome::AlreadyDone);}- let bytes = match tokio::fs::read(&path).await {+ let step = crate::storage::import_state::step_id(MARKER_COLLECTION, &marker);+ let claimed = crate::storage::import_state::with_claim(+ &markers,+ crate::storage::import_state::HOST_OWNER,+ &step,+ |condition| {+ let claimed_markers = markers.conditioned(condition)?;+ let claimed_backend = crate::storage::import_state::backend_with_condition(+ std::sync::Arc::clone(backend),+ condition.clone(),+ );+ let scoped = claimed_backend+ .for_scope(scope)+ .map_err(|error| anyhow::anyhow!("open claimed module scope: {:?}", error.kind()))?;+ crate::storage::block_on_anyhow(async move {+ if crate::storage::import_state::host_import_done(+ claimed_markers.scoped(),+ MARKER_COLLECTION,+ &marker,+ )+ .await+ .map_err(|error| anyhow::anyhow!("read the import state: {:?}", error.kind()))?+ {+ return Ok(Outcome::AlreadyDone);+ }+ let outcome = import_payload(&scoped, &path).await?;+ if matches!(outcome, Outcome::NoSource) {+ return Ok(outcome);+ }+ crate::storage::import_state::host_import_mark(+ claimed_markers.scoped(),+ MARKER_COLLECTION,+ &marker,+ )+ .await+ .map_err(|error| anyhow::anyhow!("write the import state: {:?}", error.kind()))?;+ Ok(outcome)+ })+ },+ )?;+ Ok(claimed.unwrap_or(Outcome::InProgress))+}++async fn import_payload(+ scoped: &tinystoragedrivers::ScopedStorage,+ path: &Path,+) -> anyhow::Result<Outcome> {+ let bytes = match tokio::fs::read(path).await {Ok(bytes) => bytes,Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Outcome::NoSource),Err(error) => return Err(error.into()),};- anyhow::ensure!(- bytes.len() as u64 <= MAX_LEGACY_BYTES,- "legacy delivery file is oversized"- );+ anyhow::ensure!(bytes.len() as u64 <= MAX_LEGACY_BYTES, "legacy delivery file is oversized");let legacy: LegacyState = serde_json::from_slice(&bytes).map_err(|error| anyhow::anyhow!("legacy delivery file is unreadable: {error}"))?;-+ let docs = scoped.documents();for name in [PENDING, FAILED, RESULTS] {docs.ensure_collection(&CollectionSpec::new(name)).await.map_err(|error| anyhow::anyhow!("declare {name}: {:?}", error.kind()))?;}-let now = now_unix_ms();let ttl_ms = DELIVERY_RESULT_TTL_SECONDS.saturating_mul(1000);let mut records: Vec<(&str, String, Value)> = Vec::new();@@ -134,7 +186,6 @@ async fn import_into(});records.push((RESULTS, id, doc));}-let found = records.len();let mut written = 0usize;for (collection, id, doc) in records {@@ -142,17 +193,9 @@ async fn import_into(Ok(_) => written += 1,Err(error)if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => {}- Err(error) => {- return Err(anyhow::anyhow!(- "write {collection} record: {:?}",- error.kind()- ))- }+ Err(error) => return Err(anyhow::anyhow!("write {collection} record: {:?}", error.kind())),}}- crate::storage::import_state::host_import_mark(scoped, MARKER_COLLECTION, &marker)- .await- .map_err(|error| anyhow::anyhow!("write the import state: {:?}", error.kind()))?;tracing::info!("{LOG_PREFIX} imported found={found} written={written}");Ok(Outcome::Imported { found, written })}