Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(core): update legacy delivery import logic

Steven Enamakeld70a27ea06b1
diff --git a/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs b/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs
index 486d393921..f035b75f25 100644
--- a/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs
+++ b/crates/openhuman-core/src/channels/runtime/startup/delivery_import.rs
@@ -24,7 +24,8 @@ use tinychannels_bus::module_delivery::{
DeliveryQueueEntry, DeliveryTerminalResult, DELIVERY_PENDING_CAPACITY, DELIVERY_RECORD_BYTES,
DELIVERY_RESULT_CAPACITY, DELIVERY_RESULT_TTL_SECONDS,
};
-use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition};
+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, StorageBackend};
+use crate::storage::{Scope, documents::Repo};
const LOG_PREFIX: &str = "[channel-delivery-import]";
const MODULE_ID: &str = "tinychannels";
@@ -55,6 +56,8 @@ pub(super) enum Outcome {
NoBackend,
/// The marker says the import already finished.
AlreadyDone,
+ /// The claim could not be acquired before its wait budget expired.
+ InProgress,
/// There is no legacy file.
NoSource,
/// The legacy file was read and its records written.
@@ -72,43 +75,92 @@ pub(super) async fn import_legacy_delivery(
};
let scope = crate::modules::storage_host::current_module_scope(ctx, MODULE_ID)
.map_err(|error| anyhow::anyhow!("resolve the module scope: {:?}", error.kind()))?;
- let scoped = backend
- .for_scope(&scope)
- .map_err(|error| anyhow::anyhow!("open the module scope: {:?}", error.kind()))?;
- import_into(&scoped, workspace).await
+ import_into(&backend, &scope, workspace).await
}
async fn import_into(
- scoped: &tinystoragedrivers::ScopedStorage,
+ backend: &std::sync::Arc<dyn StorageBackend>,
+ scope: &Scope,
workspace: &Path,
) -> anyhow::Result<Outcome> {
- let docs = scoped.documents();
let path = workspace.join(LEGACY_FILE);
let marker = marker_id(workspace);
- if crate::storage::import_state::host_import_done(scoped, MARKER_COLLECTION, &marker)
+ let markers = Repo::on(
+ backend,
+ scope,
+ "storage::import",
+ crate::storage::import_state::collection_specs,
+ )?;
+ let initial = backend
+ .for_scope(scope)
+ .map_err(|error| anyhow::anyhow!("open the module scope: {:?}", error.kind()))?;
+ if crate::storage::import_state::host_import_done(&initial, MARKER_COLLECTION, &marker)
.await
.map_err(|error| anyhow::anyhow!("read the import state: {:?}", error.kind()))?
{
return Ok(Outcome::AlreadyDone);
}
- let bytes = match tokio::fs::read(&path).await {
+ let step = crate::storage::import_state::step_id(MARKER_COLLECTION, &marker);
+ let claimed = crate::storage::import_state::with_claim(
+ &markers,
+ crate::storage::import_state::HOST_OWNER,
+ &step,
+ |condition| {
+ let claimed_markers = markers.conditioned(condition)?;
+ let claimed_backend = crate::storage::import_state::backend_with_condition(
+ std::sync::Arc::clone(backend),
+ condition.clone(),
+ );
+ let scoped = claimed_backend
+ .for_scope(scope)
+ .map_err(|error| anyhow::anyhow!("open claimed module scope: {:?}", error.kind()))?;
+ crate::storage::block_on_anyhow(async move {
+ if crate::storage::import_state::host_import_done(
+ claimed_markers.scoped(),
+ MARKER_COLLECTION,
+ &marker,
+ )
+ .await
+ .map_err(|error| anyhow::anyhow!("read the import state: {:?}", error.kind()))?
+ {
+ return Ok(Outcome::AlreadyDone);
+ }
+ let outcome = import_payload(&scoped, &path).await?;
+ if matches!(outcome, Outcome::NoSource) {
+ return Ok(outcome);
+ }
+ crate::storage::import_state::host_import_mark(
+ claimed_markers.scoped(),
+ MARKER_COLLECTION,
+ &marker,
+ )
+ .await
+ .map_err(|error| anyhow::anyhow!("write the import state: {:?}", error.kind()))?;
+ Ok(outcome)
+ })
+ },
+ )?;
+ Ok(claimed.unwrap_or(Outcome::InProgress))
+}
+
+async fn import_payload(
+ scoped: &tinystoragedrivers::ScopedStorage,
+ path: &Path,
+) -> anyhow::Result<Outcome> {
+ let bytes = match tokio::fs::read(path).await {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Outcome::NoSource),
Err(error) => return Err(error.into()),
};
- anyhow::ensure!(
- bytes.len() as u64 <= MAX_LEGACY_BYTES,
- "legacy delivery file is oversized"
- );
+ anyhow::ensure!(bytes.len() as u64 <= MAX_LEGACY_BYTES, "legacy delivery file is oversized");
let legacy: LegacyState = serde_json::from_slice(&bytes)
.map_err(|error| anyhow::anyhow!("legacy delivery file is unreadable: {error}"))?;
-
+ let docs = scoped.documents();
for name in [PENDING, FAILED, RESULTS] {
docs.ensure_collection(&CollectionSpec::new(name))
.await
.map_err(|error| anyhow::anyhow!("declare {name}: {:?}", error.kind()))?;
}
-
let now = now_unix_ms();
let ttl_ms = DELIVERY_RESULT_TTL_SECONDS.saturating_mul(1000);
let mut records: Vec<(&str, String, Value)> = Vec::new();
@@ -134,7 +186,6 @@ async fn import_into(
});
records.push((RESULTS, id, doc));
}
-
let found = records.len();
let mut written = 0usize;
for (collection, id, doc) in records {
@@ -142,17 +193,9 @@ async fn import_into(
Ok(_) => written += 1,
Err(error)
if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => {}
- Err(error) => {
- return Err(anyhow::anyhow!(
- "write {collection} record: {:?}",
- error.kind()
- ))
- }
+ Err(error) => return Err(anyhow::anyhow!("write {collection} record: {:?}", error.kind())),
}
}
- crate::storage::import_state::host_import_mark(scoped, MARKER_COLLECTION, &marker)
- .await
- .map_err(|error| anyhow::anyhow!("write the import state: {:?}", error.kind()))?;
tracing::info!("{LOG_PREFIX} imported found={found} written={written}");
Ok(Outcome::Imported { found, written })
}