Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

chore: files changed crates/openhuman-core/src/modules/storage_import_support.rs,crates/openhuman-co

Steven Enamakel0e137d6692e3
diff --git a/crates/openhuman-core/src/modules/storage_import_support.rs b/crates/openhuman-core/src/modules/storage_import_support.rs
index e02949c88a..bd74cb730c 100644
--- a/crates/openhuman-core/src/modules/storage_import_support.rs
+++ b/crates/openhuman-core/src/modules/storage_import_support.rs
@@ -8,9 +8,8 @@
use std::path::Path;
use std::sync::Arc;
-use std::future::Future;
-use crate::storage::{DocumentStore, ErrorKind, Precondition, Scope, Storage, StorageBackend};
+use crate::storage::{DocumentStore, ErrorKind, Precondition, Storage};
use anyhow::{Result, anyhow};
use serde_json::Value;
use sha2::Digest;
diff --git a/crates/openhuman-core/src/modules/storage_import_tinybox.rs b/crates/openhuman-core/src/modules/storage_import_tinybox.rs
index ca5d730cc4..e94d01c681 100644
--- a/crates/openhuman-core/src/modules/storage_import_tinybox.rs
+++ b/crates/openhuman-core/src/modules/storage_import_tinybox.rs
@@ -53,18 +53,26 @@ pub(crate) enum Outcome {
/// Imports the index under `<workspace_dir>/jails` into the module's scope of
/// the installed backend.
pub(crate) async fn import_legacy(ctx: &DispatchCtx, workspace_dir: &Path) -> Result<Outcome> {
- let Some(scoped) = support::module_scope(ctx, MODULE_ID)? else {
+ let base = workspace_dir.join(BASE_DIR);
+ let index = base.join(INDEX_FILE);
+ if !index.is_file() {
+ return Ok(Outcome::NoSource);
+ }
+ let marker = support::marker_id("tinybox", &base);
+ let result = support::with_module_claim(ctx, MODULE_ID, MARKER_COLLECTION, &marker, move |scoped| {
+ crate::storage::block_on_anyhow(async move { import_into(&scoped, &base).await })
+ })?;
+ let Some(result) = result else {
return Ok(Outcome::NoBackend);
};
- import_into(&scoped, &workspace_dir.join(BASE_DIR)).await
+ Ok(match result {
+ support::ClaimedImport::AlreadyDone => Outcome::AlreadyDone,
+ support::ClaimedImport::Imported(outcome) => outcome,
+ })
}
async fn import_into(scoped: &Storage, base: &Path) -> Result<Outcome> {
let docs = scoped.documents();
- let marker = support::marker_id("tinybox", base);
- if support::already_imported(scoped, MARKER_COLLECTION, &marker).await? {
- return Ok(Outcome::AlreadyDone);
- }
let index = base.join(INDEX_FILE);
if !index.is_file() {
return Ok(Outcome::NoSource);
@@ -99,7 +107,6 @@ async fn import_into(scoped: &Storage, base: &Path) -> Result<Outcome> {
_ => tracing::warn!("{LOG_PREFIX} skipped an index file that is not a JSON object"),
}
- support::mark_imported(scoped, MARKER_COLLECTION, &marker).await?;
tracing::info!("{LOG_PREFIX} imported written={written}");
Ok(Outcome::Imported { written })
}
diff --git a/crates/openhuman-core/src/modules/storage_import_tinyconnectors.rs b/crates/openhuman-core/src/modules/storage_import_tinyconnectors.rs
index 1ace3e5ae5..d78b2ed52d 100644
--- a/crates/openhuman-core/src/modules/storage_import_tinyconnectors.rs
+++ b/crates/openhuman-core/src/modules/storage_import_tinyconnectors.rs
@@ -62,10 +62,26 @@ pub(crate) enum Outcome {
/// Imports the legacy files under `state_dir` into the module's scope of the
/// installed backend.
pub(crate) async fn import_legacy(ctx: &DispatchCtx, state_dir: &Path) -> Result<Outcome> {
- let Some(scoped) = support::module_scope(ctx, MODULE_ID)? else {
+ let prefs_dir = state_dir.join(PREFS_DIR);
+ let triggers_dir = state_dir.join(TRIGGERS_DIR);
+ if !prefs_dir.is_dir() && !triggers_dir.is_dir() {
+ return Ok(Outcome::NoSource);
+ }
+ let marker = support::marker_id("connectors", state_dir);
+ let state_dir = state_dir.to_owned();
+ let ctx_for_read = ctx.clone();
+ let result = support::with_module_claim(ctx, MODULE_ID, MARKER_COLLECTION, &marker, move |scoped| {
+ crate::storage::block_on_anyhow(async move {
+ import_into(&ctx_for_read, &scoped, &state_dir).await
+ })
+ })?;
+ let Some(result) = result else {
return Ok(Outcome::NoBackend);
};
- import_into(ctx, &scoped, state_dir).await
+ Ok(match result {
+ support::ClaimedImport::AlreadyDone => Outcome::AlreadyDone,
+ support::ClaimedImport::Imported(outcome) => outcome,
+ })
}
/// A preference file: the toolkit key and its bare JSON value.
@@ -88,10 +104,6 @@ struct Legacy {
async fn import_into(ctx: &DispatchCtx, scoped: &Storage, state_dir: &Path) -> Result<Outcome> {
let docs = scoped.documents();
- let marker = support::marker_id("connectors", state_dir);
- if support::already_imported(scoped, MARKER_COLLECTION, &marker).await? {
- return Ok(Outcome::AlreadyDone);
- }
let prefs_dir = state_dir.join(PREFS_DIR);
let triggers_dir = state_dir.join(TRIGGERS_DIR);
if !prefs_dir.is_dir() && !triggers_dir.is_dir() {
@@ -126,7 +138,6 @@ async fn import_into(ctx: &DispatchCtx, scoped: &Storage, state_dir: &Path) -> R
triggers += support::append_if_empty(scoped, &day.stream, day.records).await?;
}
- support::mark_imported(scoped, MARKER_COLLECTION, &marker).await?;
tracing::info!(
"{LOG_PREFIX} imported prefs={prefs} triggers={triggers} skipped={}",
legacy.skipped
diff --git a/crates/openhuman-core/src/modules/storage_import_tinyjuice.rs b/crates/openhuman-core/src/modules/storage_import_tinyjuice.rs
index 8fa44b1b26..9d6b1fd2a0 100644
--- a/crates/openhuman-core/src/modules/storage_import_tinyjuice.rs
+++ b/crates/openhuman-core/src/modules/storage_import_tinyjuice.rs
@@ -68,7 +68,28 @@ pub(crate) async fn import_legacy(
let Some(scoped) = support::module_scope(ctx, MODULE_ID)? else {
return Ok(Outcome::NoBackend);
};
- import_into(ctx, &scoped, root, ttl, SystemTime::now()).await
+ if !root.is_dir() {
+ return Ok(Outcome::NoSource);
+ }
+ let blobs = scoped.blobs();
+ if ttl.is_some() && !blobs.capabilities().contains(Capability::BlobTtl) {
+ return Ok(Outcome::Unsupported);
+ }
+ let marker = support::marker_id("ccr", root);
+ let root = root.to_owned();
+ let ctx_for_read = ctx.clone();
+ let result = support::with_module_claim(ctx, MODULE_ID, MARKER_COLLECTION, &marker, move |scoped| {
+ crate::storage::block_on_anyhow(async move {
+ import_into(&ctx_for_read, &scoped, &root, ttl, SystemTime::now()).await
+ })
+ })?;
+ let Some(result) = result else {
+ return Ok(Outcome::NoBackend);
+ };
+ Ok(match result {
+ support::ClaimedImport::AlreadyDone => Outcome::AlreadyDone,
+ support::ClaimedImport::Imported(outcome) => outcome,
+ })
}
/// One readable original: its token, text and age.
@@ -85,10 +106,6 @@ async fn import_into(
ttl: Option<Duration>,
now: SystemTime,
) -> Result<Outcome> {
- let marker = support::marker_id("ccr", root);
- if support::already_imported(scoped, MARKER_COLLECTION, &marker).await? {
- return Ok(Outcome::AlreadyDone);
- }
if !root.is_dir() {
return Ok(Outcome::NoSource);
}
@@ -138,7 +155,6 @@ async fn import_into(
imported += 1;
}
- support::mark_imported(scoped, MARKER_COLLECTION, &marker).await?;
tracing::info!("{LOG_PREFIX} imported={imported} expired={expired} skipped={skipped}");
Ok(Outcome::Imported {
imported,