Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(import): introduce condition-based imports for storage operations

Steven Enamakel78e43561a28f
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rs
index 0f35074030..8b9b844fba 100644
--- a/crates/openhuman-core/src/storage/import/sessions.rs
+++ b/crates/openhuman-core/src/storage/import/sessions.rs
@@ -67,11 +67,13 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>
.join("memory")
.join("conversations")
.join("turn_states");
- marked(backend, &workspace, SESSIONS_DOMAIN, || {
+ marked(backend, &workspace, SESSIONS_DOMAIN, |condition| {
if !db.is_file() && !raw.is_dir() && !turns.is_dir() {
return Ok(None);
}
- let provider = DriverSessionStores::new(Arc::clone(backend), Scope::local())
+ let claimed_backend =
+ super::super::import_state::backend_with_condition(Arc::clone(backend), condition.clone());
+ let provider = DriverSessionStores::new(claimed_backend, Scope::local())
.map_err(|error| anyhow::anyhow!("start the session store bridge: {error}"))?
.recover_on_open(false);
let mut found = 0usize;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rs
index 80c16b6f4b..ffc4be463a 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs
@@ -69,7 +69,7 @@ pub(super) fn marked(
backend: &Arc<dyn StorageBackend>,
workspace: &Path,
domain: &str,
- body: impl FnOnce() -> Result<Option<(usize, usize, usize)>>,
+ body: impl FnOnce(&tinystoragedrivers::WriteCondition) -> Result<Option<(usize, usize, usize)>>,
) -> Result<Outcome> {
let step = marker_step(domain, workspace);
let markers = Repo::on(
@@ -81,11 +81,12 @@ pub(super) fn marked(
if import_state::is_done(&markers, MARKER_OWNER, &step)? {
return Ok(Outcome::AlreadyDone);
}
- let claimed = import_state::with_claim(&markers, MARKER_OWNER, &step, || {
- if import_state::is_done(&markers, MARKER_OWNER, &step)? {
+ let claimed = import_state::with_claim(&markers, MARKER_OWNER, &step, |condition| {
+ let claimed_markers = markers.conditioned(condition)?;
+ if import_state::is_done(&claimed_markers, MARKER_OWNER, &step)? {
return Ok(Outcome::AlreadyDone);
}
- let Some((found, written, skipped)) = body()? else {
+ let Some((found, written, skipped)) = body(condition)? else {
return Ok(Outcome::NoSource);
};
if skipped > 0 {
@@ -101,7 +102,7 @@ pub(super) fn marked(
skipped,
});
}
- import_state::set_done(&markers, MARKER_OWNER, &step)?;
+ import_state::set_done(&claimed_markers, MARKER_OWNER, &step)?;
Ok(Outcome::Imported { found, written })
})?;
Ok(claimed.unwrap_or(Outcome::InProgress))
@@ -428,12 +429,12 @@ fn import_flows_after_migration(
};
let dir = config.workspace_dir.join("flows");
- marked(backend, &config.workspace_dir, FLOWS_DOMAIN, || {
+ marked(backend, &config.workspace_dir, FLOWS_DOMAIN, |condition| {
if !flow_sources_present(&dir) {
return Ok(None);
}
let dir = dir.clone();
- let backend = Arc::clone(backend);
+ let backend = import_state::backend_with_condition(Arc::clone(backend), condition.clone());
let scope = scope.clone();
let config = config.clone();
let totals = Arc::new(Mutex::new(ImportTotals::default()));
@@ -530,7 +531,7 @@ pub fn import_cron(
format!("{CRON_DOMAIN}/{label}")
};
let jobs_db = legacy_jobs_db(scope_dir);
- marked(backend, &config.workspace_dir, &domain, || {
+ marked(backend, &config.workspace_dir, &domain, |condition| {
if !jobs_db.is_file() {
return Ok(None);
}
@@ -538,7 +539,8 @@ pub fn import_cron(
// records only) and replayed through the module's `import_job` /
// `import_run` store operations, which write only what is absent and
// file a flow-schedule job under the id its flow derives.
- let doc_scope = DocScope::on_backend(config, Arc::clone(backend), scope.clone());
+ let backend = import_state::backend_with_condition(Arc::clone(backend), condition.clone());
+ let doc_scope = DocScope::on_backend(config, backend, scope.clone());
let config = config.clone();
let report = crate::storage::block_on_anyhow({
let jobs_db = jobs_db.clone();
diff --git a/crates/openhuman-core/src/threads/store/legacy_import.rs b/crates/openhuman-core/src/threads/store/legacy_import.rs
index b9ef725aba..28352d80cc 100644
--- a/crates/openhuman-core/src/threads/store/legacy_import.rs
+++ b/crates/openhuman-core/src/threads/store/legacy_import.rs
@@ -149,7 +149,6 @@ fn fold_threads(root: &Path) -> Result<Vec<CreateConversationThread>> {
pub(super) fn import_once(
workspace_dir: &Path,
repo: &Repo,
- store: &DriverConversationStore,
) -> Result<()> {
let root = super::backend::legacy_root(workspace_dir);
let source = crate::storage::source_key(&root);
@@ -167,8 +166,9 @@ pub(super) fn import_once(
log::debug!("{LOG_PREFIX} already imported");
return Ok(());
}
- let claimed = import_state::with_claim(repo, OWNER, &step, || {
- if import_state::is_done(repo, OWNER, &step)? {
+ let claimed = import_state::with_claim(repo, OWNER, &step, |condition| {
+ let claimed_repo = repo.conditioned(condition)?;
+ if import_state::is_done(&claimed_repo, OWNER, &step)? {
return Ok(());
}
let threads = fold_threads(&root)?;
@@ -178,12 +178,12 @@ pub(super) fn import_once(
&root, &thread.id,
))?);
}
- let store = store.clone();
+ let store = DriverConversationStore::new(claimed_repo.scoped().clone());
let (threads_n, messages_n) =
crate::storage::block_on_anyhow(
async move { copy(&store, threads, messages).await },
)?;
- import_state::set_done(repo, OWNER, &step)?;
+ import_state::set_done(&claimed_repo, OWNER, &step)?;
log::info!("{LOG_PREFIX} imported threads={threads_n} messages={messages_n}");
Ok(())
})?;