The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(import): introduce condition-based imports for storage operations
Steven Enamakel78e43561a28f
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rsindex 0f35074030..8b9b844fba 100644--- a/crates/openhuman-core/src/storage/import/sessions.rs+++ b/crates/openhuman-core/src/storage/import/sessions.rs@@ -67,11 +67,13 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>.join("memory").join("conversations").join("turn_states");- marked(backend, &workspace, SESSIONS_DOMAIN, || {+ marked(backend, &workspace, SESSIONS_DOMAIN, |condition| {if !db.is_file() && !raw.is_dir() && !turns.is_dir() {return Ok(None);}- let provider = DriverSessionStores::new(Arc::clone(backend), Scope::local())+ let claimed_backend =+ super::super::import_state::backend_with_condition(Arc::clone(backend), condition.clone());+ let provider = DriverSessionStores::new(claimed_backend, Scope::local()).map_err(|error| anyhow::anyhow!("start the session store bridge: {error}"))?.recover_on_open(false);let mut found = 0usize;diff --git a/crates/openhuman-core/src/storage/import/tinyflows.rs b/crates/openhuman-core/src/storage/import/tinyflows.rsindex 80c16b6f4b..ffc4be463a 100644--- a/crates/openhuman-core/src/storage/import/tinyflows.rs+++ b/crates/openhuman-core/src/storage/import/tinyflows.rs@@ -69,7 +69,7 @@ pub(super) fn marked(backend: &Arc<dyn StorageBackend>,workspace: &Path,domain: &str,- body: impl FnOnce() -> Result<Option<(usize, usize, usize)>>,+ body: impl FnOnce(&tinystoragedrivers::WriteCondition) -> Result<Option<(usize, usize, usize)>>,) -> Result<Outcome> {let step = marker_step(domain, workspace);let markers = Repo::on(@@ -81,11 +81,12 @@ pub(super) fn marked(if import_state::is_done(&markers, MARKER_OWNER, &step)? {return Ok(Outcome::AlreadyDone);}- let claimed = import_state::with_claim(&markers, MARKER_OWNER, &step, || {- if import_state::is_done(&markers, MARKER_OWNER, &step)? {+ let claimed = import_state::with_claim(&markers, MARKER_OWNER, &step, |condition| {+ let claimed_markers = markers.conditioned(condition)?;+ if import_state::is_done(&claimed_markers, MARKER_OWNER, &step)? {return Ok(Outcome::AlreadyDone);}- let Some((found, written, skipped)) = body()? else {+ let Some((found, written, skipped)) = body(condition)? else {return Ok(Outcome::NoSource);};if skipped > 0 {@@ -101,7 +102,7 @@ pub(super) fn marked(skipped,});}- import_state::set_done(&markers, MARKER_OWNER, &step)?;+ import_state::set_done(&claimed_markers, MARKER_OWNER, &step)?;Ok(Outcome::Imported { found, written })})?;Ok(claimed.unwrap_or(Outcome::InProgress))@@ -428,12 +429,12 @@ fn import_flows_after_migration(};let dir = config.workspace_dir.join("flows");- marked(backend, &config.workspace_dir, FLOWS_DOMAIN, || {+ marked(backend, &config.workspace_dir, FLOWS_DOMAIN, |condition| {if !flow_sources_present(&dir) {return Ok(None);}let dir = dir.clone();- let backend = Arc::clone(backend);+ let backend = import_state::backend_with_condition(Arc::clone(backend), condition.clone());let scope = scope.clone();let config = config.clone();let totals = Arc::new(Mutex::new(ImportTotals::default()));@@ -530,7 +531,7 @@ pub fn import_cron(format!("{CRON_DOMAIN}/{label}")};let jobs_db = legacy_jobs_db(scope_dir);- marked(backend, &config.workspace_dir, &domain, || {+ marked(backend, &config.workspace_dir, &domain, |condition| {if !jobs_db.is_file() {return Ok(None);}@@ -538,7 +539,8 @@ pub fn import_cron(// records only) and replayed through the module's `import_job` /// `import_run` store operations, which write only what is absent and// file a flow-schedule job under the id its flow derives.- let doc_scope = DocScope::on_backend(config, Arc::clone(backend), scope.clone());+ let backend = import_state::backend_with_condition(Arc::clone(backend), condition.clone());+ let doc_scope = DocScope::on_backend(config, backend, scope.clone());let config = config.clone();let report = crate::storage::block_on_anyhow({let jobs_db = jobs_db.clone();diff --git a/crates/openhuman-core/src/threads/store/legacy_import.rs b/crates/openhuman-core/src/threads/store/legacy_import.rsindex b9ef725aba..28352d80cc 100644--- a/crates/openhuman-core/src/threads/store/legacy_import.rs+++ b/crates/openhuman-core/src/threads/store/legacy_import.rs@@ -149,7 +149,6 @@ fn fold_threads(root: &Path) -> Result<Vec<CreateConversationThread>> {pub(super) fn import_once(workspace_dir: &Path,repo: &Repo,- store: &DriverConversationStore,) -> Result<()> {let root = super::backend::legacy_root(workspace_dir);let source = crate::storage::source_key(&root);@@ -167,8 +166,9 @@ pub(super) fn import_once(log::debug!("{LOG_PREFIX} already imported");return Ok(());}- let claimed = import_state::with_claim(repo, OWNER, &step, || {- if import_state::is_done(repo, OWNER, &step)? {+ let claimed = import_state::with_claim(repo, OWNER, &step, |condition| {+ let claimed_repo = repo.conditioned(condition)?;+ if import_state::is_done(&claimed_repo, OWNER, &step)? {return Ok(());}let threads = fold_threads(&root)?;@@ -178,12 +178,12 @@ pub(super) fn import_once(&root, &thread.id,))?);}- let store = store.clone();+ let store = DriverConversationStore::new(claimed_repo.scoped().clone());let (threads_n, messages_n) =crate::storage::block_on_anyhow(async move { copy(&store, threads, messages).await },)?;- import_state::set_done(repo, OWNER, &step)?;+ import_state::set_done(&claimed_repo, OWNER, &step)?;log::info!("{LOG_PREFIX} imported threads={threads_n} messages={messages_n}");Ok(())})?;