The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): split legacy session import into staged batches
Steven Enamakele02513a84a2a
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rsindex 4b903bb2d3..697afb3f99 100644--- a/crates/openhuman-core/src/storage/import/sessions.rs+++ b/crates/openhuman-core/src/storage/import/sessions.rs@@ -33,8 +33,9 @@ use std::sync::Arc;use anyhow::{Context, Result};use tinyagents_session::port::{SessionStoreProvider, TurnStates};+use tinyagents_session::run_ledger::history::SessionHistoryImport;use tinyagents_session::run_ledger::RunLedgerStore;-use tinyagents_session::run_ledger::{ImportCounts, LegacyDatabaseReport, LegacySessionImport};+use tinyagents_session::run_ledger::{AgentFamilyImport, ImportCounts, LegacyLedgerRows};use tinyagents_session::transcript::{session_stem, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory, TranscriptMessage,TranscriptMeta, TranscriptPartial, TranscriptTurn,@@ -93,33 +94,74 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>if db.is_file() {let snapshot = Snapshot::of(&db, SESSIONS_DB).context("[sessions] copy sessions.db for reading")?;- let ledger = provider- .driver_run_ledger(DEFAULT_AGENT)- .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;let mut rows = super::session_db::SessionDatabase::read(snapshot.db()).context("[sessions] read the session database snapshot")?;sanitize_legacy_system_messages(&mut rows.messages);- let batch = LegacySessionImport {+ let fingerprint = source_fingerprint(snapshot.db())?;+ let history_batch = SessionHistoryImport {sessions: rows.sessions,messages: rows.messages,tool_calls: rows.tool_calls,- agent_runs: rows.agent_runs,- telemetry: rows.telemetry,+ };+ let ledger = provider+ .driver_run_ledger(DEFAULT_AGENT)+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;+ let history = crate::storage::block_on_anyhow(async move {+ ledger+ .import_session_history(history_batch)+ .await+ .map_err(|error| anyhow::anyhow!("{error}"))+ })+ .context("[sessions] import classic session history")?;++ let agent_batch = AgentFamilyImport {+ operation_id: workspace_operation_id(&workspace),+ source_fingerprint: fingerprint,+ runs: rows.agent_runs,events: rows.events,+ telemetry: rows.telemetry,+ };+ let ledger = provider+ .driver_run_ledger(DEFAULT_AGENT)+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;+ let agent = crate::storage::block_on_anyhow(async move {+ ledger+ .import_agent_family(agent_batch)+ .await+ .map_err(|error| anyhow::anyhow!("{error}"))+ })+ .context("[sessions] import classic agent run family")?;++ let legacy_batch = LegacyLedgerRows {workflow_runs: rows.workflows,teams: rows.teams,members: rows.members,tasks: rows.tasks,tool_effects: rows.tool_effects,};- let report = crate::storage::block_on_anyhow(async move {+ let ledger = provider+ .driver_run_ledger(DEFAULT_AGENT)+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;+ let legacy = crate::storage::block_on_anyhow(async move {ledger- .import_legacy_batch(batch)+ .import_legacy_rows(legacy_batch).await.map_err(|error| anyhow::anyhow!("{error}"))})- .context("[sessions] import sessions.db")?;- let (database_found, database_written) = database_totals(&report);+ .context("[sessions] import classic ledger rows")?;+ let (database_found, database_written) = database_totals([+ history.sessions,+ history.messages,+ history.tool_calls,+ agent.agent_runs,+ agent.events,+ agent.telemetry,+ legacy.workflow_runs,+ legacy.teams,+ legacy.members,+ legacy.tasks,+ legacy.tool_effects,+ ]);found += database_found;written += database_written;}@@ -147,27 +189,31 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>/// The native importer reports source rows per stored family. FTS rows are a/// derived index and do not represent additional legacy records.-fn database_totals(report: &LegacyDatabaseReport) -> (usize, usize) {- [- report.sessions,- report.messages,- report.tool_calls,- report.agent_runs,- report.telemetry,- report.events,- report.workflow_runs,- report.teams,- report.members,- report.tasks,- report.tool_effects,- ]- .into_iter()- .fold(+fn database_totals(counts: [ImportCounts; 11]) -> (usize, usize) {+ counts.into_iter().fold((0, 0),|(found, written), ImportCounts { read, imported, .. }| (found + read, written + imported),)}+fn source_fingerprint(path: &Path) -> Result<String> {+ use sha2::{Digest, Sha256};+ let bytes =+ std::fs::read(path).with_context(|| format!("read source snapshot {}", path.display()))?;+ Ok(format!("sha256:{}", hex::encode(Sha256::digest(bytes))))+}++fn workspace_operation_id(workspace: &Path) -> String {+ use sha2::{Digest, Sha256};+ let identity = workspace+ .canonicalize()+ .unwrap_or_else(|_| workspace.to_path_buf());+ format!(+ "workspace-sessions-v1:{}",+ hex::encode(Sha256::digest(identity.to_string_lossy().as_bytes()))+ )+}+fn sanitize_legacy_system_messages(messages: &mut [tinyagents_session::types::SessionMessage]) {for message in messages {if message.role == "system" {diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rsindex da333bb637..82a41f503f 100644--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs@@ -386,21 +386,10 @@ fn database_import_totals_source_families_not_derived_fts_rows() {imported,already_present: read - imported,};- let report = LegacyDatabaseReport {- sessions: counts(1, 1),- messages: counts(2, 1),- tool_calls: counts(3, 2),- agent_runs: counts(4, 3),- telemetry: counts(5, 4),- events: counts(6, 5),- workflow_runs: counts(7, 6),- teams: counts(8, 7),- members: counts(9, 8),- tasks: counts(10, 9),- tool_effects: counts(11, 10),- entries: ImportCounts::default(),- fts_rows: 1_000,- };-- assert_eq!(database_totals(&report), (66, 56));+ let counts = [+ counts(1, 1), counts(2, 1), counts(3, 2), counts(4, 3), counts(5, 4),+ counts(6, 5), counts(7, 6), counts(8, 7), counts(9, 8), counts(10, 9),+ counts(11, 10),+ ];+ assert_eq!(database_totals(counts), (66, 56));}diff --git a/vendor/tinyagents b/vendor/tinyagentsindex 2790939b74..b1478a60fd 160000--- a/vendor/tinyagents+++ b/vendor/tinyagents@@ -1 +1 @@-Subproject commit 2790939b74d44fc2eb9d6c67d54630eec7e4b42a+Subproject commit b1478a60fd8bff8f511a17997d978c51c86ab417