Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): split legacy session import into staged batches

Steven Enamakele02513a84a2a
diff --git a/crates/openhuman-core/src/storage/import/sessions.rs b/crates/openhuman-core/src/storage/import/sessions.rs
index 4b903bb2d3..697afb3f99 100644
--- a/crates/openhuman-core/src/storage/import/sessions.rs
+++ b/crates/openhuman-core/src/storage/import/sessions.rs
@@ -33,8 +33,9 @@ use std::sync::Arc;
use anyhow::{Context, Result};
use tinyagents_session::port::{SessionStoreProvider, TurnStates};
+use tinyagents_session::run_ledger::history::SessionHistoryImport;
use tinyagents_session::run_ledger::RunLedgerStore;
-use tinyagents_session::run_ledger::{ImportCounts, LegacyDatabaseReport, LegacySessionImport};
+use tinyagents_session::run_ledger::{AgentFamilyImport, ImportCounts, LegacyLedgerRows};
use tinyagents_session::transcript::{
session_stem, DisplayMessage, DisplayRecord, SessionRef, TranscriptHistory, TranscriptMessage,
TranscriptMeta, TranscriptPartial, TranscriptTurn,
@@ -93,33 +94,74 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>
if db.is_file() {
let snapshot = Snapshot::of(&db, SESSIONS_DB)
.context("[sessions] copy sessions.db for reading")?;
- let ledger = provider
- .driver_run_ledger(DEFAULT_AGENT)
- .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;
let mut rows = super::session_db::SessionDatabase::read(snapshot.db())
.context("[sessions] read the session database snapshot")?;
sanitize_legacy_system_messages(&mut rows.messages);
- let batch = LegacySessionImport {
+ let fingerprint = source_fingerprint(snapshot.db())?;
+ let history_batch = SessionHistoryImport {
sessions: rows.sessions,
messages: rows.messages,
tool_calls: rows.tool_calls,
- agent_runs: rows.agent_runs,
- telemetry: rows.telemetry,
+ };
+ let ledger = provider
+ .driver_run_ledger(DEFAULT_AGENT)
+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;
+ let history = crate::storage::block_on_anyhow(async move {
+ ledger
+ .import_session_history(history_batch)
+ .await
+ .map_err(|error| anyhow::anyhow!("{error}"))
+ })
+ .context("[sessions] import classic session history")?;
+
+ let agent_batch = AgentFamilyImport {
+ operation_id: workspace_operation_id(&workspace),
+ source_fingerprint: fingerprint,
+ runs: rows.agent_runs,
events: rows.events,
+ telemetry: rows.telemetry,
+ };
+ let ledger = provider
+ .driver_run_ledger(DEFAULT_AGENT)
+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;
+ let agent = crate::storage::block_on_anyhow(async move {
+ ledger
+ .import_agent_family(agent_batch)
+ .await
+ .map_err(|error| anyhow::anyhow!("{error}"))
+ })
+ .context("[sessions] import classic agent run family")?;
+
+ let legacy_batch = LegacyLedgerRows {
workflow_runs: rows.workflows,
teams: rows.teams,
members: rows.members,
tasks: rows.tasks,
tool_effects: rows.tool_effects,
};
- let report = crate::storage::block_on_anyhow(async move {
+ let ledger = provider
+ .driver_run_ledger(DEFAULT_AGENT)
+ .map_err(|error| anyhow::anyhow!("open the run ledger: {error}"))?;
+ let legacy = crate::storage::block_on_anyhow(async move {
ledger
- .import_legacy_batch(batch)
+ .import_legacy_rows(legacy_batch)
.await
.map_err(|error| anyhow::anyhow!("{error}"))
})
- .context("[sessions] import sessions.db")?;
- let (database_found, database_written) = database_totals(&report);
+ .context("[sessions] import classic ledger rows")?;
+ let (database_found, database_written) = database_totals([
+ history.sessions,
+ history.messages,
+ history.tool_calls,
+ agent.agent_runs,
+ agent.events,
+ agent.telemetry,
+ legacy.workflow_runs,
+ legacy.teams,
+ legacy.members,
+ legacy.tasks,
+ legacy.tool_effects,
+ ]);
found += database_found;
written += database_written;
}
@@ -147,27 +189,31 @@ fn import(backend: &Arc<dyn StorageBackend>, config: &Config) -> Result<Outcome>
/// The native importer reports source rows per stored family. FTS rows are a
/// derived index and do not represent additional legacy records.
-fn database_totals(report: &LegacyDatabaseReport) -> (usize, usize) {
- [
- report.sessions,
- report.messages,
- report.tool_calls,
- report.agent_runs,
- report.telemetry,
- report.events,
- report.workflow_runs,
- report.teams,
- report.members,
- report.tasks,
- report.tool_effects,
- ]
- .into_iter()
- .fold(
+fn database_totals(counts: [ImportCounts; 11]) -> (usize, usize) {
+ counts.into_iter().fold(
(0, 0),
|(found, written), ImportCounts { read, imported, .. }| (found + read, written + imported),
)
}
+fn source_fingerprint(path: &Path) -> Result<String> {
+ use sha2::{Digest, Sha256};
+ let bytes =
+ std::fs::read(path).with_context(|| format!("read source snapshot {}", path.display()))?;
+ Ok(format!("sha256:{}", hex::encode(Sha256::digest(bytes))))
+}
+
+fn workspace_operation_id(workspace: &Path) -> String {
+ use sha2::{Digest, Sha256};
+ let identity = workspace
+ .canonicalize()
+ .unwrap_or_else(|_| workspace.to_path_buf());
+ format!(
+ "workspace-sessions-v1:{}",
+ hex::encode(Sha256::digest(identity.to_string_lossy().as_bytes()))
+ )
+}
+
fn sanitize_legacy_system_messages(messages: &mut [tinyagents_session::types::SessionMessage]) {
for message in messages {
if message.role == "system" {
diff --git a/crates/openhuman-core/src/storage/import/sessions_tests.rs b/crates/openhuman-core/src/storage/import/sessions_tests.rs
index da333bb637..82a41f503f 100644
--- a/crates/openhuman-core/src/storage/import/sessions_tests.rs
+++ b/crates/openhuman-core/src/storage/import/sessions_tests.rs
@@ -386,21 +386,10 @@ fn database_import_totals_source_families_not_derived_fts_rows() {
imported,
already_present: read - imported,
};
- let report = LegacyDatabaseReport {
- sessions: counts(1, 1),
- messages: counts(2, 1),
- tool_calls: counts(3, 2),
- agent_runs: counts(4, 3),
- telemetry: counts(5, 4),
- events: counts(6, 5),
- workflow_runs: counts(7, 6),
- teams: counts(8, 7),
- members: counts(9, 8),
- tasks: counts(10, 9),
- tool_effects: counts(11, 10),
- entries: ImportCounts::default(),
- fts_rows: 1_000,
- };
-
- assert_eq!(database_totals(&report), (66, 56));
+ let counts = [
+ counts(1, 1), counts(2, 1), counts(3, 2), counts(4, 3), counts(5, 4),
+ counts(6, 5), counts(7, 6), counts(8, 7), counts(9, 8), counts(10, 9),
+ counts(11, 10),
+ ];
+ assert_eq!(database_totals(counts), (66, 56));
}
diff --git a/vendor/tinyagents b/vendor/tinyagents
index 2790939b74..b1478a60fd 160000
--- a/vendor/tinyagents
+++ b/vendor/tinyagents
@@ -1 +1 @@
-Subproject commit 2790939b74d44fc2eb9d6c67d54630eec7e4b42a
+Subproject commit b1478a60fd8bff8f511a17997d978c51c86ab417