Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(storage): add tinyflows import readers

Steven Enamakelbc89e8a46903
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs
new file mode 100644
index 0000000000..bd4ad72e15
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs
@@ -0,0 +1,382 @@
+//! Bounded host-side readers for the retired SQLite and JSONL checkpointers.
+
+use std::future::Future;
+use std::io::{self, BufRead, BufReader};
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use tinyflows_bus::catalog::{
+ LegacyImportKind, LegacyImportRecord, LegacySqliteCheckpointWrite,
+ MAX_IMPORT_RECORD_PAYLOAD_BYTES,
+};
+
+use crate::flow_reader::BatchSender;
+use crate::legacy_db::LegacyDb;
+
+const SQLITE_PAGE_ROWS: i64 = 8;
+
+/// Reads SQLite and file/JSONL checkpoint sources as bounded callback batches.
+///
+/// A torn invalid final JSONL line is ignored as the retired file reader did.
+/// The source paths remain host-side and are never serialized into records.
+pub async fn read_legacy_checkpoint_batches<F, Fut>(
+ sqlite_path: &Path,
+ file_dir: &Path,
+ send: F,
+) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ read_sources(Some(sqlite_path), Some(file_dir), send).await
+}
+
+/// Reads only the retired SQLite checkpoint database as bounded raw batches.
+pub async fn read_legacy_sqlite_checkpoint_batches<F, Fut>(
+ sqlite_path: &Path,
+ send: F,
+) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ read_sources(Some(sqlite_path), None, send).await
+}
+
+/// Reads only the retired file/JSONL checkpoint directory as bounded raw batches.
+pub async fn read_legacy_file_checkpoint_batches<F, Fut>(file_dir: &Path, send: F) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ read_sources(None, Some(file_dir), send).await
+}
+
+async fn read_sources<F, Fut>(
+ sqlite_path: Option<&Path>,
+ file_dir: Option<&Path>,
+ mut send: F,
+) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let mut batches = BatchSender::default();
+ if let Some(sqlite_path) = sqlite_path
+ && let Some(db) = LegacyDb::open(sqlite_path)?
+ {
+ let tables = db.tables().await?;
+ if tables.contains("checkpoints") {
+ let mut offset = 0_i64;
+ loop {
+ let rows = db
+ .read("checkpoints", move |connection| {
+ let mut statement = connection.prepare(
+ "SELECT CASE WHEN length(CAST(record AS BLOB)) <= ?1 THEN CAST(record AS BLOB) END, seq FROM checkpoints ORDER BY seq LIMIT ?2 OFFSET ?3",
+ )?;
+ statement
+ .query_map(
+ [MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64, SQLITE_PAGE_ROWS, offset],
+ |row| {
+ Ok((
+ row.get::<_, Option<Vec<u8>>>(0)?,
+ row.get::<_, i64>(1)?,
+ ))
+ },
+ )?
+ .collect::<std::result::Result<Vec<_>, _>>()
+ })
+ .await?;
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for (raw, sequence) in rows {
+ let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {
+ Some(raw_json) => LegacyImportRecord::Checkpoint { raw_json },
+ None => LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Checkpoint,
+ identity: Some(format!("seq:{sequence}")),
+ message: "checkpoint source record exceeds the import byte limit or is not UTF-8".into(),
+ },
+ };
+ batches.push(record, &mut send).await?;
+ }
+ }
+ }
+ if tables.contains("checkpoint_writes") {
+ let mut offset = 0_i64;
+ loop {
+ let rows = db.read("checkpoint writes", move |connection| {
+ let mut statement = connection.prepare(
+ "WITH source_rows AS (SELECT rowid, thread_id, namespace, checkpoint_id, task_id, idx, node, channel, payload, length(COALESCE(CAST(thread_id AS BLOB), X'')) + length(COALESCE(CAST(namespace AS BLOB), X'')) + length(COALESCE(CAST(checkpoint_id AS BLOB), X'')) + length(COALESCE(CAST(task_id AS BLOB), X'')) + length(COALESCE(CAST(node AS BLOB), X'')) + length(COALESCE(CAST(channel AS BLOB), X'')) + length(COALESCE(CAST(payload AS BLOB), X'')) AS raw_bytes FROM checkpoint_writes) SELECT rowid, raw_bytes <= ?1, CASE WHEN raw_bytes <= ?1 THEN CAST(thread_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(namespace AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(checkpoint_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(task_id AS BLOB) END, idx, CASE WHEN raw_bytes <= ?1 THEN CAST(node AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(channel AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(payload AS BLOB) END FROM source_rows ORDER BY rowid LIMIT ?2 OFFSET ?3",
+ )?;
+ let mapped = statement.query_map(
+ [
+ MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64,
+ SQLITE_PAGE_ROWS,
+ offset,
+ ],
+ |row| {
+ Ok((
+ row.get::<_, i64>(0)?,
+ row.get::<_, bool>(1)?,
+ row.get::<_, Option<Vec<u8>>>(2)?,
+ row.get::<_, Option<Vec<u8>>>(3)?,
+ row.get::<_, Option<Vec<u8>>>(4)?,
+ row.get::<_, Option<Vec<u8>>>(5)?,
+ row.get::<_, i64>(6)?,
+ row.get::<_, Option<Vec<u8>>>(7)?,
+ row.get::<_, Option<Vec<u8>>>(8)?,
+ row.get::<_, Option<Vec<u8>>>(9)?,
+ ))
+ },
+ )?;
+ mapped.collect::<std::result::Result<Vec<_>, _>>()
+ }).await?;
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for (
+ row_id,
+ within_limit,
+ thread,
+ namespace,
+ checkpoint,
+ task,
+ idx,
+ node,
+ channel,
+ payload,
+ ) in rows
+ {
+ let record = if within_limit {
+ let decoded = (|| {
+ Some(LegacySqliteCheckpointWrite {
+ thread_id: String::from_utf8(thread?).ok()?,
+ namespace_json: String::from_utf8(namespace?).ok()?,
+ checkpoint_id: String::from_utf8(checkpoint?).ok()?,
+ task_id: String::from_utf8(task?).ok()?,
+ idx,
+ node: String::from_utf8(node?).ok()?,
+ channel: String::from_utf8(channel?).ok()?,
+ payload_json: String::from_utf8(payload?).ok()?,
+ })
+ })();
+ decoded.map_or_else(
+ || LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Write,
+ identity: Some(format!("row:{row_id}")),
+ message:
+ "checkpoint write row contains a missing or non-UTF-8 field"
+ .into(),
+ },
+ LegacyImportRecord::SqliteCheckpointWrite,
+ )
+ } else {
+ LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Write,
+ identity: Some(format!("row:{row_id}")),
+ message: "checkpoint write row exceeds the import byte limit".into(),
+ }
+ };
+ batches.push(record, &mut send).await?;
+ }
+ }
+ }
+ }
+ if let Some(file_dir) = file_dir {
+ read_file_checkpoints(file_dir, &mut batches, &mut send).await?;
+ }
+ batches.flush(&mut send).await
+}
+
+async fn read_file_checkpoints<F, Fut>(
+ dir: &Path,
+ batches: &mut BatchSender,
+ send: &mut F,
+) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let entries = match std::fs::read_dir(dir) {
+ Ok(entries) => entries,
+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
+ Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())),
+ };
+ let mut files: Vec<PathBuf> = Vec::new();
+ for entry in entries {
+ let path = entry
+ .with_context(|| format!("reading an entry under {}", dir.display()))?
+ .path();
+ if path.is_file() {
+ files.push(path);
+ }
+ }
+ files.sort();
+ for path in files {
+ let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
+ continue;
+ };
+ if let Some(stem) = name.strip_suffix(".writes.jsonl") {
+ let Some(thread_id) = unescape_thread_id(stem) else {
+ batches
+ .push(
+ LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Write,
+ identity: Some(name.chars().take(128).collect()),
+ message: "not a valid checkpoint thread filename".into(),
+ },
+ send,
+ )
+ .await?;
+ continue;
+ };
+ read_jsonl(
+ &path,
+ name,
+ thread_id,
+ LegacyImportKind::Write,
+ batches,
+ send,
+ )
+ .await?;
+ } else if name.ends_with(".jsonl") {
+ let Some(thread_id) = name.strip_suffix(".jsonl").and_then(unescape_thread_id) else {
+ batches
+ .push(
+ LegacyImportRecord::Invalid {
+ record_kind: LegacyImportKind::Checkpoint,
+ identity: Some(name.chars().take(128).collect()),
+ message: "not a valid checkpoint thread filename".into(),
+ },
+ send,
+ )
+ .await?;
+ continue;
+ };
+ read_jsonl(
+ &path,
+ name,
+ thread_id,
+ LegacyImportKind::Checkpoint,
+ batches,
+ send,
+ )
+ .await?;
+ }
+ }
+ Ok(())
+}
+
+async fn read_jsonl<F, Fut>(
+ path: &Path,
+ name: &str,
+ thread_id: String,
+ kind: LegacyImportKind,
+ batches: &mut BatchSender,
+ send: &mut F,
+) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let file = std::fs::File::open(path).with_context(|| format!("opening {}", path.display()))?;
+ let mut reader = BufReader::new(file);
+ while let Some((line, terminated, oversized)) = read_bounded_line(&mut reader)? {
+ if oversized {
+ batches
+ .push(
+ LegacyImportRecord::Invalid {
+ record_kind: kind,
+ identity: Some(thread_id.chars().take(128).collect()),
+ message: "checkpoint JSONL record exceeds the import byte limit".into(),
+ },
+ send,
+ )
+ .await?;
+ continue;
+ }
+ if line.iter().all(u8::is_ascii_whitespace) {
+ continue;
+ }
+ if !terminated && serde_json::from_slice::<serde_json::Value>(&line).is_err() {
+ tracing::warn!(target: "tinyflows_import", file = name, "[import] ignoring a torn trailing checkpoint line");
+ continue;
+ }
+ let Ok(raw_json) = String::from_utf8(line) else {
+ batches
+ .push(
+ LegacyImportRecord::Invalid {
+ record_kind: kind,
+ identity: Some(thread_id.chars().take(128).collect()),
+ message: "checkpoint JSONL record is not UTF-8".into(),
+ },
+ send,
+ )
+ .await?;
+ continue;
+ };
+ let record = match kind {
+ LegacyImportKind::Checkpoint => LegacyImportRecord::Checkpoint { raw_json },
+ LegacyImportKind::Write => LegacyImportRecord::FileCheckpointWrite {
+ thread_id: thread_id.clone(),
+ raw_json,
+ },
+ _ => unreachable!("only checkpoint and write files are read here"),
+ };
+ batches.push(record, send).await?;
+ }
+ Ok(())
+}
+
+fn read_bounded_line<R: BufRead>(reader: &mut R) -> io::Result<Option<(Vec<u8>, bool, bool)>> {
+ let mut line = Vec::new();
+ let mut saw_bytes = false;
+ let mut oversized = false;
+ loop {
+ let available = reader.fill_buf()?;
+ if available.is_empty() {
+ return Ok(saw_bytes.then_some((line, false, oversized)));
+ }
+ saw_bytes = true;
+ let newline = available.iter().position(|byte| *byte == b'\n');
+ let consumed = newline.map_or(available.len(), |index| index + 1);
+ let content_len = newline.map_or(consumed, |index| index);
+ if !oversized {
+ if line.len() + content_len > MAX_IMPORT_RECORD_PAYLOAD_BYTES {
+ oversized = true;
+ line.clear();
+ } else {
+ line.extend_from_slice(&available[..content_len]);
+ }
+ }
+ reader.consume(consumed);
+ if newline.is_some() {
+ return Ok(Some((line, true, oversized)));
+ }
+ }
+}
+
+pub(crate) fn unescape_thread_id(escaped: &str) -> Option<String> {
+ let bytes = escaped.as_bytes();
+ let mut out = Vec::with_capacity(bytes.len());
+ let mut index = 0;
+ while index < bytes.len() {
+ if bytes[index] == b'%' {
+ let hex = escaped.get(index + 1..index + 3)?;
+ out.push(u8::from_str_radix(hex, 16).ok()?);
+ index += 3;
+ } else {
+ out.push(bytes[index]);
+ index += 1;
+ }
+ }
+ String::from_utf8(out).ok()
+}
+
+#[cfg(test)]
+#[path = "checkpoint_reader_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
new file mode 100644
index 0000000000..39633e5b34
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs
@@ -0,0 +1,267 @@
+//! Bounded, host-side readers for the retired flow catalog and draft files.
+
+use std::future::Future;
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{Context, Result};
+use tinyflows_bus::catalog::{
+ LegacyDraftRecord, LegacyFlowRecord, LegacyImportRecord, LegacyRevisionRecord, LegacyRunRecord,
+ LegacyStateRecord, LegacySuggestionRecord, MAX_IMPORT_BATCH_RECORDS,
+ MAX_IMPORT_RECORD_PAYLOAD_BYTES,
+};
+
+use crate::legacy_db::{LegacyDb, column_or};
+
+/// Reads legacy flow database rows and draft files as bounded ordered batches.
+///
+/// The caller owns source discovery, the target storage scope and migration
+/// marker. Only file contents and safe file names cross the callback boundary.
+/// Each callback completes before the next batch is read and sent.
+pub async fn read_legacy_flow_catalog<F, Fut>(dir: &Path, mut send: F) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let mut batches = BatchSender::default();
+ if let Some(db) = LegacyDb::open(&dir.join("flows.db"))? {
+ let tables = db.tables().await?;
+ if tables.contains("flow_definitions") {
+ let columns = db.columns("flow_definitions").await?;
+ let select = format!(
+ "SELECT id,name,graph_json,enabled,created_at,updated_at,last_run_at,last_status,{} FROM flow_definitions ORDER BY created_at,id LIMIT ?1 OFFSET ?2",
+ column_or(&columns, "require_approval", "0")
+ );
+ let mut offset = 0_i64;
+ loop {
+ let query = select.clone();
+ let rows = db
+ .read("flow definitions", move |conn| {
+ let mut statement = conn.prepare(&query)?;
+ let mapped = statement.query_map([64_i64, offset], |row| {
+ Ok(LegacyFlowRecord {
+ id: row.get(0)?,
+ name: row.get(1)?,
+ graph_json: row.get(2)?,
+ enabled: row.get::<_, i64>(3)? != 0,
+ created_at: row.get(4)?,
+ updated_at: row.get(5)?,
+ last_run_at: row.get(6)?,
+ last_status: row.get(7)?,
+ require_approval: row.get::<_, i64>(8)? != 0,
+ })
+ })?;
+ mapped.collect::<std::result::Result<Vec<_>, _>>()
+ })
+ .await?;
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for row in rows {
+ batches
+ .push(LegacyImportRecord::Flow(row), &mut send)
+ .await?;
+ }
+ }
+ }
+ if tables.contains("flow_revisions") {
+ page_rows(&db, "flow revisions", "SELECT id,flow_id,graph_json,name,require_approval,created_at FROM flow_revisions ORDER BY created_at,id LIMIT ?1 OFFSET ?2", |row| {
+ Ok(LegacyRevisionRecord { id: row.get(0)?, flow_id: row.get(1)?, graph_json: row.get(2)?, name: row.get(3)?, require_approval: row.get::<_, i64>(4)? != 0, created_at: row.get(5)? })
+ }, |row| LegacyImportRecord::Revision(row), &mut batches, &mut send).await?;
+ }
+ if tables.contains("flow_runs") {
+ let columns = db.columns("flow_runs").await?;
+ let select = format!(
+ "SELECT id,flow_id,thread_id,status,started_at,finished_at,steps_json,pending_approvals_json,error,{} FROM flow_runs ORDER BY started_at,id LIMIT ?1 OFFSET ?2",
+ column_or(&columns, "graph_hash", "NULL")
+ );
+ page_rows(
+ &db,
+ "flow runs",
+ &select,
+ |row| {
+ Ok(LegacyRunRecord {
+ id: row.get(0)?,
+ flow_id: row.get(1)?,
+ thread_id: row.get(2)?,
+ status: row.get(3)?,
+ started_at: row.get(4)?,
+ finished_at: row.get(5)?,
+ steps_json: row.get(6)?,
+ pending_approvals_json: row.get(7)?,
+ error: row.get(8)?,
+ graph_hash: row.get(9)?,
+ })
+ },
+ |row| LegacyImportRecord::Run(row),
+ &mut batches,
+ &mut send,
+ )
+ .await?;
+ }
+ if tables.contains("flow_state") {
+ page_rows(&db, "flow state", "SELECT namespace,key,value FROM flow_state ORDER BY namespace,key LIMIT ?1 OFFSET ?2", |row| {
+ Ok(LegacyStateRecord { namespace: row.get(0)?, key: row.get(1)?, value_json: row.get(2)? })
+ }, |row| LegacyImportRecord::State(row), &mut batches, &mut send).await?;
+ }
+ if tables.contains("flow_suggestions") {
+ page_rows(&db, "flow suggestions", "SELECT id,title,one_liner,rationale,trigger_hint,steps_json,connections_json,slugs_json,build_prompt,confidence,status,created_at,source_run_id FROM flow_suggestions ORDER BY created_at,id LIMIT ?1 OFFSET ?2", |row| {
+ Ok(LegacySuggestionRecord { id: row.get(0)?, title: row.get(1)?, one_liner: row.get(2)?, rationale: row.get(3)?, trigger_hint: row.get(4)?, steps_json: row.get(5)?, connections_json: row.get(6)?, slugs_json: row.get(7)?, build_prompt: row.get(8)?, confidence: row.get(9)?, status: row.get(10)?, created_at: row.get(11)?, source_run_id: row.get(12)? })
+ }, |row| LegacyImportRecord::Suggestion(row), &mut batches, &mut send).await?;
+ }
+ }
+ read_drafts(&dir.join("drafts"), &mut batches, &mut send).await?;
+ batches.flush(&mut send).await
+}
+
+async fn page_rows<T, ToRecord, F, Fut>(
+ db: &LegacyDb,
+ what: &'static str,
+ query: &str,
+ decode: impl Fn(
+ &tinystoragedrivers_sqlite::rusqlite::Row<'_>,
+ ) -> tinystoragedrivers_sqlite::rusqlite::Result<T>
+ + Send
+ + Sync
+ + 'static,
+ to_record: ToRecord,
+ batches: &mut BatchSender,
+ send: &mut F,
+) -> Result<()>
+where
+ T: Send + 'static,
+ ToRecord: Fn(T) -> LegacyImportRecord,
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let query = query.to_string();
+ let decode = Arc::new(decode);
+ let mut offset = 0_i64;
+ loop {
+ let statement_query = query.clone();
+ let decode = Arc::clone(&decode);
+ let rows = db
+ .read(what, move |conn| {
+ let mut statement = conn.prepare(&statement_query)?;
+ let mapped = statement.query_map([64_i64, offset], |row| decode(row))?;
+ mapped.collect::<std::result::Result<Vec<_>, _>>()
+ })
+ .await?;
+ if rows.is_empty() {
+ break;
+ }
+ offset += rows.len() as i64;
+ for row in rows {
+ batches.push(to_record(row), send).await?;
+ }
+ }
+ Ok(())
+}
+
+async fn read_drafts<F, Fut>(dir: &Path, batches: &mut BatchSender, send: &mut F) -> Result<()>
+where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let entries = match std::fs::read_dir(dir) {
+ Ok(entries) => entries,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+ Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())),
+ };
+ let mut paths: Vec<_> = entries
+ .filter_map(|entry| entry.ok().map(|entry| entry.path()))
+ .filter(|path| {
+ path.is_file()
+ && path.extension().is_some_and(|ext| ext == "json")
+ && !path
+ .file_name()
+ .is_some_and(|name| name.to_string_lossy().starts_with('.'))
+ })
+ .collect();
+ paths.sort();
+ for path in paths {
+ let bytes = std::fs::read(&path).with_context(|| format!("reading {}", path.display()))?;
+ let file_name = path
+ .file_name()
+ .map(|name| name.to_string_lossy().into_owned())
+ .unwrap_or_default();
+ batches
+ .push(
+ LegacyImportRecord::Draft(LegacyDraftRecord { file_name, bytes }),
+ send,
+ )
+ .await?;
+ }
+ Ok(())
+}
+
+#[derive(Default)]
+pub(crate) struct BatchSender {
+ current: Vec<LegacyImportRecord>,
+ bytes: usize,
+}
+
+impl BatchSender {
+ pub(crate) async fn push<F, Fut>(
+ &mut self,
+ mut record: LegacyImportRecord,
+ send: &mut F,
+ ) -> Result<()>
+ where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+ {
+ let mut size = serde_json::to_vec(&record)?.len();
+ if size > MAX_IMPORT_RECORD_PAYLOAD_BYTES {
+ let kind = record.record_kind();
+ let identity = record_identity(&record);
+ record = LegacyImportRecord::Invalid {
+ record_kind: kind,
+ identity,
+ message: "record exceeds the import byte limit".into(),
+ };
+ size = serde_json::to_vec(&record)?.len();
+ }
+ if !self.current.is_empty()
+ && (self.current.len() >= MAX_IMPORT_BATCH_RECORDS
+ || self.bytes + size > MAX_IMPORT_RECORD_PAYLOAD_BYTES)
+ {
+ self.flush(send).await?;
+ }
+ self.bytes += size;
+ self.current.push(record);
+ Ok(())
+ }
+
+ pub(crate) async fn flush<F, Fut>(&mut self, send: &mut F) -> Result<()>
+ where
+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,
+ Fut: Future<Output = Result<()>>,
+ {
+ if self.current.is_empty() {
+ return Ok(());
+ }
+ self.bytes = 0;
+ send(std::mem::take(&mut self.current)).await
+ }
+}
+
+fn record_identity(record: &LegacyImportRecord) -> Option<String> {
+ let value = match record {
+ LegacyImportRecord::Flow(row) => Some(row.id.clone()),
+ LegacyImportRecord::Revision(row) => Some(row.id.clone()),
+ LegacyImportRecord::Run(row) => Some(row.id.clone()),
+ LegacyImportRecord::State(row) => Some(row.key.clone()),
+ LegacyImportRecord::Suggestion(row) => Some(row.id.clone()),
+ LegacyImportRecord::Draft(row) => Some(row.file_name.clone()),
+ LegacyImportRecord::FileCheckpointWrite { thread_id, .. } => Some(thread_id.clone()),
+ LegacyImportRecord::SqliteCheckpointWrite(row) => Some(row.task_id.clone()),
+ LegacyImportRecord::Checkpoint { .. } | LegacyImportRecord::Invalid { .. } => None,
+ };
+ value.map(|identity| identity.chars().take(128).collect())
+}
+
+#[cfg(test)]
+#[path = "flow_reader_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
new file mode 100644
index 0000000000..40458eaa3a
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs
@@ -0,0 +1,291 @@
+//! Reads the legacy `jobs.db` (the cron store before it moved to documents) into
+//! plain bus records. It names only the transport vocabulary
+//! (`tinyflows_bus::schedule::types`), so a host that reaches the scheduler
+//! through the bus can replay the file through the module's
+//! `import_job` / `import_run` store operations without linking the scheduler
+//! implementation.
+//!
+//! Two tables, `cron_jobs` and `cron_runs`. `cron_jobs` gained most of its
+//! columns over time, so a database from before a column existed reads as if
+//! the column held its default. An older `cron_job_policies` side table, which
+//! the old store folded into `cron_jobs.retries` / `single_flight` on its next
+//! open, is read the same way here.
+
+use std::collections::HashSet;
+use std::path::Path;
+
+use anyhow::{Context, Result, anyhow};
+use chrono::{DateTime, Utc};
+use std::collections::HashMap;
+use tinyflows_bus::schedule::types::{
+ CronJob, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobPolicy, JobType, Schedule,
+ SessionTarget,
+};
+
+use crate::legacy_db::{LegacyDb, column_or};
+use crate::report::ImportReport;
+
+/// What `jobs.db` held, decoded.
+#[derive(Debug, Default)]
+pub struct LegacyCron {
+ /// Every decodable job, oldest first.
+ pub jobs: Vec<CronJob>,
+ /// Every decodable run, in row order. Its `job_id` is the job's id in the
+ /// legacy file; a flow-schedule job is filed under another id by the
+ /// target store, so a caller remaps it from the id the import returns.
+ pub runs: Vec<CronRun>,
+ /// Rows that could not be decoded; also counted in the report as invalid.
+ pub report: ImportReport,
+}
+
+/// Reads `jobs_db` (the file the old store was opened with). A missing file or
+/// one without a `cron_jobs` table reads as empty.
+///
+/// # Errors
+///
+/// When the file exists but cannot be read. A row that cannot be decoded is
+/// counted in the report as invalid.
+pub async fn read_legacy_cron(jobs_db: &Path) -> Result<LegacyCron> {
+ let mut legacy = LegacyCron::default();
+ let Some(db) = LegacyDb::open(jobs_db)? else {
+ return Ok(legacy);
+ };
+ let tables = db.tables().await?;
+ if !tables.contains("cron_jobs") {
+ return Ok(legacy);
+ }
+ legacy.jobs = read_jobs(&db, &tables, &mut legacy.report).await?;
+ if tables.contains("cron_runs") {
+ legacy.runs = read_runs(&db, &mut legacy.report).await?;
+ }
+ Ok(legacy)
+}
+
+/// One `cron_jobs` row with the raw text columns still raw.
+struct JobRow {
+ id: String,
+ expression: String,
+ command: String,
+ schedule: Option<String>,
+ job_type: String,
+ prompt: Option<String>,
+ name: Option<String>,
+ session_target: String,
+ model: Option<String>,
+ enabled: i64,
+ delivery: Option<String>,
+ delete_after_run: i64,
+ created_at: String,
+ next_run: String,
+ last_run: Option<String>,
+ last_status: Option<String>,
+ last_output: Option<String>,
+ agent_id: Option<String>,
+ origin: Option<String>,
+ retries: Option<i64>,
+ single_flight: i64,
+ claim_generation: i64,
+}
+
+async fn read_jobs(
+ db: &LegacyDb,
+ tables: &HashSet<String>,
+ report: &mut ImportReport,
+) -> Result<Vec<CronJob>> {
+ let have = db.columns("cron_jobs").await?;
+ let select = format!(
+ "SELECT id, expression, command, {}, {}, {}, {}, {}, {}, {}, {}, {}, created_at, next_run, \
+ last_run, last_status, last_output, {}, {}, {}, {}, {} FROM cron_jobs ORDER BY created_at, id",
+ column_or(&have, "schedule", "NULL"),
+ column_or(&have, "job_type", "'shell'"),
+ column_or(&have, "prompt", "NULL"),
+ column_or(&have, "name", "NULL"),
+ column_or(&have, "session_target", "'isolated'"),
+ column_or(&have, "model", "NULL"),
+ column_or(&have, "enabled", "1"),
+ column_or(&have, "delivery", "NULL"),
+ column_or(&have, "delete_after_run", "0"),
+ column_or(&have, "agent_id", "NULL"),
+ column_or(&have, "origin", "NULL"),
+ column_or(&have, "retries", "NULL"),
+ column_or(&have, "single_flight", "0"),
+ column_or(&have, "claim_generation", "0"),
+ );
+ let mut rows = db
+ .read("cron jobs", move |conn| {
+ let mut statement = conn.prepare(&select)?;
+ statement
+ .query_map([], |row| {
+ Ok(JobRow {
+ id: row.get(0)?,
+ expression: row.get(1)?,
+ command: row.get(2)?,
+ schedule: row.get(3)?,
+ job_type: row.get(4)?,
+ prompt: row.get(5)?,
+ name: row.get(6)?,
+ session_target: row.get(7)?,
+ model: row.get(8)?,
+ enabled: row.get(9)?,
+ delivery: row.get(10)?,
+ delete_after_run: row.get(11)?,
+ created_at: row.get(12)?,
+ next_run: row.get(13)?,
+ last_run: row.get(14)?,
+ last_status: row.get(15)?,
+ last_output: row.get(16)?,
+ agent_id: row.get(17)?,
+ origin: row.get(18)?,
+ retries: row.get(19)?,
+ single_flight: row.get(20)?,
+ claim_generation: row.get(21)?,
+ })
+ })?
+ .collect::<Result<Vec<_>, _>>()
+ })
+ .await?;
+ if tables.contains("cron_job_policies") {
+ let policies = read_policies(db).await?;
+ for row in &mut rows {
+ if let Some((retries, single_flight)) = policies.get(&row.id) {
+ row.retries = *retries;
+ row.single_flight = *single_flight;
+ }
+ }
+ }
+ let mut jobs = Vec::new();
+ for row in rows {
+ let id = row.id.clone();
+ match decode_job(row) {
+ Ok(job) => jobs.push(job),
+ Err(error) => report.invalid("jobs", format!("job '{id}': {error:#}")),
+ }
+ }
+ Ok(jobs)
+}
+
+/// `cron_job_policies`: `job_id -> (retries, single_flight)`.
+async fn read_policies(db: &LegacyDb) -> Result<HashMap<String, (Option<i64>, i64)>> {
+ db.read("cron job policies", |conn| {
+ let mut statement =
+ conn.prepare("SELECT job_id, retries, single_flight FROM cron_job_policies")?;
+ statement
+ .query_map([], |row| {
+ Ok((row.get::<_, String>(0)?, (row.get(1)?, row.get(2)?)))
+ })?
+ .collect::<Result<HashMap<_, _>, _>>()
+ })
+ .await
+}
+
+fn decode_job(row: JobRow) -> Result<CronJob> {
+ let schedule = match row.schedule.as_deref().map(str::trim) {
+ Some(raw) if !raw.is_empty() => serde_json::from_str(raw)
+ .with_context(|| format!("cron schedule JSON does not parse: {raw}"))?,
+ _ if row.expression.trim().is_empty() => {
+ return Err(anyhow!("missing schedule and legacy expression"));
+ }
+ // A job written before `schedule` existed carried only its expression.
+ _ => Schedule::Cron {
+ expr: row.expression.clone(),
+ tz: None,
+ active_hours: None,
+ },
+ };
+ let delivery = match row.delivery.as_deref().map(str::trim) {
+ Some(raw) if !raw.is_empty() => serde_json::from_str(raw)
+ .with_context(|| format!("cron delivery JSON does not parse: {raw}"))?,
+ _ => DeliveryConfig::default(),
+ };
+ let origin = match row.origin.as_deref().map(str::trim) {
+ Some(raw) if !raw.is_empty() => Some(
+ serde_json::from_str::<JobOrigin>(raw)
+ .with_context(|| format!("cron origin JSON does not parse: {raw}"))?,
+ ),
+ _ => None,
+ };
+ Ok(CronJob {
+ claim_generation: u64::try_from(row.claim_generation)
+ .context("claim generation is negative")?,
+ policy: JobPolicy {
+ retries: row
+ .retries
+ .map(|n| u32::try_from(n.max(0)).unwrap_or(u32::MAX)),
+ single_flight: row.single_flight != 0,
+ },
+ id: row.id,
+ expression: row.expression,
+ schedule,
+ command: row.command,
+ job_type: JobType::parse(&row.job_type),
+ prompt: row.prompt,
+ name: row.name,
+ session_target: SessionTarget::parse(&row.session_target),
+ model: row.model,
+ agent_id: row.agent_id,
+ enabled: row.enabled != 0,
+ delivery,
+ delete_after_run: row.delete_after_run != 0,
+ created_at: instant(&row.created_at)?,
+ next_run: instant(&row.next_run)?,
+ last_run: row.last_run.as_deref().map(instant).transpose()?,
+ last_status: row.last_status,
+ last_output: row.last_output,
+ origin,
+ })
+}
+
+fn instant(raw: &str) -> Result<DateTime<Utc>> {
+ Ok(DateTime::parse_from_rfc3339(raw)
+ .with_context(|| format!("invalid RFC3339 timestamp: {raw}"))?
+ .with_timezone(&Utc))
+}
+
+async fn read_runs(db: &LegacyDb, report: &mut ImportReport) -> Result<Vec<CronRun>> {
+ let have = db.columns("cron_runs").await?;
+ let select = format!(
+ "SELECT id, job_id, started_at, finished_at, status, output, duration_ms, {} \
+ FROM cron_runs ORDER BY id",
+ column_or(&have, "delivery_status", "NULL"),
+ );
+ let rows = db
+ .read("cron runs", move |conn| {
+ let mut statement = conn.prepare(&select)?;
+ statement
+ .query_map([], |row| {
+ Ok((
+ row.get::<_, i64>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, String>(2)?,
+ row.get::<_, String>(3)?,
+ row.get::<_, String>(4)?,
+ row.get::<_, Option<String>>(5)?,
+ row.get::<_, Option<i64>>(6)?,
+ row.get::<_, Option<String>>(7)?,
+ ))
+ })?
+ .collect::<Result<Vec<_>, _>>()
+ })
+ .await?;
+ let mut runs = Vec::new();
+ for (id, job_id, started_at, finished_at, status, output, duration_ms, delivery_status) in rows
+ {
+ match (instant(&started_at), instant(&finished_at)) {
+ (Ok(started_at), Ok(finished_at)) => runs.push(CronRun {
+ id,
+ job_id,
+ run_id: None,
+ started_at,
+ finished_at,
+ status,
+ output,
+ duration_ms,
+ delivery_status: delivery_status.as_deref().and_then(DeliveryStatus::parse),
+ }),
+ (Err(error), _) | (_, Err(error)) => {
+ report.invalid("runs", format!("run {id}: {error:#}"));
+ }
+ }
+ }
+ Ok(runs)
+}
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs
new file mode 100644
index 0000000000..9cc9f6160f
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs
@@ -0,0 +1,105 @@
+//! Reading a legacy SQLite file without touching it.
+//!
+//! The file is copied, with its `-wal` and `-shm` if it has them, to a scratch
+//! directory, and the copy is opened through the SQLite driver's native handle.
+//! The driver creates what it needs and sets WAL mode on open, so opening the
+//! original would modify it; the copy takes that instead and is deleted when
+//! the [`LegacyDb`] drops.
+
+use std::collections::HashSet;
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use tinystoragedrivers_sqlite::SqliteNative;
+use tinystoragedrivers_sqlite::rusqlite::Connection;
+
+/// A scratch copy of a legacy SQLite database, open for reading.
+pub(crate) struct LegacyDb {
+ // Held for its drop: the directory goes when the handle does.
+ _scratch: tempfile::TempDir,
+ native: SqliteNative,
+}
+
+impl LegacyDb {
+ /// Opens a copy of the database at `path`; `None` when there is no such
+ /// file, which is the normal state of a store that was never used.
+ pub(crate) fn open(path: &Path) -> Result<Option<Self>> {
+ if !path.is_file() {
+ return Ok(None);
+ }
+ let scratch = tempfile::tempdir().context("creating a scratch directory")?;
+ let name = path
+ .file_name()
+ .context("a database path has a file name")?
+ .to_os_string();
+ for suffix in ["", "-wal", "-shm"] {
+ let mut from = path.as_os_str().to_os_string();
+ from.push(suffix);
+ let from = std::path::PathBuf::from(from);
+ if from.is_file() {
+ let mut to = name.clone();
+ to.push(suffix);
+ std::fs::copy(&from, scratch.path().join(to))
+ .with_context(|| format!("copying {}", from.display()))?;
+ }
+ }
+ let native = SqliteNative::open(scratch.path().join(&name))
+ .with_context(|| format!("opening a copy of {}", path.display()))?;
+ Ok(Some(Self {
+ _scratch: scratch,
+ native,
+ }))
+ }
+
+ /// Runs `read` on the copy's connection.
+ pub(crate) async fn read<T, F>(&self, what: &'static str, read: F) -> Result<T>
+ where
+ T: Send + 'static,
+ F: FnOnce(&Connection) -> tinystoragedrivers_sqlite::rusqlite::Result<T> + Send + 'static,
+ {
+ self.native
+ .with_connection(read)
+ .await
+ .with_context(|| format!("reading {what} from the legacy database"))
+ }
+
+ /// The tables the database has.
+ pub(crate) async fn tables(&self) -> Result<HashSet<String>> {
+ self.read("the table list", |conn| {
+ let mut statement =
+ conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?;
+ let names = statement
+ .query_map([], |row| row.get::<_, String>(0))?
+ .collect::<Result<HashSet<_>, _>>()?;
+ Ok(names)
+ })
+ .await
+ }
+
+ /// The columns `table` has; empty when it has none or does not exist.
+ pub(crate) async fn columns(&self, table: &'static str) -> Result<HashSet<String>> {
+ self.read("a column list", move |conn| {
+ let mut statement = conn.prepare(&format!("PRAGMA table_info({table})"))?;
+ let names = statement
+ .query_map([], |row| row.get::<_, String>(1))?
+ .collect::<Result<HashSet<_>, _>>()?;
+ Ok(names)
+ })
+ .await
+ }
+}
+
+/// `column` when `have` lists it, else the literal `default`, as a select-list
+/// entry: how a database written before a column existed is read as if the
+/// column held its default.
+pub(crate) fn column_or(have: &HashSet<String>, column: &str, default: &str) -> String {
+ if have.contains(column) {
+ column.to_string()
+ } else {
+ format!("{default} AS {column}")
+ }
+}
+
+#[cfg(test)]
+#[path = "legacy_db_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rs
new file mode 100644
index 0000000000..46ed8fbcaa
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rs
@@ -0,0 +1,134 @@
+//! What an import did, record kind by record kind.
+
+use std::collections::BTreeMap;
+
+#[cfg(all(
+ feature = "flow-read",
+ any(feature = "flow-write", feature = "checkpoint-write")
+))]
+fn legacy_kind_name(kind: tinyflows_bus::catalog::LegacyImportKind) -> &'static str {
+ use tinyflows_bus::catalog::LegacyImportKind as Kind;
+ match kind {
+ Kind::Flow => "flows",
+ Kind::Revision => "revisions",
+ Kind::Run => "runs",
+ Kind::State => "state",
+ Kind::Suggestion => "suggestions",
+ Kind::Draft => "drafts",
+ Kind::Checkpoint => "checkpoints",
+ Kind::Write => "writes",
+ }
+}
+
+/// How many records of one kind an import wrote, found already present, or
+/// could not decode.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct Tally {
+ /// Records this import wrote.
+ pub written: usize,
+ /// Records the target already held, left exactly as they were.
+ pub existing: usize,
+ /// Records in the legacy store that could not be decoded or were refused.
+ pub invalid: usize,
+}
+
+/// The outcome of one `import_*` call.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct ImportReport {
+ kinds: BTreeMap<&'static str, Tally>,
+ problems: Vec<String>,
+}
+
+impl ImportReport {
+ #[cfg(all(
+ feature = "flow-read",
+ any(feature = "flow-write", feature = "checkpoint-write")
+ ))]
+ pub(crate) fn merge_legacy_batch(
+ &mut self,
+ response: tinyflows_bus::catalog::ImportLegacyBatchResponse,
+ ) {
+ for tally in response.tallies {
+ let kind = legacy_kind_name(tally.kind);
+ let total = self.kinds.entry(kind).or_default();
+ total.written += tally.written as usize;
+ total.existing += tally.existing as usize;
+ total.invalid += tally.invalid as usize;
+ }
+ for problem in response.problems {
+ let identity = problem.identity.map_or_else(String::new, |id| {
+ let label = match problem.kind {
+ tinyflows_bus::catalog::LegacyImportKind::Flow => "flow",
+ tinyflows_bus::catalog::LegacyImportKind::Revision => "revision",
+ tinyflows_bus::catalog::LegacyImportKind::Run => "run",
+ tinyflows_bus::catalog::LegacyImportKind::State => "state",
+ tinyflows_bus::catalog::LegacyImportKind::Suggestion => "suggestion",
+ tinyflows_bus::catalog::LegacyImportKind::Draft => "draft",
+ tinyflows_bus::catalog::LegacyImportKind::Checkpoint => "checkpoint",
+ tinyflows_bus::catalog::LegacyImportKind::Write => "write",
+ };
+ format!(" {label} '{id}'")
+ });
+ let message = format!(
+ "{}{}: {}",
+ legacy_kind_name(problem.kind),
+ identity,
+ problem.message
+ );
+ tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}");
+ self.problems.push(message);
+ }
+ }
+
+ pub fn wrote(&mut self, kind: &'static str, written: bool) {
+ let tally = self.kinds.entry(kind).or_default();
+ if written {
+ tally.written += 1;
+ } else {
+ tally.existing += 1;
+ }
+ }
+
+ /// Records one record that could not be imported, with why.
+ pub fn invalid(&mut self, kind: &'static str, what: impl std::fmt::Display) {
+ self.kinds.entry(kind).or_default().invalid += 1;
+ let message = format!("{kind}: {what}");
+ tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}");
+ self.problems.push(message);
+ }
+
+ /// The tally of `kind` (`"flows"`, `"runs"`, `"jobs"`, ...); all zero when
+ /// the store held none.
+ pub fn tally(&self, kind: &str) -> Tally {
+ self.kinds.get(kind).copied().unwrap_or_default()
+ }
+
+ /// How many records of `kind` this import wrote.
+ pub fn written(&self, kind: &str) -> usize {
+ self.tally(kind).written
+ }
+
+ /// Every kind seen, with its tally, in name order.
+ pub fn kinds(&self) -> impl Iterator<Item = (&'static str, Tally)> + '_ {
+ self.kinds.iter().map(|(kind, tally)| (*kind, *tally))
+ }
+
+ /// One line per record that could not be imported.
+ pub fn problems(&self) -> &[String] {
+ &self.problems
+ }
+
+ /// Records written across every kind.
+ pub fn total_written(&self) -> usize {
+ self.kinds.values().map(|tally| tally.written).sum()
+ }
+
+ /// Whether every legacy record was either written or already present.
+ pub fn is_clean(&self) -> bool {
+ self.problems.is_empty()
+ }
+}
+
+#[cfg(test)]
+#[path = "report_tests.rs"]
+mod tests;