The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(storage): add tinyflows import readers
Steven Enamakelbc89e8a46903
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rsnew file mode 100644index 0000000000..bd4ad72e15--- /dev/null+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/checkpoint_reader.rs@@ -0,0 +1,382 @@+//! Bounded host-side readers for the retired SQLite and JSONL checkpointers.++use std::future::Future;+use std::io::{self, BufRead, BufReader};+use std::path::{Path, PathBuf};++use anyhow::{Context, Result};+use tinyflows_bus::catalog::{+ LegacyImportKind, LegacyImportRecord, LegacySqliteCheckpointWrite,+ MAX_IMPORT_RECORD_PAYLOAD_BYTES,+};++use crate::flow_reader::BatchSender;+use crate::legacy_db::LegacyDb;++const SQLITE_PAGE_ROWS: i64 = 8;++/// Reads SQLite and file/JSONL checkpoint sources as bounded callback batches.+///+/// A torn invalid final JSONL line is ignored as the retired file reader did.+/// The source paths remain host-side and are never serialized into records.+pub async fn read_legacy_checkpoint_batches<F, Fut>(+ sqlite_path: &Path,+ file_dir: &Path,+ send: F,+) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ read_sources(Some(sqlite_path), Some(file_dir), send).await+}++/// Reads only the retired SQLite checkpoint database as bounded raw batches.+pub async fn read_legacy_sqlite_checkpoint_batches<F, Fut>(+ sqlite_path: &Path,+ send: F,+) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ read_sources(Some(sqlite_path), None, send).await+}++/// Reads only the retired file/JSONL checkpoint directory as bounded raw batches.+pub async fn read_legacy_file_checkpoint_batches<F, Fut>(file_dir: &Path, send: F) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ read_sources(None, Some(file_dir), send).await+}++async fn read_sources<F, Fut>(+ sqlite_path: Option<&Path>,+ file_dir: Option<&Path>,+ mut send: F,+) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let mut batches = BatchSender::default();+ if let Some(sqlite_path) = sqlite_path+ && let Some(db) = LegacyDb::open(sqlite_path)?+ {+ let tables = db.tables().await?;+ if tables.contains("checkpoints") {+ let mut offset = 0_i64;+ loop {+ let rows = db+ .read("checkpoints", move |connection| {+ let mut statement = connection.prepare(+ "SELECT CASE WHEN length(CAST(record AS BLOB)) <= ?1 THEN CAST(record AS BLOB) END, seq FROM checkpoints ORDER BY seq LIMIT ?2 OFFSET ?3",+ )?;+ statement+ .query_map(+ [MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64, SQLITE_PAGE_ROWS, offset],+ |row| {+ Ok((+ row.get::<_, Option<Vec<u8>>>(0)?,+ row.get::<_, i64>(1)?,+ ))+ },+ )?+ .collect::<std::result::Result<Vec<_>, _>>()+ })+ .await?;+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for (raw, sequence) in rows {+ let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) {+ Some(raw_json) => LegacyImportRecord::Checkpoint { raw_json },+ None => LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Checkpoint,+ identity: Some(format!("seq:{sequence}")),+ message: "checkpoint source record exceeds the import byte limit or is not UTF-8".into(),+ },+ };+ batches.push(record, &mut send).await?;+ }+ }+ }+ if tables.contains("checkpoint_writes") {+ let mut offset = 0_i64;+ loop {+ let rows = db.read("checkpoint writes", move |connection| {+ let mut statement = connection.prepare(+ "WITH source_rows AS (SELECT rowid, thread_id, namespace, checkpoint_id, task_id, idx, node, channel, payload, length(COALESCE(CAST(thread_id AS BLOB), X'')) + length(COALESCE(CAST(namespace AS BLOB), X'')) + length(COALESCE(CAST(checkpoint_id AS BLOB), X'')) + length(COALESCE(CAST(task_id AS BLOB), X'')) + length(COALESCE(CAST(node AS BLOB), X'')) + length(COALESCE(CAST(channel AS BLOB), X'')) + length(COALESCE(CAST(payload AS BLOB), X'')) AS raw_bytes FROM checkpoint_writes) SELECT rowid, raw_bytes <= ?1, CASE WHEN raw_bytes <= ?1 THEN CAST(thread_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(namespace AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(checkpoint_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(task_id AS BLOB) END, idx, CASE WHEN raw_bytes <= ?1 THEN CAST(node AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(channel AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(payload AS BLOB) END FROM source_rows ORDER BY rowid LIMIT ?2 OFFSET ?3",+ )?;+ let mapped = statement.query_map(+ [+ MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64,+ SQLITE_PAGE_ROWS,+ offset,+ ],+ |row| {+ Ok((+ row.get::<_, i64>(0)?,+ row.get::<_, bool>(1)?,+ row.get::<_, Option<Vec<u8>>>(2)?,+ row.get::<_, Option<Vec<u8>>>(3)?,+ row.get::<_, Option<Vec<u8>>>(4)?,+ row.get::<_, Option<Vec<u8>>>(5)?,+ row.get::<_, i64>(6)?,+ row.get::<_, Option<Vec<u8>>>(7)?,+ row.get::<_, Option<Vec<u8>>>(8)?,+ row.get::<_, Option<Vec<u8>>>(9)?,+ ))+ },+ )?;+ mapped.collect::<std::result::Result<Vec<_>, _>>()+ }).await?;+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for (+ row_id,+ within_limit,+ thread,+ namespace,+ checkpoint,+ task,+ idx,+ node,+ channel,+ payload,+ ) in rows+ {+ let record = if within_limit {+ let decoded = (|| {+ Some(LegacySqliteCheckpointWrite {+ thread_id: String::from_utf8(thread?).ok()?,+ namespace_json: String::from_utf8(namespace?).ok()?,+ checkpoint_id: String::from_utf8(checkpoint?).ok()?,+ task_id: String::from_utf8(task?).ok()?,+ idx,+ node: String::from_utf8(node?).ok()?,+ channel: String::from_utf8(channel?).ok()?,+ payload_json: String::from_utf8(payload?).ok()?,+ })+ })();+ decoded.map_or_else(+ || LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Write,+ identity: Some(format!("row:{row_id}")),+ message:+ "checkpoint write row contains a missing or non-UTF-8 field"+ .into(),+ },+ LegacyImportRecord::SqliteCheckpointWrite,+ )+ } else {+ LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Write,+ identity: Some(format!("row:{row_id}")),+ message: "checkpoint write row exceeds the import byte limit".into(),+ }+ };+ batches.push(record, &mut send).await?;+ }+ }+ }+ }+ if let Some(file_dir) = file_dir {+ read_file_checkpoints(file_dir, &mut batches, &mut send).await?;+ }+ batches.flush(&mut send).await+}++async fn read_file_checkpoints<F, Fut>(+ dir: &Path,+ batches: &mut BatchSender,+ send: &mut F,+) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let entries = match std::fs::read_dir(dir) {+ Ok(entries) => entries,+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),+ Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())),+ };+ let mut files: Vec<PathBuf> = Vec::new();+ for entry in entries {+ let path = entry+ .with_context(|| format!("reading an entry under {}", dir.display()))?+ .path();+ if path.is_file() {+ files.push(path);+ }+ }+ files.sort();+ for path in files {+ let Some(name) = path.file_name().and_then(|name| name.to_str()) else {+ continue;+ };+ if let Some(stem) = name.strip_suffix(".writes.jsonl") {+ let Some(thread_id) = unescape_thread_id(stem) else {+ batches+ .push(+ LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Write,+ identity: Some(name.chars().take(128).collect()),+ message: "not a valid checkpoint thread filename".into(),+ },+ send,+ )+ .await?;+ continue;+ };+ read_jsonl(+ &path,+ name,+ thread_id,+ LegacyImportKind::Write,+ batches,+ send,+ )+ .await?;+ } else if name.ends_with(".jsonl") {+ let Some(thread_id) = name.strip_suffix(".jsonl").and_then(unescape_thread_id) else {+ batches+ .push(+ LegacyImportRecord::Invalid {+ record_kind: LegacyImportKind::Checkpoint,+ identity: Some(name.chars().take(128).collect()),+ message: "not a valid checkpoint thread filename".into(),+ },+ send,+ )+ .await?;+ continue;+ };+ read_jsonl(+ &path,+ name,+ thread_id,+ LegacyImportKind::Checkpoint,+ batches,+ send,+ )+ .await?;+ }+ }+ Ok(())+}++async fn read_jsonl<F, Fut>(+ path: &Path,+ name: &str,+ thread_id: String,+ kind: LegacyImportKind,+ batches: &mut BatchSender,+ send: &mut F,+) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let file = std::fs::File::open(path).with_context(|| format!("opening {}", path.display()))?;+ let mut reader = BufReader::new(file);+ while let Some((line, terminated, oversized)) = read_bounded_line(&mut reader)? {+ if oversized {+ batches+ .push(+ LegacyImportRecord::Invalid {+ record_kind: kind,+ identity: Some(thread_id.chars().take(128).collect()),+ message: "checkpoint JSONL record exceeds the import byte limit".into(),+ },+ send,+ )+ .await?;+ continue;+ }+ if line.iter().all(u8::is_ascii_whitespace) {+ continue;+ }+ if !terminated && serde_json::from_slice::<serde_json::Value>(&line).is_err() {+ tracing::warn!(target: "tinyflows_import", file = name, "[import] ignoring a torn trailing checkpoint line");+ continue;+ }+ let Ok(raw_json) = String::from_utf8(line) else {+ batches+ .push(+ LegacyImportRecord::Invalid {+ record_kind: kind,+ identity: Some(thread_id.chars().take(128).collect()),+ message: "checkpoint JSONL record is not UTF-8".into(),+ },+ send,+ )+ .await?;+ continue;+ };+ let record = match kind {+ LegacyImportKind::Checkpoint => LegacyImportRecord::Checkpoint { raw_json },+ LegacyImportKind::Write => LegacyImportRecord::FileCheckpointWrite {+ thread_id: thread_id.clone(),+ raw_json,+ },+ _ => unreachable!("only checkpoint and write files are read here"),+ };+ batches.push(record, send).await?;+ }+ Ok(())+}++fn read_bounded_line<R: BufRead>(reader: &mut R) -> io::Result<Option<(Vec<u8>, bool, bool)>> {+ let mut line = Vec::new();+ let mut saw_bytes = false;+ let mut oversized = false;+ loop {+ let available = reader.fill_buf()?;+ if available.is_empty() {+ return Ok(saw_bytes.then_some((line, false, oversized)));+ }+ saw_bytes = true;+ let newline = available.iter().position(|byte| *byte == b'\n');+ let consumed = newline.map_or(available.len(), |index| index + 1);+ let content_len = newline.map_or(consumed, |index| index);+ if !oversized {+ if line.len() + content_len > MAX_IMPORT_RECORD_PAYLOAD_BYTES {+ oversized = true;+ line.clear();+ } else {+ line.extend_from_slice(&available[..content_len]);+ }+ }+ reader.consume(consumed);+ if newline.is_some() {+ return Ok(Some((line, true, oversized)));+ }+ }+}++pub(crate) fn unescape_thread_id(escaped: &str) -> Option<String> {+ let bytes = escaped.as_bytes();+ let mut out = Vec::with_capacity(bytes.len());+ let mut index = 0;+ while index < bytes.len() {+ if bytes[index] == b'%' {+ let hex = escaped.get(index + 1..index + 3)?;+ out.push(u8::from_str_radix(hex, 16).ok()?);+ index += 3;+ } else {+ out.push(bytes[index]);+ index += 1;+ }+ }+ String::from_utf8(out).ok()+}++#[cfg(test)]+#[path = "checkpoint_reader_tests.rs"]+mod tests;diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rsnew file mode 100644index 0000000000..39633e5b34--- /dev/null+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/flow_reader.rs@@ -0,0 +1,267 @@+//! Bounded, host-side readers for the retired flow catalog and draft files.++use std::future::Future;+use std::path::Path;+use std::sync::Arc;++use anyhow::{Context, Result};+use tinyflows_bus::catalog::{+ LegacyDraftRecord, LegacyFlowRecord, LegacyImportRecord, LegacyRevisionRecord, LegacyRunRecord,+ LegacyStateRecord, LegacySuggestionRecord, MAX_IMPORT_BATCH_RECORDS,+ MAX_IMPORT_RECORD_PAYLOAD_BYTES,+};++use crate::legacy_db::{LegacyDb, column_or};++/// Reads legacy flow database rows and draft files as bounded ordered batches.+///+/// The caller owns source discovery, the target storage scope and migration+/// marker. Only file contents and safe file names cross the callback boundary.+/// Each callback completes before the next batch is read and sent.+pub async fn read_legacy_flow_catalog<F, Fut>(dir: &Path, mut send: F) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let mut batches = BatchSender::default();+ if let Some(db) = LegacyDb::open(&dir.join("flows.db"))? {+ let tables = db.tables().await?;+ if tables.contains("flow_definitions") {+ let columns = db.columns("flow_definitions").await?;+ let select = format!(+ "SELECT id,name,graph_json,enabled,created_at,updated_at,last_run_at,last_status,{} FROM flow_definitions ORDER BY created_at,id LIMIT ?1 OFFSET ?2",+ column_or(&columns, "require_approval", "0")+ );+ let mut offset = 0_i64;+ loop {+ let query = select.clone();+ let rows = db+ .read("flow definitions", move |conn| {+ let mut statement = conn.prepare(&query)?;+ let mapped = statement.query_map([64_i64, offset], |row| {+ Ok(LegacyFlowRecord {+ id: row.get(0)?,+ name: row.get(1)?,+ graph_json: row.get(2)?,+ enabled: row.get::<_, i64>(3)? != 0,+ created_at: row.get(4)?,+ updated_at: row.get(5)?,+ last_run_at: row.get(6)?,+ last_status: row.get(7)?,+ require_approval: row.get::<_, i64>(8)? != 0,+ })+ })?;+ mapped.collect::<std::result::Result<Vec<_>, _>>()+ })+ .await?;+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for row in rows {+ batches+ .push(LegacyImportRecord::Flow(row), &mut send)+ .await?;+ }+ }+ }+ if tables.contains("flow_revisions") {+ page_rows(&db, "flow revisions", "SELECT id,flow_id,graph_json,name,require_approval,created_at FROM flow_revisions ORDER BY created_at,id LIMIT ?1 OFFSET ?2", |row| {+ Ok(LegacyRevisionRecord { id: row.get(0)?, flow_id: row.get(1)?, graph_json: row.get(2)?, name: row.get(3)?, require_approval: row.get::<_, i64>(4)? != 0, created_at: row.get(5)? })+ }, |row| LegacyImportRecord::Revision(row), &mut batches, &mut send).await?;+ }+ if tables.contains("flow_runs") {+ let columns = db.columns("flow_runs").await?;+ let select = format!(+ "SELECT id,flow_id,thread_id,status,started_at,finished_at,steps_json,pending_approvals_json,error,{} FROM flow_runs ORDER BY started_at,id LIMIT ?1 OFFSET ?2",+ column_or(&columns, "graph_hash", "NULL")+ );+ page_rows(+ &db,+ "flow runs",+ &select,+ |row| {+ Ok(LegacyRunRecord {+ id: row.get(0)?,+ flow_id: row.get(1)?,+ thread_id: row.get(2)?,+ status: row.get(3)?,+ started_at: row.get(4)?,+ finished_at: row.get(5)?,+ steps_json: row.get(6)?,+ pending_approvals_json: row.get(7)?,+ error: row.get(8)?,+ graph_hash: row.get(9)?,+ })+ },+ |row| LegacyImportRecord::Run(row),+ &mut batches,+ &mut send,+ )+ .await?;+ }+ if tables.contains("flow_state") {+ page_rows(&db, "flow state", "SELECT namespace,key,value FROM flow_state ORDER BY namespace,key LIMIT ?1 OFFSET ?2", |row| {+ Ok(LegacyStateRecord { namespace: row.get(0)?, key: row.get(1)?, value_json: row.get(2)? })+ }, |row| LegacyImportRecord::State(row), &mut batches, &mut send).await?;+ }+ if tables.contains("flow_suggestions") {+ page_rows(&db, "flow suggestions", "SELECT id,title,one_liner,rationale,trigger_hint,steps_json,connections_json,slugs_json,build_prompt,confidence,status,created_at,source_run_id FROM flow_suggestions ORDER BY created_at,id LIMIT ?1 OFFSET ?2", |row| {+ Ok(LegacySuggestionRecord { id: row.get(0)?, title: row.get(1)?, one_liner: row.get(2)?, rationale: row.get(3)?, trigger_hint: row.get(4)?, steps_json: row.get(5)?, connections_json: row.get(6)?, slugs_json: row.get(7)?, build_prompt: row.get(8)?, confidence: row.get(9)?, status: row.get(10)?, created_at: row.get(11)?, source_run_id: row.get(12)? })+ }, |row| LegacyImportRecord::Suggestion(row), &mut batches, &mut send).await?;+ }+ }+ read_drafts(&dir.join("drafts"), &mut batches, &mut send).await?;+ batches.flush(&mut send).await+}++async fn page_rows<T, ToRecord, F, Fut>(+ db: &LegacyDb,+ what: &'static str,+ query: &str,+ decode: impl Fn(+ &tinystoragedrivers_sqlite::rusqlite::Row<'_>,+ ) -> tinystoragedrivers_sqlite::rusqlite::Result<T>+ + Send+ + Sync+ + 'static,+ to_record: ToRecord,+ batches: &mut BatchSender,+ send: &mut F,+) -> Result<()>+where+ T: Send + 'static,+ ToRecord: Fn(T) -> LegacyImportRecord,+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let query = query.to_string();+ let decode = Arc::new(decode);+ let mut offset = 0_i64;+ loop {+ let statement_query = query.clone();+ let decode = Arc::clone(&decode);+ let rows = db+ .read(what, move |conn| {+ let mut statement = conn.prepare(&statement_query)?;+ let mapped = statement.query_map([64_i64, offset], |row| decode(row))?;+ mapped.collect::<std::result::Result<Vec<_>, _>>()+ })+ .await?;+ if rows.is_empty() {+ break;+ }+ offset += rows.len() as i64;+ for row in rows {+ batches.push(to_record(row), send).await?;+ }+ }+ Ok(())+}++async fn read_drafts<F, Fut>(dir: &Path, batches: &mut BatchSender, send: &mut F) -> Result<()>+where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let entries = match std::fs::read_dir(dir) {+ Ok(entries) => entries,+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),+ Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())),+ };+ let mut paths: Vec<_> = entries+ .filter_map(|entry| entry.ok().map(|entry| entry.path()))+ .filter(|path| {+ path.is_file()+ && path.extension().is_some_and(|ext| ext == "json")+ && !path+ .file_name()+ .is_some_and(|name| name.to_string_lossy().starts_with('.'))+ })+ .collect();+ paths.sort();+ for path in paths {+ let bytes = std::fs::read(&path).with_context(|| format!("reading {}", path.display()))?;+ let file_name = path+ .file_name()+ .map(|name| name.to_string_lossy().into_owned())+ .unwrap_or_default();+ batches+ .push(+ LegacyImportRecord::Draft(LegacyDraftRecord { file_name, bytes }),+ send,+ )+ .await?;+ }+ Ok(())+}++#[derive(Default)]+pub(crate) struct BatchSender {+ current: Vec<LegacyImportRecord>,+ bytes: usize,+}++impl BatchSender {+ pub(crate) async fn push<F, Fut>(+ &mut self,+ mut record: LegacyImportRecord,+ send: &mut F,+ ) -> Result<()>+ where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+ {+ let mut size = serde_json::to_vec(&record)?.len();+ if size > MAX_IMPORT_RECORD_PAYLOAD_BYTES {+ let kind = record.record_kind();+ let identity = record_identity(&record);+ record = LegacyImportRecord::Invalid {+ record_kind: kind,+ identity,+ message: "record exceeds the import byte limit".into(),+ };+ size = serde_json::to_vec(&record)?.len();+ }+ if !self.current.is_empty()+ && (self.current.len() >= MAX_IMPORT_BATCH_RECORDS+ || self.bytes + size > MAX_IMPORT_RECORD_PAYLOAD_BYTES)+ {+ self.flush(send).await?;+ }+ self.bytes += size;+ self.current.push(record);+ Ok(())+ }++ pub(crate) async fn flush<F, Fut>(&mut self, send: &mut F) -> Result<()>+ where+ F: FnMut(Vec<LegacyImportRecord>) -> Fut,+ Fut: Future<Output = Result<()>>,+ {+ if self.current.is_empty() {+ return Ok(());+ }+ self.bytes = 0;+ send(std::mem::take(&mut self.current)).await+ }+}++fn record_identity(record: &LegacyImportRecord) -> Option<String> {+ let value = match record {+ LegacyImportRecord::Flow(row) => Some(row.id.clone()),+ LegacyImportRecord::Revision(row) => Some(row.id.clone()),+ LegacyImportRecord::Run(row) => Some(row.id.clone()),+ LegacyImportRecord::State(row) => Some(row.key.clone()),+ LegacyImportRecord::Suggestion(row) => Some(row.id.clone()),+ LegacyImportRecord::Draft(row) => Some(row.file_name.clone()),+ LegacyImportRecord::FileCheckpointWrite { thread_id, .. } => Some(thread_id.clone()),+ LegacyImportRecord::SqliteCheckpointWrite(row) => Some(row.task_id.clone()),+ LegacyImportRecord::Checkpoint { .. } | LegacyImportRecord::Invalid { .. } => None,+ };+ value.map(|identity| identity.chars().take(128).collect())+}++#[cfg(test)]+#[path = "flow_reader_tests.rs"]+mod tests;diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rsnew file mode 100644index 0000000000..40458eaa3a--- /dev/null+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_cron.rs@@ -0,0 +1,291 @@+//! Reads the legacy `jobs.db` (the cron store before it moved to documents) into+//! plain bus records. It names only the transport vocabulary+//! (`tinyflows_bus::schedule::types`), so a host that reaches the scheduler+//! through the bus can replay the file through the module's+//! `import_job` / `import_run` store operations without linking the scheduler+//! implementation.+//!+//! Two tables, `cron_jobs` and `cron_runs`. `cron_jobs` gained most of its+//! columns over time, so a database from before a column existed reads as if+//! the column held its default. An older `cron_job_policies` side table, which+//! the old store folded into `cron_jobs.retries` / `single_flight` on its next+//! open, is read the same way here.++use std::collections::HashSet;+use std::path::Path;++use anyhow::{Context, Result, anyhow};+use chrono::{DateTime, Utc};+use std::collections::HashMap;+use tinyflows_bus::schedule::types::{+ CronJob, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobPolicy, JobType, Schedule,+ SessionTarget,+};++use crate::legacy_db::{LegacyDb, column_or};+use crate::report::ImportReport;++/// What `jobs.db` held, decoded.+#[derive(Debug, Default)]+pub struct LegacyCron {+ /// Every decodable job, oldest first.+ pub jobs: Vec<CronJob>,+ /// Every decodable run, in row order. Its `job_id` is the job's id in the+ /// legacy file; a flow-schedule job is filed under another id by the+ /// target store, so a caller remaps it from the id the import returns.+ pub runs: Vec<CronRun>,+ /// Rows that could not be decoded; also counted in the report as invalid.+ pub report: ImportReport,+}++/// Reads `jobs_db` (the file the old store was opened with). A missing file or+/// one without a `cron_jobs` table reads as empty.+///+/// # Errors+///+/// When the file exists but cannot be read. A row that cannot be decoded is+/// counted in the report as invalid.+pub async fn read_legacy_cron(jobs_db: &Path) -> Result<LegacyCron> {+ let mut legacy = LegacyCron::default();+ let Some(db) = LegacyDb::open(jobs_db)? else {+ return Ok(legacy);+ };+ let tables = db.tables().await?;+ if !tables.contains("cron_jobs") {+ return Ok(legacy);+ }+ legacy.jobs = read_jobs(&db, &tables, &mut legacy.report).await?;+ if tables.contains("cron_runs") {+ legacy.runs = read_runs(&db, &mut legacy.report).await?;+ }+ Ok(legacy)+}++/// One `cron_jobs` row with the raw text columns still raw.+struct JobRow {+ id: String,+ expression: String,+ command: String,+ schedule: Option<String>,+ job_type: String,+ prompt: Option<String>,+ name: Option<String>,+ session_target: String,+ model: Option<String>,+ enabled: i64,+ delivery: Option<String>,+ delete_after_run: i64,+ created_at: String,+ next_run: String,+ last_run: Option<String>,+ last_status: Option<String>,+ last_output: Option<String>,+ agent_id: Option<String>,+ origin: Option<String>,+ retries: Option<i64>,+ single_flight: i64,+ claim_generation: i64,+}++async fn read_jobs(+ db: &LegacyDb,+ tables: &HashSet<String>,+ report: &mut ImportReport,+) -> Result<Vec<CronJob>> {+ let have = db.columns("cron_jobs").await?;+ let select = format!(+ "SELECT id, expression, command, {}, {}, {}, {}, {}, {}, {}, {}, {}, created_at, next_run, \+ last_run, last_status, last_output, {}, {}, {}, {}, {} FROM cron_jobs ORDER BY created_at, id",+ column_or(&have, "schedule", "NULL"),+ column_or(&have, "job_type", "'shell'"),+ column_or(&have, "prompt", "NULL"),+ column_or(&have, "name", "NULL"),+ column_or(&have, "session_target", "'isolated'"),+ column_or(&have, "model", "NULL"),+ column_or(&have, "enabled", "1"),+ column_or(&have, "delivery", "NULL"),+ column_or(&have, "delete_after_run", "0"),+ column_or(&have, "agent_id", "NULL"),+ column_or(&have, "origin", "NULL"),+ column_or(&have, "retries", "NULL"),+ column_or(&have, "single_flight", "0"),+ column_or(&have, "claim_generation", "0"),+ );+ let mut rows = db+ .read("cron jobs", move |conn| {+ let mut statement = conn.prepare(&select)?;+ statement+ .query_map([], |row| {+ Ok(JobRow {+ id: row.get(0)?,+ expression: row.get(1)?,+ command: row.get(2)?,+ schedule: row.get(3)?,+ job_type: row.get(4)?,+ prompt: row.get(5)?,+ name: row.get(6)?,+ session_target: row.get(7)?,+ model: row.get(8)?,+ enabled: row.get(9)?,+ delivery: row.get(10)?,+ delete_after_run: row.get(11)?,+ created_at: row.get(12)?,+ next_run: row.get(13)?,+ last_run: row.get(14)?,+ last_status: row.get(15)?,+ last_output: row.get(16)?,+ agent_id: row.get(17)?,+ origin: row.get(18)?,+ retries: row.get(19)?,+ single_flight: row.get(20)?,+ claim_generation: row.get(21)?,+ })+ })?+ .collect::<Result<Vec<_>, _>>()+ })+ .await?;+ if tables.contains("cron_job_policies") {+ let policies = read_policies(db).await?;+ for row in &mut rows {+ if let Some((retries, single_flight)) = policies.get(&row.id) {+ row.retries = *retries;+ row.single_flight = *single_flight;+ }+ }+ }+ let mut jobs = Vec::new();+ for row in rows {+ let id = row.id.clone();+ match decode_job(row) {+ Ok(job) => jobs.push(job),+ Err(error) => report.invalid("jobs", format!("job '{id}': {error:#}")),+ }+ }+ Ok(jobs)+}++/// `cron_job_policies`: `job_id -> (retries, single_flight)`.+async fn read_policies(db: &LegacyDb) -> Result<HashMap<String, (Option<i64>, i64)>> {+ db.read("cron job policies", |conn| {+ let mut statement =+ conn.prepare("SELECT job_id, retries, single_flight FROM cron_job_policies")?;+ statement+ .query_map([], |row| {+ Ok((row.get::<_, String>(0)?, (row.get(1)?, row.get(2)?)))+ })?+ .collect::<Result<HashMap<_, _>, _>>()+ })+ .await+}++fn decode_job(row: JobRow) -> Result<CronJob> {+ let schedule = match row.schedule.as_deref().map(str::trim) {+ Some(raw) if !raw.is_empty() => serde_json::from_str(raw)+ .with_context(|| format!("cron schedule JSON does not parse: {raw}"))?,+ _ if row.expression.trim().is_empty() => {+ return Err(anyhow!("missing schedule and legacy expression"));+ }+ // A job written before `schedule` existed carried only its expression.+ _ => Schedule::Cron {+ expr: row.expression.clone(),+ tz: None,+ active_hours: None,+ },+ };+ let delivery = match row.delivery.as_deref().map(str::trim) {+ Some(raw) if !raw.is_empty() => serde_json::from_str(raw)+ .with_context(|| format!("cron delivery JSON does not parse: {raw}"))?,+ _ => DeliveryConfig::default(),+ };+ let origin = match row.origin.as_deref().map(str::trim) {+ Some(raw) if !raw.is_empty() => Some(+ serde_json::from_str::<JobOrigin>(raw)+ .with_context(|| format!("cron origin JSON does not parse: {raw}"))?,+ ),+ _ => None,+ };+ Ok(CronJob {+ claim_generation: u64::try_from(row.claim_generation)+ .context("claim generation is negative")?,+ policy: JobPolicy {+ retries: row+ .retries+ .map(|n| u32::try_from(n.max(0)).unwrap_or(u32::MAX)),+ single_flight: row.single_flight != 0,+ },+ id: row.id,+ expression: row.expression,+ schedule,+ command: row.command,+ job_type: JobType::parse(&row.job_type),+ prompt: row.prompt,+ name: row.name,+ session_target: SessionTarget::parse(&row.session_target),+ model: row.model,+ agent_id: row.agent_id,+ enabled: row.enabled != 0,+ delivery,+ delete_after_run: row.delete_after_run != 0,+ created_at: instant(&row.created_at)?,+ next_run: instant(&row.next_run)?,+ last_run: row.last_run.as_deref().map(instant).transpose()?,+ last_status: row.last_status,+ last_output: row.last_output,+ origin,+ })+}++fn instant(raw: &str) -> Result<DateTime<Utc>> {+ Ok(DateTime::parse_from_rfc3339(raw)+ .with_context(|| format!("invalid RFC3339 timestamp: {raw}"))?+ .with_timezone(&Utc))+}++async fn read_runs(db: &LegacyDb, report: &mut ImportReport) -> Result<Vec<CronRun>> {+ let have = db.columns("cron_runs").await?;+ let select = format!(+ "SELECT id, job_id, started_at, finished_at, status, output, duration_ms, {} \+ FROM cron_runs ORDER BY id",+ column_or(&have, "delivery_status", "NULL"),+ );+ let rows = db+ .read("cron runs", move |conn| {+ let mut statement = conn.prepare(&select)?;+ statement+ .query_map([], |row| {+ Ok((+ row.get::<_, i64>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, String>(2)?,+ row.get::<_, String>(3)?,+ row.get::<_, String>(4)?,+ row.get::<_, Option<String>>(5)?,+ row.get::<_, Option<i64>>(6)?,+ row.get::<_, Option<String>>(7)?,+ ))+ })?+ .collect::<Result<Vec<_>, _>>()+ })+ .await?;+ let mut runs = Vec::new();+ for (id, job_id, started_at, finished_at, status, output, duration_ms, delivery_status) in rows+ {+ match (instant(&started_at), instant(&finished_at)) {+ (Ok(started_at), Ok(finished_at)) => runs.push(CronRun {+ id,+ job_id,+ run_id: None,+ started_at,+ finished_at,+ status,+ output,+ duration_ms,+ delivery_status: delivery_status.as_deref().and_then(DeliveryStatus::parse),+ }),+ (Err(error), _) | (_, Err(error)) => {+ report.invalid("runs", format!("run {id}: {error:#}"));+ }+ }+ }+ Ok(runs)+}diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rsnew file mode 100644index 0000000000..9cc9f6160f--- /dev/null+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs@@ -0,0 +1,105 @@+//! Reading a legacy SQLite file without touching it.+//!+//! The file is copied, with its `-wal` and `-shm` if it has them, to a scratch+//! directory, and the copy is opened through the SQLite driver's native handle.+//! The driver creates what it needs and sets WAL mode on open, so opening the+//! original would modify it; the copy takes that instead and is deleted when+//! the [`LegacyDb`] drops.++use std::collections::HashSet;+use std::path::Path;++use anyhow::{Context, Result};+use tinystoragedrivers_sqlite::SqliteNative;+use tinystoragedrivers_sqlite::rusqlite::Connection;++/// A scratch copy of a legacy SQLite database, open for reading.+pub(crate) struct LegacyDb {+ // Held for its drop: the directory goes when the handle does.+ _scratch: tempfile::TempDir,+ native: SqliteNative,+}++impl LegacyDb {+ /// Opens a copy of the database at `path`; `None` when there is no such+ /// file, which is the normal state of a store that was never used.+ pub(crate) fn open(path: &Path) -> Result<Option<Self>> {+ if !path.is_file() {+ return Ok(None);+ }+ let scratch = tempfile::tempdir().context("creating a scratch directory")?;+ let name = path+ .file_name()+ .context("a database path has a file name")?+ .to_os_string();+ for suffix in ["", "-wal", "-shm"] {+ let mut from = path.as_os_str().to_os_string();+ from.push(suffix);+ let from = std::path::PathBuf::from(from);+ if from.is_file() {+ let mut to = name.clone();+ to.push(suffix);+ std::fs::copy(&from, scratch.path().join(to))+ .with_context(|| format!("copying {}", from.display()))?;+ }+ }+ let native = SqliteNative::open(scratch.path().join(&name))+ .with_context(|| format!("opening a copy of {}", path.display()))?;+ Ok(Some(Self {+ _scratch: scratch,+ native,+ }))+ }++ /// Runs `read` on the copy's connection.+ pub(crate) async fn read<T, F>(&self, what: &'static str, read: F) -> Result<T>+ where+ T: Send + 'static,+ F: FnOnce(&Connection) -> tinystoragedrivers_sqlite::rusqlite::Result<T> + Send + 'static,+ {+ self.native+ .with_connection(read)+ .await+ .with_context(|| format!("reading {what} from the legacy database"))+ }++ /// The tables the database has.+ pub(crate) async fn tables(&self) -> Result<HashSet<String>> {+ self.read("the table list", |conn| {+ let mut statement =+ conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?;+ let names = statement+ .query_map([], |row| row.get::<_, String>(0))?+ .collect::<Result<HashSet<_>, _>>()?;+ Ok(names)+ })+ .await+ }++ /// The columns `table` has; empty when it has none or does not exist.+ pub(crate) async fn columns(&self, table: &'static str) -> Result<HashSet<String>> {+ self.read("a column list", move |conn| {+ let mut statement = conn.prepare(&format!("PRAGMA table_info({table})"))?;+ let names = statement+ .query_map([], |row| row.get::<_, String>(1))?+ .collect::<Result<HashSet<_>, _>>()?;+ Ok(names)+ })+ .await+ }+}++/// `column` when `have` lists it, else the literal `default`, as a select-list+/// entry: how a database written before a column existed is read as if the+/// column held its default.+pub(crate) fn column_or(have: &HashSet<String>, column: &str, default: &str) -> String {+ if have.contains(column) {+ column.to_string()+ } else {+ format!("{default} AS {column}")+ }+}++#[cfg(test)]+#[path = "legacy_db_tests.rs"]+mod tests;diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rsnew file mode 100644index 0000000000..46ed8fbcaa--- /dev/null+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/report.rs@@ -0,0 +1,134 @@+//! What an import did, record kind by record kind.++use std::collections::BTreeMap;++#[cfg(all(+ feature = "flow-read",+ any(feature = "flow-write", feature = "checkpoint-write")+))]+fn legacy_kind_name(kind: tinyflows_bus::catalog::LegacyImportKind) -> &'static str {+ use tinyflows_bus::catalog::LegacyImportKind as Kind;+ match kind {+ Kind::Flow => "flows",+ Kind::Revision => "revisions",+ Kind::Run => "runs",+ Kind::State => "state",+ Kind::Suggestion => "suggestions",+ Kind::Draft => "drafts",+ Kind::Checkpoint => "checkpoints",+ Kind::Write => "writes",+ }+}++/// How many records of one kind an import wrote, found already present, or+/// could not decode.+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]+pub struct Tally {+ /// Records this import wrote.+ pub written: usize,+ /// Records the target already held, left exactly as they were.+ pub existing: usize,+ /// Records in the legacy store that could not be decoded or were refused.+ pub invalid: usize,+}++/// The outcome of one `import_*` call.+#[derive(Debug, Default, Clone, PartialEq, Eq)]+pub struct ImportReport {+ kinds: BTreeMap<&'static str, Tally>,+ problems: Vec<String>,+}++impl ImportReport {+ #[cfg(all(+ feature = "flow-read",+ any(feature = "flow-write", feature = "checkpoint-write")+ ))]+ pub(crate) fn merge_legacy_batch(+ &mut self,+ response: tinyflows_bus::catalog::ImportLegacyBatchResponse,+ ) {+ for tally in response.tallies {+ let kind = legacy_kind_name(tally.kind);+ let total = self.kinds.entry(kind).or_default();+ total.written += tally.written as usize;+ total.existing += tally.existing as usize;+ total.invalid += tally.invalid as usize;+ }+ for problem in response.problems {+ let identity = problem.identity.map_or_else(String::new, |id| {+ let label = match problem.kind {+ tinyflows_bus::catalog::LegacyImportKind::Flow => "flow",+ tinyflows_bus::catalog::LegacyImportKind::Revision => "revision",+ tinyflows_bus::catalog::LegacyImportKind::Run => "run",+ tinyflows_bus::catalog::LegacyImportKind::State => "state",+ tinyflows_bus::catalog::LegacyImportKind::Suggestion => "suggestion",+ tinyflows_bus::catalog::LegacyImportKind::Draft => "draft",+ tinyflows_bus::catalog::LegacyImportKind::Checkpoint => "checkpoint",+ tinyflows_bus::catalog::LegacyImportKind::Write => "write",+ };+ format!(" {label} '{id}'")+ });+ let message = format!(+ "{}{}: {}",+ legacy_kind_name(problem.kind),+ identity,+ problem.message+ );+ tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}");+ self.problems.push(message);+ }+ }++ pub fn wrote(&mut self, kind: &'static str, written: bool) {+ let tally = self.kinds.entry(kind).or_default();+ if written {+ tally.written += 1;+ } else {+ tally.existing += 1;+ }+ }++ /// Records one record that could not be imported, with why.+ pub fn invalid(&mut self, kind: &'static str, what: impl std::fmt::Display) {+ self.kinds.entry(kind).or_default().invalid += 1;+ let message = format!("{kind}: {what}");+ tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}");+ self.problems.push(message);+ }++ /// The tally of `kind` (`"flows"`, `"runs"`, `"jobs"`, ...); all zero when+ /// the store held none.+ pub fn tally(&self, kind: &str) -> Tally {+ self.kinds.get(kind).copied().unwrap_or_default()+ }++ /// How many records of `kind` this import wrote.+ pub fn written(&self, kind: &str) -> usize {+ self.tally(kind).written+ }++ /// Every kind seen, with its tally, in name order.+ pub fn kinds(&self) -> impl Iterator<Item = (&'static str, Tally)> + '_ {+ self.kinds.iter().map(|(kind, tally)| (*kind, *tally))+ }++ /// One line per record that could not be imported.+ pub fn problems(&self) -> &[String] {+ &self.problems+ }++ /// Records written across every kind.+ pub fn total_written(&self) -> usize {+ self.kinds.values().map(|tally| tally.written).sum()+ }++ /// Whether every legacy record was either written or already present.+ pub fn is_clean(&self) -> bool {+ self.problems.is_empty()+ }+}++#[cfg(test)]+#[path = "report_tests.rs"]+mod tests;