Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): update legacy db reader to use snapshot abstraction

Steven Enamakelc2d71f6d6806
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs
index 9cc9f6160f..4e6d7d89de 100644
--- a/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/legacy_db.rs
@@ -1,105 +1,81 @@
-//! Reading a legacy SQLite file without touching it.
-//!
-//! The file is copied, with its `-wal` and `-shm` if it has them, to a scratch
-//! directory, and the copy is opened through the SQLite driver's native handle.
-//! The driver creates what it needs and sets WAL mode on open, so opening the
-//! original would modify it; the copy takes that instead and is deleted when
-//! the [`LegacyDb`] drops.
+//! Read retired TinyFlows SQLite files through a consistent scratch snapshot.
use std::collections::HashSet;
-use std::path::Path;
+use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
-use tinystoragedrivers_sqlite::SqliteNative;
-use tinystoragedrivers_sqlite::rusqlite::Connection;
-/// A scratch copy of a legacy SQLite database, open for reading.
-pub(crate) struct LegacyDb {
- // Held for its drop: the directory goes when the handle does.
- _scratch: tempfile::TempDir,
- native: SqliteNative,
+use crate::storage::import::Snapshot;
+use crate::storage::rusqlite;
+
+/// A consistent scratch snapshot of a legacy database.
+pub(super) struct LegacyDb {
+ snapshot: Snapshot,
}
impl LegacyDb {
- /// Opens a copy of the database at `path`; `None` when there is no such
- /// file, which is the normal state of a store that was never used.
- pub(crate) fn open(path: &Path) -> Result<Option<Self>> {
+ /// Takes a read-only snapshot of an existing database.
+ pub(super) fn open(path: &Path) -> Result<Option<Self>> {
if !path.is_file() {
return Ok(None);
}
- let scratch = tempfile::tempdir().context("creating a scratch directory")?;
let name = path
.file_name()
- .context("a database path has a file name")?
- .to_os_string();
- for suffix in ["", "-wal", "-shm"] {
- let mut from = path.as_os_str().to_os_string();
- from.push(suffix);
- let from = std::path::PathBuf::from(from);
- if from.is_file() {
- let mut to = name.clone();
- to.push(suffix);
- std::fs::copy(&from, scratch.path().join(to))
- .with_context(|| format!("copying {}", from.display()))?;
- }
- }
- let native = SqliteNative::open(scratch.path().join(&name))
- .with_context(|| format!("opening a copy of {}", path.display()))?;
+ .and_then(|name| name.to_str())
+ .context("legacy database path has no UTF-8 file name")?;
Ok(Some(Self {
- _scratch: scratch,
- native,
+ snapshot: Snapshot::of(path, name)?,
}))
}
- /// Runs `read` on the copy's connection.
- pub(crate) async fn read<T, F>(&self, what: &'static str, read: F) -> Result<T>
+ /// Runs a read query against the private snapshot.
+ pub(super) async fn read<T, F>(&self, what: &'static str, read: F) -> Result<T>
where
T: Send + 'static,
- F: FnOnce(&Connection) -> tinystoragedrivers_sqlite::rusqlite::Result<T> + Send + 'static,
+ F: FnOnce(&rusqlite::Connection) -> rusqlite::Result<T> + Send + 'static,
{
- self.native
- .with_connection(read)
- .await
- .with_context(|| format!("reading {what} from the legacy database"))
+ let path: PathBuf = self.snapshot.db().to_path_buf();
+ tokio::task::spawn_blocking(move || {
+ let connection = rusqlite::Connection::open_with_flags(
+ path,
+ rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
+ )?;
+ read(&connection)
+ })
+ .await
+ .with_context(|| format!("reading {what} from the legacy database"))?
+ .map_err(anyhow::Error::from)
}
- /// The tables the database has.
- pub(crate) async fn tables(&self) -> Result<HashSet<String>> {
- self.read("the table list", |conn| {
+ /// Lists tables present in the snapshot.
+ pub(super) async fn tables(&self) -> Result<HashSet<String>> {
+ self.read("the table list", |connection| {
let mut statement =
- conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?;
- let names = statement
+ connection.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?;
+ statement
.query_map([], |row| row.get::<_, String>(0))?
- .collect::<Result<HashSet<_>, _>>()?;
- Ok(names)
+ .collect::<rusqlite::Result<HashSet<_>>>()
})
.await
}
- /// The columns `table` has; empty when it has none or does not exist.
- pub(crate) async fn columns(&self, table: &'static str) -> Result<HashSet<String>> {
- self.read("a column list", move |conn| {
- let mut statement = conn.prepare(&format!("PRAGMA table_info({table})"))?;
- let names = statement
+ /// Lists columns present in a table.
+ pub(super) async fn columns(&self, table: &'static str) -> Result<HashSet<String>> {
+ self.read("a column list", move |connection| {
+ let mut statement = connection.prepare(&format!("PRAGMA table_info({table})"))?;
+ statement
.query_map([], |row| row.get::<_, String>(1))?
- .collect::<Result<HashSet<_>, _>>()?;
- Ok(names)
+ .collect::<rusqlite::Result<HashSet<_>>>()
})
.await
}
}
-/// `column` when `have` lists it, else the literal `default`, as a select-list
-/// entry: how a database written before a column existed is read as if the
-/// column held its default.
-pub(crate) fn column_or(have: &HashSet<String>, column: &str, default: &str) -> String {
+/// Selects the actual column or a literal historical default.
+pub(super) fn column_or(have: &HashSet<String>, column: &str, default: &str) -> String {
if have.contains(column) {
column.to_string()
} else {
format!("{default} AS {column}")
}
}
-
-#[cfg(test)]
-#[path = "legacy_db_tests.rs"]
-mod tests;
diff --git a/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs
new file mode 100644
index 0000000000..e8911f75d7
--- /dev/null
+++ b/crates/openhuman-core/src/storage/import/tinyflows_reader/mod.rs
@@ -0,0 +1,14 @@
+//! Host-owned read-only decoders for TinyFlows' retired SQLite and JSONL
+//! formats. The returned values are bus DTOs; this module contains no owner
+//! storage implementation and writes only to its private SQLite snapshots.
+
+mod checkpoint_reader;
+mod flow_reader;
+mod legacy_cron;
+mod legacy_db;
+mod report;
+
+pub(super) use checkpoint_reader::read_legacy_checkpoint_batches;
+pub(super) use flow_reader::read_legacy_flow_catalog;
+pub(super) use legacy_cron::{LegacyCron, read_legacy_cron};
+pub(super) use report::ImportReport;