Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): rewrite driver_docs to use rusqlite directly

Steven Enamakel51ae080a3ab2
diff --git a/crates/openhuman-core/src/storage/import/snapshot.rs b/crates/openhuman-core/src/storage/import/snapshot.rs
index f74c1dbb59..245d64038b 100644
--- a/crates/openhuman-core/src/storage/import/snapshot.rs
+++ b/crates/openhuman-core/src/storage/import/snapshot.rs
@@ -71,32 +71,121 @@ impl Snapshot {
///
/// When the copy cannot be opened with the SQLite driver or read.
pub fn driver_docs(&self, collections: fn() -> Vec<CollectionSpec>) -> Result<Vec<Record>> {
- use crate::storage::DocumentStoreExt;
- use tinystoragedrivers::{Query, StorageBackend};
-
- let storage = tinystoragedrivers::sqlite::SqliteStorage::open(&self.db)
- .with_context(|| format!("open the copy {}", self.db.display()))?;
- let scoped = storage
- .for_scope(&crate::storage::tenant_scope(None))
- .context("open the local scope of the copy")?;
- let docs = std::sync::Arc::clone(scoped.documents());
- let specs = collections();
- let found = crate::storage::block_on(async move {
- let mut out = Vec::new();
- for spec in specs {
- docs.ensure_collection(&spec).await?;
- for stored in docs.query_all(&spec.name, &Query::all()).await? {
- out.push(Record {
- collection: spec.name.clone(),
- id: stored.id,
- doc: stored.doc,
- created_ms: stored.meta.created_ms,
- });
- }
+ let conn = rusqlite::Connection::open_with_flags(
+ &self.db,
+ rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
+ )
+ .with_context(|| format!("open the copy {} read-only", self.db.display()))?;
+ let table = if table_exists(&conn, "docs")? {
+ Some(("docs", "body"))
+ } else if table_exists(&conn, "_tsd_docs")? {
+ Some(("_tsd_docs", "doc"))
+ } else {
+ None
+ };
+ let Some((table, body_column)) = table else {
+ return Ok(Vec::new());
+ };
+ let columns = table_columns(&conn, table)?;
+ for required in ["scope", "coll", "id", body_column] {
+ if !columns.contains(required) {
+ anyhow::bail!("legacy driver table {table} is missing {required}");
+ }
+ }
+
+ // `table` and `body_column` are selected from the fixed names above;
+ // collection names remain SQL parameters. Both the v0 document table
+ // and layout-2 table are only read from the scratch snapshot.
+ let sql = format!(
+ "SELECT coll, id, {body_column} FROM \"{table}\" WHERE scope = ?1 AND coll = ?2 AND COALESCE(deleted, 0) = 0 ORDER BY id"
+ );
+ let mut out = Vec::new();
+ for spec in collections() {
+ if !table_has_column(&columns, "deleted") {
+ let sql = format!(
+ "SELECT coll, id, {body_column} FROM \"{table}\" WHERE scope = ?1 AND coll = ?2 ORDER BY id"
+ );
+ read_collection(&conn, &sql, &spec.name, &mut out)?;
+ } else {
+ read_collection(&conn, &sql, &spec.name, &mut out)?;
}
- Ok(out)
- })?;
- Ok(found)
+ }
+ Ok(out)
+ }
+}
+
+fn table_exists(conn: &rusqlite::Connection, table: &str) -> Result<bool> {
+ conn.query_row(
+ "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
+ [table],
+ |row| row.get(0),
+ )
+ .context("inspect legacy driver tables")
+}
+
+fn table_columns(
+ conn: &rusqlite::Connection,
+ table: &str,
+) -> Result<std::collections::HashSet<String>> {
+ let mut statement = conn
+ .prepare(&format!("PRAGMA table_info(\"{table}\")"))
+ .context("inspect legacy driver document columns")?;
+ let columns = statement
+ .query_map([], |row| row.get::<_, String>(1))
+ .context("read legacy driver document columns")?
+ .collect::<rusqlite::Result<std::collections::HashSet<_>>>()
+ .context("collect legacy driver document columns")?;
+ Ok(columns)
+}
+
+fn table_has_column(columns: &std::collections::HashSet<String>, name: &str) -> bool {
+ columns.contains(name)
+}
+
+fn read_collection(
+ conn: &rusqlite::Connection,
+ sql: &str,
+ collection: &str,
+ out: &mut Vec<Record>,
+) -> Result<()> {
+ use rusqlite::types::Value as SqlValue;
+
+ let mut statement = conn
+ .prepare(sql)
+ .context("prepare legacy driver document query")?;
+ let rows = statement
+ .query_map(["local", collection], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, SqlValue>(2)?,
+ ))
+ })
+ .context("query legacy driver documents")?;
+ for row in rows {
+ let (stored_collection, id, body) = row.context("read legacy driver document")?;
+ out.push(Record {
+ collection: stored_collection,
+ id,
+ doc: decode_legacy_body(body)?,
+ created_ms: None,
+ });
+ }
+ Ok(())
+}
+
+fn decode_legacy_body(body: rusqlite::types::Value) -> Result<serde_json::Value> {
+ match body {
+ rusqlite::types::Value::Text(text) => {
+ serde_json::from_str(&text).context("decode legacy driver document JSON")
+ }
+ rusqlite::types::Value::Blob(bytes) if bytes.first() == Some(&1) => {
+ let json =
+ zstd::decode_all(&bytes[1..]).context("decompress legacy driver document")?;
+ serde_json::from_slice(&json).context("decode legacy driver document JSON")
+ }
+ rusqlite::types::Value::Blob(_) => anyhow::bail!("unknown legacy driver body encoding"),
+ _ => anyhow::bail!("legacy driver document body is not text or blob"),
}
}