Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): split raw SQL read and write paths

Louis Beaumont3115a700dce6
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs b/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs
index cb4791d76..a79b6b733 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs
@@ -221,15 +221,17 @@ async fn e2e_inject_db_hard_fault_impl(
};
database
- .execute_raw_sql("DROP TABLE IF EXISTS e2e_hard_fault_probe")
+ .execute_raw_sql_write("DROP TABLE IF EXISTS e2e_hard_fault_probe")
.await
.map_err(|error| format!("failed to reset corruption fixture: {error}"))?;
database
- .execute_raw_sql("CREATE TABLE e2e_hard_fault_probe(id INTEGER PRIMARY KEY, payload BLOB)")
+ .execute_raw_sql_write(
+ "CREATE TABLE e2e_hard_fault_probe(id INTEGER PRIMARY KEY, payload BLOB)",
+ )
.await
.map_err(|error| format!("failed to create corruption fixture: {error}"))?;
database
- .execute_raw_sql(
+ .execute_raw_sql_write(
"WITH RECURSIVE rows(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM rows WHERE id < 200) \
INSERT INTO e2e_hard_fault_probe(id, payload) SELECT id, randomblob(3000) FROM rows",
)
@@ -238,7 +240,7 @@ async fn e2e_inject_db_hard_fault_impl(
checkpoint_hard_fault_fixture(&database).await?;
let page_size = database
- .execute_raw_sql("PRAGMA page_size")
+ .query_raw_sql("PRAGMA page_size")
.await
.map_err(|error| format!("failed to read page size: {error}"))?
.as_array()
@@ -247,7 +249,7 @@ async fn e2e_inject_db_hard_fault_impl(
.and_then(|value| value.as_u64())
.ok_or_else(|| "SQLite returned no page size".to_string())?;
let leaf_page = database
- .execute_raw_sql(
+ .query_raw_sql(
"SELECT pageno FROM dbstat WHERE name = 'e2e_hard_fault_probe' \
AND pagetype = 'leaf' ORDER BY pageno DESC LIMIT 1",
)
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index 5e955f602..f10674a12 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -43,11 +43,29 @@ async fn run_wal_restart_checkpoint(
}
impl DatabaseManager {
- pub async fn execute_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {
- // This API intentionally executes caller-supplied maintenance SQL.
- // Keep that trust boundary explicit for SQLx 0.9's dynamic-SQL audit.
+ /// Execute trusted, row-returning dynamic SQL on the query pool.
+ pub async fn query_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {
+ Self::raw_sql_on_pool(&self.pool, query).await
+ }
+
+ /// Execute trusted dynamic SQL that may mutate the database through the
+ /// dedicated, process-coordinated writer lane.
+ pub async fn execute_raw_sql_write(
+ &self,
+ query: &str,
+ ) -> Result<serde_json::Value, sqlx::Error> {
+ let writer = self.coordinated_writer().lock().await?;
+ Self::raw_sql_on_pool(writer.pool(), query).await
+ }
+
+ async fn raw_sql_on_pool(
+ pool: &SqlitePool,
+ query: &str,
+ ) -> Result<serde_json::Value, sqlx::Error> {
+ // These APIs intentionally execute caller-supplied trusted SQL. Keep
+ // that trust boundary explicit for SQLx 0.9's dynamic-SQL audit.
let rows = sqlx::query(sqlx::AssertSqlSafe(query))
- .fetch_all(&self.pool)
+ .fetch_all(pool)
.await?;
let result: Vec<serde_json::Map<String, serde_json::Value>> = rows
@@ -1707,6 +1725,47 @@ mod wal_maintenance_tests {
use sqlx::Row;
use std::time::Duration;
+ #[tokio::test]
+ async fn raw_sql_writes_wait_for_coordinator_and_reads_return_rows() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let db_path = dir.path().join("db.sqlite");
+ let db_path_string = db_path.to_string_lossy().into_owned();
+ let db = DatabaseManager::new(&db_path_string, DbConfig::for_tier(DeviceTier::Low))
+ .await
+ .expect("database manager");
+
+ db.execute_raw_sql_write("CREATE TABLE raw_boundary (value INTEGER NOT NULL)")
+ .await
+ .expect("create through writer API");
+
+ let held_writer = db
+ .coordinated_writer()
+ .lock()
+ .await
+ .expect("hold writer lane");
+ let mut queued_write =
+ Box::pin(db.execute_raw_sql_write("INSERT INTO raw_boundary (value) VALUES (42)"));
+ assert!(
+ tokio::time::timeout(Duration::from_millis(50), &mut queued_write)
+ .await
+ .is_err(),
+ "raw write bypassed the coordinator"
+ );
+ drop(held_writer);
+ tokio::time::timeout(Duration::from_secs(1), queued_write)
+ .await
+ .expect("queued write timed out")
+ .expect("queued write failed");
+
+ let rows = db
+ .query_raw_sql("SELECT value FROM raw_boundary LIMIT 1")
+ .await
+ .expect("query through reader API");
+ assert_eq!(rows, serde_json::json!([{ "value": 42 }]));
+
+ db.close().await;
+ }
+
#[tokio::test]
async fn routine_checkpoint_never_truncates_the_live_wal() {
let dir = tempfile::tempdir().expect("temp dir");
diff --git a/crates/screenpipe-engine/src/retention.rs b/crates/screenpipe-engine/src/retention.rs
index fd550d945..ff496f847 100644
--- a/crates/screenpipe-engine/src/retention.rs
+++ b/crates/screenpipe-engine/src/retention.rs
@@ -678,7 +678,10 @@ async fn do_local_cleanup(
// becomes real if/when the DB is migrated to incremental auto_vacuum.
if matches!(mode, RetentionMode::All | RetentionMode::Lean) {
info!("retention: running incremental vacuum (reclaims pages only under auto_vacuum=incremental)");
- if let Err(e) = db.execute_raw_sql("PRAGMA incremental_vacuum(1000)").await {
+ if let Err(e) = db
+ .execute_raw_sql_write("PRAGMA incremental_vacuum(1000)")
+ .await
+ {
warn!("retention: incremental vacuum failed: {}", e);
}
}
@@ -919,7 +922,9 @@ mod tests {
// instantly: `begin_immediate_with_retry` only retries
// connection-acquisition errors, not query errors after BEGIN, so a
// missing table surfaces immediately as Err, no backoff needed.
- db.execute_raw_sql("DROP TABLE video_chunks").await.unwrap();
+ db.execute_raw_sql_write("DROP TABLE video_chunks")
+ .await
+ .unwrap();
let now = Utc::now();
let start = now - Duration::hours(2);
diff --git a/crates/screenpipe-engine/src/routes/activity_summary.rs b/crates/screenpipe-engine/src/routes/activity_summary.rs
index d4653b330..5ea1d65a3 100644
--- a/crates/screenpipe-engine/src/routes/activity_summary.rs
+++ b/crates/screenpipe-engine/src/routes/activity_summary.rs
@@ -583,13 +583,13 @@ async fn collect_summary_core(
edited_files_result,
active_ts_result,
) = tokio::join!(
- db.execute_raw_sql(&apps_query),
- db.execute_raw_sql(&windows_query),
- db.execute_raw_sql(&texts_query),
- db.execute_raw_sql(&audio_speakers_query),
- db.execute_raw_sql(&audio_transcripts_query),
- db.execute_raw_sql(&edited_files_query),
- db.execute_raw_sql(&active_ts_query),
+ db.query_raw_sql(&apps_query),
+ db.query_raw_sql(&windows_query),
+ db.query_raw_sql(&texts_query),
+ db.query_raw_sql(&audio_speakers_query),
+ db.query_raw_sql(&audio_transcripts_query),
+ db.query_raw_sql(&edited_files_query),
+ db.query_raw_sql(&active_ts_query),
);
let mut apps = Vec::new();
@@ -762,10 +762,7 @@ async fn load_recording_status(
(SELECT ROUND((JULIANDAY('{now}') - JULIANDAY(MAX(timestamp))) * 86400) FROM audio_transcriptions) AS seconds_since_last_audio"
);
- let rows = db
- .execute_raw_sql(&query)
- .await
- .map_err(|e| e.to_string())?;
+ let rows = db.query_raw_sql(&query).await.map_err(|e| e.to_string())?;
let row = rows
.as_array()
.and_then(|a| a.first())
@@ -891,7 +888,7 @@ async fn load_snippets(
}
let audio_rows = db
- .execute_raw_sql(&audio_query)
+ .query_raw_sql(&audio_query)
.await
.map_err(|e| e.to_string())?;
if let Some(rows) = audio_rows.as_array() {
@@ -1604,12 +1601,12 @@ mod db_tests {
sql_val(window),
if focused { 1 } else { 0 }
);
- db.execute_raw_sql(&q).await.expect("insert frame");
+ db.execute_raw_sql_write(&q).await.expect("insert frame");
}
async fn last_frame_id(db: &DatabaseManager) -> i64 {
let rows = db
- .execute_raw_sql("SELECT MAX(id) AS id FROM frames")
+ .query_raw_sql("SELECT MAX(id) AS id FROM frames")
.await
.expect("select latest frame id");
rows.as_array()
@@ -1871,7 +1868,7 @@ mod db_tests {
let (db, _d) = fresh_db().await;
let p = "/Users/me/proj/main.rs";
for ts in ["10:00:00", "10:00:20"] {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO frames (timestamp, app_name, window_name, document_path) \
VALUES ('{DAY} {ts}', 'Code', 'main.rs', '{p}')"
))
@@ -1893,17 +1890,17 @@ mod db_tests {
let (db, _d) = fresh_db().await;
// audio_transcriptions.audio_chunk_id is a NOT NULL FK (sqlx enables
// foreign_keys), so the parent chunk must exist first.
- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
.await
.unwrap();
- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
.await
.unwrap();
for (m, text) in [
(0, "hello team this is the weekly sync"),
(1, "lets review the roadmap now"),
] {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO audio_transcriptions \
(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
VALUES (1, {m}, '{DAY} 10:0{m}:00', '{text}', 'mic', 1)"
@@ -1925,10 +1922,10 @@ mod db_tests {
#[tokio::test]
async fn weekly_audio_context_is_not_consumed_by_the_last_few_minutes() {
let (db, _d) = fresh_db().await;
- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
.await
.unwrap();
- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
.await
.unwrap();
@@ -1936,7 +1933,7 @@ mod db_tests {
// the final minutes to crowd every older row out of the old global
// length sort and newest-first snippet query.
for day in 1..=7 {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO audio_transcriptions \
(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
VALUES (1, {day}, '2026-06-0{day} 10:00:00', \
@@ -1946,7 +1943,7 @@ mod db_tests {
.unwrap();
}
for second in 0..24 {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO audio_transcriptions \
(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
VALUES (1, {}, '2026-06-07 23:59:{second:02}', \
@@ -2022,15 +2019,15 @@ mod db_tests {
#[tokio::test]
async fn audio_context_backfills_capacity_from_one_dense_period() {
let (db, _d) = fresh_db().await;
- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
.await
.unwrap();
- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
.await
.unwrap();
for second in 0..24 {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO audio_transcriptions \
(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
VALUES (1, {second}, '2026-06-07 23:59:{second:02}', \
@@ -2073,7 +2070,7 @@ mod db_tests {
.await;
// The frame just inserted is the only row, so its id is 1.
let txt = "Quarterly planning notes for the leadership offsite";
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO elements (frame_id, source, role, text, depth, sort_order) \
VALUES (1, 'accessibility', 'AXTextField', '{txt}', 0, 0)"
))
@@ -2100,7 +2097,7 @@ mod db_tests {
let txt = format!(
"weekly day {day} planning notes with enough detail for summary regression"
);
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO elements (frame_id, source, role, text, depth, sort_order) \
VALUES ({frame_id}, 'accessibility', 'AXTextField', '{}', 0, 0)",
txt.replace('\'', "''")
@@ -2189,7 +2186,7 @@ mod db_tests {
"2026-06-03T10:00:00Z",
),
] {
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO memories (content, source, tags, importance, created_at, updated_at) \
VALUES ('{}', 'test', '[]', 0.9, '{created_at}', '{created_at}')",
content.replace('\'', "''")
@@ -2250,7 +2247,7 @@ mod db_tests {
));
i += 1;
}
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO frames (timestamp, app_name, window_name) VALUES {}",
vals.join(",")
))
@@ -2349,7 +2346,7 @@ mod db_tests {
));
i += 1;
}
- db.execute_raw_sql(&format!(
+ db.execute_raw_sql_write(&format!(
"INSERT INTO frames (timestamp, app_name, window_name) VALUES {}",
vals.join(",")
))
@@ -2389,7 +2386,7 @@ mod db_tests {
let (db, _d) = fresh_db().await;
// Mirrors the WHERE clause shared by the apps / windows / active-ts queries.
let plan = db
- .execute_raw_sql(
+ .query_raw_sql(
"EXPLAIN QUERY PLAN SELECT timestamp FROM frames \
WHERE timestamp BETWEEN '2026-05-15T00:00:00Z' AND '2026-05-15T01:00:00Z' \
AND app_name IS NOT NULL AND app_name != ''",
diff --git a/crates/screenpipe-engine/src/routes/content.rs b/crates/screenpipe-engine/src/routes/content.rs
index 4c09889f5..aa6cd4ded 100644
--- a/crates/screenpipe-engine/src/routes/content.rs
+++ b/crates/screenpipe-engine/src/routes/content.rs
@@ -719,7 +719,7 @@ pub(crate) async fn execute_raw_sql(
return Err((StatusCode::BAD_REQUEST, JsonResponse(json!({"error": msg}))));
}
- match state.db.execute_raw_sql(&payload.query).await {
+ match state.db.query_raw_sql(&payload.query).await {
Ok(result) => Ok(JsonResponse(result)),
// A database-level error means SQLite rejected the *query* itself —
// unknown table/column, syntax error, etc. That's a caller mistake, not
diff --git a/packages/sdk/recorder-core/tests/sqlite_coordinator.rs b/packages/sdk/recorder-core/tests/sqlite_coordinator.rs
index 28a997300..1f8efacba 100644
--- a/packages/sdk/recorder-core/tests/sqlite_coordinator.rs
+++ b/packages/sdk/recorder-core/tests/sqlite_coordinator.rs
@@ -48,7 +48,7 @@ async fn sdk_database_uses_safe_sqlite_and_shared_checkpoint_coordinator() {
assert_eq!(busy, 0, "checkpoint reported busy");
let integrity = db
- .execute_raw_sql("PRAGMA integrity_check")
+ .query_raw_sql("PRAGMA integrity_check")
.await
.expect("integrity_check");
let status = integrity