YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): split raw SQL read and write paths
Louis Beaumont3115a700dce6
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs b/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rsindex cb4791d76..a79b6b733 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/e2e_seed.rs@@ -221,15 +221,17 @@ async fn e2e_inject_db_hard_fault_impl(};database- .execute_raw_sql("DROP TABLE IF EXISTS e2e_hard_fault_probe")+ .execute_raw_sql_write("DROP TABLE IF EXISTS e2e_hard_fault_probe").await.map_err(|error| format!("failed to reset corruption fixture: {error}"))?;database- .execute_raw_sql("CREATE TABLE e2e_hard_fault_probe(id INTEGER PRIMARY KEY, payload BLOB)")+ .execute_raw_sql_write(+ "CREATE TABLE e2e_hard_fault_probe(id INTEGER PRIMARY KEY, payload BLOB)",+ ).await.map_err(|error| format!("failed to create corruption fixture: {error}"))?;database- .execute_raw_sql(+ .execute_raw_sql_write("WITH RECURSIVE rows(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM rows WHERE id < 200) \INSERT INTO e2e_hard_fault_probe(id, payload) SELECT id, randomblob(3000) FROM rows",)@@ -238,7 +240,7 @@ async fn e2e_inject_db_hard_fault_impl(checkpoint_hard_fault_fixture(&database).await?;let page_size = database- .execute_raw_sql("PRAGMA page_size")+ .query_raw_sql("PRAGMA page_size").await.map_err(|error| format!("failed to read page size: {error}"))?.as_array()@@ -247,7 +249,7 @@ async fn e2e_inject_db_hard_fault_impl(.and_then(|value| value.as_u64()).ok_or_else(|| "SQLite returned no page size".to_string())?;let leaf_page = database- .execute_raw_sql(+ .query_raw_sql("SELECT pageno FROM dbstat WHERE name = 'e2e_hard_fault_probe' \AND pagetype = 'leaf' ORDER BY pageno DESC LIMIT 1",)diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rsindex 5e955f602..f10674a12 100644--- a/crates/screenpipe-db/src/db/maintenance.rs+++ b/crates/screenpipe-db/src/db/maintenance.rs@@ -43,11 +43,29 @@ async fn run_wal_restart_checkpoint(}impl DatabaseManager {- pub async fn execute_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {- // This API intentionally executes caller-supplied maintenance SQL.- // Keep that trust boundary explicit for SQLx 0.9's dynamic-SQL audit.+ /// Execute trusted, row-returning dynamic SQL on the query pool.+ pub async fn query_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {+ Self::raw_sql_on_pool(&self.pool, query).await+ }++ /// Execute trusted dynamic SQL that may mutate the database through the+ /// dedicated, process-coordinated writer lane.+ pub async fn execute_raw_sql_write(+ &self,+ query: &str,+ ) -> Result<serde_json::Value, sqlx::Error> {+ let writer = self.coordinated_writer().lock().await?;+ Self::raw_sql_on_pool(writer.pool(), query).await+ }++ async fn raw_sql_on_pool(+ pool: &SqlitePool,+ query: &str,+ ) -> Result<serde_json::Value, sqlx::Error> {+ // These APIs intentionally execute caller-supplied trusted SQL. Keep+ // that trust boundary explicit for SQLx 0.9's dynamic-SQL audit.let rows = sqlx::query(sqlx::AssertSqlSafe(query))- .fetch_all(&self.pool)+ .fetch_all(pool).await?;let result: Vec<serde_json::Map<String, serde_json::Value>> = rows@@ -1707,6 +1725,47 @@ mod wal_maintenance_tests {use sqlx::Row;use std::time::Duration;+ #[tokio::test]+ async fn raw_sql_writes_wait_for_coordinator_and_reads_return_rows() {+ let dir = tempfile::tempdir().expect("temp dir");+ let db_path = dir.path().join("db.sqlite");+ let db_path_string = db_path.to_string_lossy().into_owned();+ let db = DatabaseManager::new(&db_path_string, DbConfig::for_tier(DeviceTier::Low))+ .await+ .expect("database manager");++ db.execute_raw_sql_write("CREATE TABLE raw_boundary (value INTEGER NOT NULL)")+ .await+ .expect("create through writer API");++ let held_writer = db+ .coordinated_writer()+ .lock()+ .await+ .expect("hold writer lane");+ let mut queued_write =+ Box::pin(db.execute_raw_sql_write("INSERT INTO raw_boundary (value) VALUES (42)"));+ assert!(+ tokio::time::timeout(Duration::from_millis(50), &mut queued_write)+ .await+ .is_err(),+ "raw write bypassed the coordinator"+ );+ drop(held_writer);+ tokio::time::timeout(Duration::from_secs(1), queued_write)+ .await+ .expect("queued write timed out")+ .expect("queued write failed");++ let rows = db+ .query_raw_sql("SELECT value FROM raw_boundary LIMIT 1")+ .await+ .expect("query through reader API");+ assert_eq!(rows, serde_json::json!([{ "value": 42 }]));++ db.close().await;+ }+#[tokio::test]async fn routine_checkpoint_never_truncates_the_live_wal() {let dir = tempfile::tempdir().expect("temp dir");diff --git a/crates/screenpipe-engine/src/retention.rs b/crates/screenpipe-engine/src/retention.rsindex fd550d945..ff496f847 100644--- a/crates/screenpipe-engine/src/retention.rs+++ b/crates/screenpipe-engine/src/retention.rs@@ -678,7 +678,10 @@ async fn do_local_cleanup(// becomes real if/when the DB is migrated to incremental auto_vacuum.if matches!(mode, RetentionMode::All | RetentionMode::Lean) {info!("retention: running incremental vacuum (reclaims pages only under auto_vacuum=incremental)");- if let Err(e) = db.execute_raw_sql("PRAGMA incremental_vacuum(1000)").await {+ if let Err(e) = db+ .execute_raw_sql_write("PRAGMA incremental_vacuum(1000)")+ .await+ {warn!("retention: incremental vacuum failed: {}", e);}}@@ -919,7 +922,9 @@ mod tests {// instantly: `begin_immediate_with_retry` only retries// connection-acquisition errors, not query errors after BEGIN, so a// missing table surfaces immediately as Err, no backoff needed.- db.execute_raw_sql("DROP TABLE video_chunks").await.unwrap();+ db.execute_raw_sql_write("DROP TABLE video_chunks")+ .await+ .unwrap();let now = Utc::now();let start = now - Duration::hours(2);diff --git a/crates/screenpipe-engine/src/routes/activity_summary.rs b/crates/screenpipe-engine/src/routes/activity_summary.rsindex d4653b330..5ea1d65a3 100644--- a/crates/screenpipe-engine/src/routes/activity_summary.rs+++ b/crates/screenpipe-engine/src/routes/activity_summary.rs@@ -583,13 +583,13 @@ async fn collect_summary_core(edited_files_result,active_ts_result,) = tokio::join!(- db.execute_raw_sql(&apps_query),- db.execute_raw_sql(&windows_query),- db.execute_raw_sql(&texts_query),- db.execute_raw_sql(&audio_speakers_query),- db.execute_raw_sql(&audio_transcripts_query),- db.execute_raw_sql(&edited_files_query),- db.execute_raw_sql(&active_ts_query),+ db.query_raw_sql(&apps_query),+ db.query_raw_sql(&windows_query),+ db.query_raw_sql(&texts_query),+ db.query_raw_sql(&audio_speakers_query),+ db.query_raw_sql(&audio_transcripts_query),+ db.query_raw_sql(&edited_files_query),+ db.query_raw_sql(&active_ts_query),);let mut apps = Vec::new();@@ -762,10 +762,7 @@ async fn load_recording_status((SELECT ROUND((JULIANDAY('{now}') - JULIANDAY(MAX(timestamp))) * 86400) FROM audio_transcriptions) AS seconds_since_last_audio");- let rows = db- .execute_raw_sql(&query)- .await- .map_err(|e| e.to_string())?;+ let rows = db.query_raw_sql(&query).await.map_err(|e| e.to_string())?;let row = rows.as_array().and_then(|a| a.first())@@ -891,7 +888,7 @@ async fn load_snippets(}let audio_rows = db- .execute_raw_sql(&audio_query)+ .query_raw_sql(&audio_query).await.map_err(|e| e.to_string())?;if let Some(rows) = audio_rows.as_array() {@@ -1604,12 +1601,12 @@ mod db_tests {sql_val(window),if focused { 1 } else { 0 });- db.execute_raw_sql(&q).await.expect("insert frame");+ db.execute_raw_sql_write(&q).await.expect("insert frame");}async fn last_frame_id(db: &DatabaseManager) -> i64 {let rows = db- .execute_raw_sql("SELECT MAX(id) AS id FROM frames")+ .query_raw_sql("SELECT MAX(id) AS id FROM frames").await.expect("select latest frame id");rows.as_array()@@ -1871,7 +1868,7 @@ mod db_tests {let (db, _d) = fresh_db().await;let p = "/Users/me/proj/main.rs";for ts in ["10:00:00", "10:00:20"] {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO frames (timestamp, app_name, window_name, document_path) \VALUES ('{DAY} {ts}', 'Code', 'main.rs', '{p}')"))@@ -1893,17 +1890,17 @@ mod db_tests {let (db, _d) = fresh_db().await;// audio_transcriptions.audio_chunk_id is a NOT NULL FK (sqlx enables// foreign_keys), so the parent chunk must exist first.- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')").await.unwrap();- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')").await.unwrap();for (m, text) in [(0, "hello team this is the weekly sync"),(1, "lets review the roadmap now"),] {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO audio_transcriptions \(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \VALUES (1, {m}, '{DAY} 10:0{m}:00', '{text}', 'mic', 1)"@@ -1925,10 +1922,10 @@ mod db_tests {#[tokio::test]async fn weekly_audio_context_is_not_consumed_by_the_last_few_minutes() {let (db, _d) = fresh_db().await;- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')").await.unwrap();- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')").await.unwrap();@@ -1936,7 +1933,7 @@ mod db_tests {// the final minutes to crowd every older row out of the old global// length sort and newest-first snippet query.for day in 1..=7 {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO audio_transcriptions \(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \VALUES (1, {day}, '2026-06-0{day} 10:00:00', \@@ -1946,7 +1943,7 @@ mod db_tests {.unwrap();}for second in 0..24 {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO audio_transcriptions \(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \VALUES (1, {}, '2026-06-07 23:59:{second:02}', \@@ -2022,15 +2019,15 @@ mod db_tests {#[tokio::test]async fn audio_context_backfills_capacity_from_one_dense_period() {let (db, _d) = fresh_db().await;- db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")+ db.execute_raw_sql_write("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')").await.unwrap();- db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")+ db.execute_raw_sql_write("INSERT INTO speakers (id, name) VALUES (1, 'Alice')").await.unwrap();for second in 0..24 {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO audio_transcriptions \(audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \VALUES (1, {second}, '2026-06-07 23:59:{second:02}', \@@ -2073,7 +2070,7 @@ mod db_tests {.await;// The frame just inserted is the only row, so its id is 1.let txt = "Quarterly planning notes for the leadership offsite";- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO elements (frame_id, source, role, text, depth, sort_order) \VALUES (1, 'accessibility', 'AXTextField', '{txt}', 0, 0)"))@@ -2100,7 +2097,7 @@ mod db_tests {let txt = format!("weekly day {day} planning notes with enough detail for summary regression");- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO elements (frame_id, source, role, text, depth, sort_order) \VALUES ({frame_id}, 'accessibility', 'AXTextField', '{}', 0, 0)",txt.replace('\'', "''")@@ -2189,7 +2186,7 @@ mod db_tests {"2026-06-03T10:00:00Z",),] {- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO memories (content, source, tags, importance, created_at, updated_at) \VALUES ('{}', 'test', '[]', 0.9, '{created_at}', '{created_at}')",content.replace('\'', "''")@@ -2250,7 +2247,7 @@ mod db_tests {));i += 1;}- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO frames (timestamp, app_name, window_name) VALUES {}",vals.join(",")))@@ -2349,7 +2346,7 @@ mod db_tests {));i += 1;}- db.execute_raw_sql(&format!(+ db.execute_raw_sql_write(&format!("INSERT INTO frames (timestamp, app_name, window_name) VALUES {}",vals.join(",")))@@ -2389,7 +2386,7 @@ mod db_tests {let (db, _d) = fresh_db().await;// Mirrors the WHERE clause shared by the apps / windows / active-ts queries.let plan = db- .execute_raw_sql(+ .query_raw_sql("EXPLAIN QUERY PLAN SELECT timestamp FROM frames \WHERE timestamp BETWEEN '2026-05-15T00:00:00Z' AND '2026-05-15T01:00:00Z' \AND app_name IS NOT NULL AND app_name != ''",diff --git a/crates/screenpipe-engine/src/routes/content.rs b/crates/screenpipe-engine/src/routes/content.rsindex 4c09889f5..aa6cd4ded 100644--- a/crates/screenpipe-engine/src/routes/content.rs+++ b/crates/screenpipe-engine/src/routes/content.rs@@ -719,7 +719,7 @@ pub(crate) async fn execute_raw_sql(return Err((StatusCode::BAD_REQUEST, JsonResponse(json!({"error": msg}))));}- match state.db.execute_raw_sql(&payload.query).await {+ match state.db.query_raw_sql(&payload.query).await {Ok(result) => Ok(JsonResponse(result)),// A database-level error means SQLite rejected the *query* itself —// unknown table/column, syntax error, etc. That's a caller mistake, notdiff --git a/packages/sdk/recorder-core/tests/sqlite_coordinator.rs b/packages/sdk/recorder-core/tests/sqlite_coordinator.rsindex 28a997300..1f8efacba 100644--- a/packages/sdk/recorder-core/tests/sqlite_coordinator.rs+++ b/packages/sdk/recorder-core/tests/sqlite_coordinator.rs@@ -48,7 +48,7 @@ async fn sdk_database_uses_safe_sqlite_and_shared_checkpoint_coordinator() {assert_eq!(busy, 0, "checkpoint reported busy");let integrity = db- .execute_raw_sql("PRAGMA integrity_check")+ .query_raw_sql("PRAGMA integrity_check").await.expect("integrity_check");let status = integrity