YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): route typed mutations through writer pool (#5737)
Louis Beaumont1c7b4bcdca7b
diff --git a/crates/screenpipe-db/src/db/audio.rs b/crates/screenpipe-db/src/db/audio.rsindex ea4155c0f..df1ddad99 100644--- a/crates/screenpipe-db/src/db/audio.rs+++ b/crates/screenpipe-db/src/db/audio.rs@@ -709,6 +709,7 @@ impl DatabaseManager {ChunkOutcome::Duplicate => "transcribed",_ => unreachable!(),};+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE audio_chunks \SET transcription_status = ?1, \@@ -720,8 +721,9 @@ impl DatabaseManager {.bind(status).bind(now).bind(audio_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -729,6 +731,7 @@ impl DatabaseManager {// Transient failure: bump attempts. If we'd hit the cap, flip// to `failed` so the sweep stops re-trying. We do this in one// UPDATE statement so a concurrent attempt can't double-flip.+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE audio_chunks \SET transcription_attempts = transcription_attempts + 1, \@@ -744,12 +747,14 @@ impl DatabaseManager {.bind(&reason).bind(MAX_TRANSCRIPTION_ATTEMPTS).bind(audio_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}ChunkOutcome::FailedPermanent { reason } => {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE audio_chunks \SET transcription_status = 'failed', \@@ -761,8 +766,9 @@ impl DatabaseManager {.bind(now).bind(&reason).bind(audio_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}}@@ -777,6 +783,7 @@ impl DatabaseManager {&self,audio_chunk_id: i64,) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE audio_chunks \SET transcription_status = 'pending', \@@ -786,8 +793,9 @@ impl DatabaseManager {WHERE id = ?1",).bind(audio_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}diff --git a/crates/screenpipe-db/src/db/frames.rs b/crates/screenpipe-db/src/db/frames.rsindex 0e24ea749..307d46ef2 100644--- a/crates/screenpipe-db/src/db/frames.rs+++ b/crates/screenpipe-db/src/db/frames.rs@@ -812,11 +812,13 @@ impl DatabaseManager {frame_id: i64,text_source: &str,) -> Result<(), anyhow::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE frames SET text_source = ?1 WHERE id = ?2").bind(text_source).bind(frame_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -1422,11 +1424,13 @@ impl DatabaseManager {// Add method to update frame namespub async fn update_frame_name(&self, frame_id: i64, name: &str) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE frames SET name = ?1 WHERE id = ?2").bind(name).bind(frame_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -1436,11 +1440,13 @@ impl DatabaseManager {video_chunk_id: i64,name: &str,) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE frames SET name = ?1 WHERE video_chunk_id = ?2").bind(name).bind(video_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}}diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rsindex b12603cf6..5e955f602 100644--- a/crates/screenpipe-db/src/db/maintenance.rs+++ b/crates/screenpipe-db/src/db/maintenance.rs@@ -1292,11 +1292,13 @@ impl DatabaseManager {chunk_id: i64,blob_id: &str,) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE video_chunks SET cloud_blob_id = ?1 WHERE id = ?2").bind(blob_id).bind(chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -1306,11 +1308,13 @@ impl DatabaseManager {frame_id: i64,blob_id: &str,) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE frames SET cloud_blob_id = ?1 WHERE id = ?2").bind(blob_id).bind(frame_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -1319,6 +1323,7 @@ impl DatabaseManager {.acquire_owned().await.map_err(|_| anyhow::anyhow!("SQLite write coordinator closed"))?;+ let mut conn = self.write_pool.acquire().await?;debug!("starting aggressive database repair process");// Force close any pending transactions@@ -1329,7 +1334,7 @@ impl DatabaseManager {];for step in emergency_steps {- if let Err(e) = sqlx::query(step).execute(&self.pool).await {+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {warn!("emergency step failed (continuing anyway): {}", e);}}@@ -1342,7 +1347,7 @@ impl DatabaseManager {];for step in wal_cleanup {- if let Err(e) = sqlx::query(step).execute(&self.pool).await {+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {warn!("wal cleanup failed (continuing anyway): {}", e);}}@@ -1361,7 +1366,7 @@ impl DatabaseManager {for (query, step) in recovery_steps {debug!("running aggressive recovery step: {}", step);- match sqlx::query(query).execute(&self.pool).await {+ match sqlx::query(query).execute(&mut *conn).await {Ok(_) => debug!("recovery step '{}' succeeded", step),Err(e) => warn!("recovery step '{}' failed: {}", step, e),}@@ -1381,14 +1386,14 @@ impl DatabaseManager {];for step in restore_steps {- if let Err(e) = sqlx::query(step).execute(&self.pool).await {+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {warn!("restore step failed: {}", e);}}// Final verificationmatch sqlx::query_scalar::<_, String>("PRAGMA quick_check;")- .fetch_one(&self.pool)+ .fetch_one(&mut *conn).await{Ok(result) if result == "ok" => {@@ -1418,7 +1423,7 @@ impl DatabaseManager {/// paths still use serialized `TRUNCATE` checkpoints when a physical reset/// is required.pub fn start_wal_maintenance(&self) {- let pool = self.pool.clone();+ let pool = self.write_pool.clone();let shutdown = self.close_token.clone();let write_queue_health = self.write_queue_health.clone();let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);@@ -1629,8 +1634,9 @@ impl DatabaseManager {if self.write_queue_health.is_hard_faulted() {return Err(SqlxError::PoolClosed);}+ let mut conn = self.write_pool.acquire().await?;let row = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")- .fetch_one(&self.pool)+ .fetch_one(&mut *conn).await?;Ok((row.get(0), row.get(1), row.get(2)))}@@ -1638,9 +1644,14 @@ impl DatabaseManager {/// Create an atomic backup of the database using `VACUUM INTO`./// The destination path must not already exist.pub async fn backup_to(&self, dest: &str) -> Result<(), sqlx::Error> {+ let _write_guard = Arc::clone(&self.write_semaphore)+ .acquire_owned()+ .await+ .map_err(|_| SqlxError::PoolClosed)?;+ let mut conn = self.write_pool.acquire().await?;sqlx::query("VACUUM INTO ?1").bind(dest)- .execute(&self.pool)+ .execute(&mut *conn).await?;Ok(())}@@ -1664,9 +1675,12 @@ impl DatabaseManager {/// insufficient disk VACUUM errors (surfaced as 500) without corrupting/// anything.pub async fn compact(&self) -> Result<(), sqlx::Error> {- let _write_guard = self.write_semaphore.acquire().await.ok();+ let _write_guard = Arc::clone(&self.write_semaphore)+ .acquire_owned()+ .await+ .map_err(|_| SqlxError::PoolClosed)?;- let mut conn = self.pool.acquire().await?;+ let mut conn = self.write_pool.acquire().await?;let _ = sqlx::query("PRAGMA busy_timeout = 60000").execute(&mut *conn).await;diff --git a/crates/screenpipe-db/src/db/semantic.rs b/crates/screenpipe-db/src/db/semantic.rsindex 78be9f520..3adc8523c 100644--- a/crates/screenpipe-db/src/db/semantic.rs+++ b/crates/screenpipe-db/src/db/semantic.rs@@ -558,11 +558,13 @@ impl DatabaseManager {/// separate until an explicit merge chooses the authoritative identity.pub async fn create_semantic_actor(&self, name: &str) -> Result<SemanticActor, sqlx::Error> {let name = validated_actor_name(name)?;+ let mut tx = self.begin_immediate_with_retry().await?;let id = sqlx::query("INSERT INTO semantic_actors (name) VALUES (?1)").bind(name)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?.last_insert_rowid();+ tx.commit().await?;self.get_semantic_actor(id).await}@@ -574,6 +576,7 @@ impl DatabaseManager {name: &str,) -> Result<SemanticActor, sqlx::Error> {let name = validated_actor_name(name)?;+ let mut tx = self.begin_immediate_with_retry().await?;let updated = sqlx::query(r#"UPDATE semantic_actorsSET name = ?1,@@ -582,11 +585,12 @@ impl DatabaseManager {).bind(name).bind(actor_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;if updated.rows_affected() != 1 {return Err(sqlx::Error::RowNotFound);}+ tx.commit().await?;self.get_semantic_actor(actor_id).await}diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex d24abbb33..e31308c4c 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -268,7 +268,7 @@ impl DatabaseManager {.await.map_err(|_| SqlxError::PoolClosed)?;match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")- .fetch_one(&db_manager.pool)+ .fetch_one(&db_manager.write_pool).await{Ok(row) => {@@ -292,16 +292,17 @@ impl DatabaseManager {warn!("startup wal checkpoint failed (continuing): {}", e);}}- drop(_checkpoint_guard);-- // Run migrations after establishing the connection- if let Err(error) = Self::run_migrations(&db_manager.pool).await {+ // Migrations mutate schema and migration bookkeeping, so keep them on+ // the same serialized writer boundary as application writes.+ if let Err(error) = Self::run_migrations(&db_manager.write_pool).await {if crate::sqlite_error::is_sqlite_hard_fault(&error) {db_manager.write_queue_health.latch_hard_fault(&error);}+ drop(_checkpoint_guard);db_manager.close().await;return Err(error);}+ drop(_checkpoint_guard);// Surface persistent-file corruption proactively at boot with a recovery// hint, instead of only discovering it later via worker query errors.diff --git a/crates/screenpipe-db/src/db/speakers.rs b/crates/screenpipe-db/src/db/speakers.rsindex 0a8b05ef5..40bb80ece 100644--- a/crates/screenpipe-db/src/db/speakers.rs+++ b/crates/screenpipe-db/src/db/speakers.rs@@ -138,11 +138,13 @@ impl DatabaseManager {embedding: &[f32],max_stored: usize,) -> Result<(), SqlxError> {+ let mut tx = self.begin_immediate_with_retry().await?;+// Count existing embeddings for this speakerlet (count,): (i64,) =sqlx::query_as("SELECT COUNT(*) FROM speaker_embeddings WHERE speaker_id = ?1").bind(speaker_id)- .fetch_one(&self.pool)+ .fetch_one(&mut **tx.conn()).await?;let bytes: &[u8] = embedding.as_bytes();@@ -154,7 +156,7 @@ impl DatabaseManager {).bind(bytes).bind(speaker_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;} else {// At capacity — replace the most redundant embedding (closest to centroid)@@ -164,7 +166,7 @@ impl DatabaseManager {let centroid_blob: Option<(Option<Vec<u8>>,)> =sqlx::query_as("SELECT centroid FROM speakers WHERE id = ?1").bind(speaker_id)- .fetch_optional(&self.pool)+ .fetch_optional(&mut **tx.conn()).await?;if let Some((Some(centroid_bytes),)) = centroid_blob {@@ -177,7 +179,7 @@ impl DatabaseManager {).bind(speaker_id).bind(¢roid_bytes[..])- .fetch_optional(&self.pool)+ .fetch_optional(&mut **tx.conn()).await?;if let Some((redundant_id,)) = most_redundant {@@ -187,7 +189,7 @@ impl DatabaseManager {).bind(bytes).bind(redundant_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;debug!("speaker {}: rotated embedding {} (closest to centroid) with new sample",@@ -197,6 +199,7 @@ impl DatabaseManager {}}+ tx.commit().await?;Ok(())}@@ -213,11 +216,13 @@ impl DatabaseManager {// keeping it responsive to voice drift over time.const MAX_EFFECTIVE_COUNT: i64 = 50;+ let mut tx = self.begin_immediate_with_retry().await?;+// Get current centroid and countlet row: Option<(Option<Vec<u8>>, i64)> =sqlx::query_as("SELECT centroid, embedding_count FROM speakers WHERE id = ?1").bind(speaker_id)- .fetch_optional(&self.pool)+ .fetch_optional(&mut **tx.conn()).await?;let (new_centroid, new_count) = match row {@@ -248,9 +253,10 @@ impl DatabaseManager {.bind(bytes).bind(new_count).bind(speaker_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -795,11 +801,13 @@ impl DatabaseManager {}pub async fn mark_speaker_as_hallucination(&self, id: i64) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE speakers SET hallucination = TRUE WHERE id = ?").bind(id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -857,11 +865,13 @@ impl DatabaseManager {embedding_id: i64,to_speaker_id: i64,) -> Result<(), sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("UPDATE speaker_embeddings SET speaker_id = ? WHERE id = ?").bind(to_speaker_id).bind(embedding_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;+ tx.commit().await?;Ok(())}@@ -871,22 +881,27 @@ impl DatabaseManager {audio_chunk_id: i64,new_speaker_id: i64,) -> Result<u64, sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;let result =sqlx::query("UPDATE audio_transcriptions SET speaker_id = ? WHERE audio_chunk_id = ?").bind(new_speaker_id).bind(audio_chunk_id)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?;- Ok(result.rows_affected())+ let rows_affected = result.rows_affected();+ tx.commit().await?;+ Ok(rows_affected)}/// Create a new speaker with a name (no embedding)pub async fn create_speaker_with_name(&self, name: &str) -> Result<Speaker, sqlx::Error> {+ let mut tx = self.begin_immediate_with_retry().await?;let id = sqlx::query("INSERT INTO speakers (name) VALUES (?)").bind(name)- .execute(&self.pool)+ .execute(&mut **tx.conn()).await?.last_insert_rowid();+ tx.commit().await?;Ok(Speaker {id,