Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): route typed mutations through writer pool (#5737)

Louis Beaumont1c7b4bcdca7b
diff --git a/crates/screenpipe-db/src/db/audio.rs b/crates/screenpipe-db/src/db/audio.rs
index ea4155c0f..df1ddad99 100644
--- a/crates/screenpipe-db/src/db/audio.rs
+++ b/crates/screenpipe-db/src/db/audio.rs
@@ -709,6 +709,7 @@ impl DatabaseManager {
ChunkOutcome::Duplicate => "transcribed",
_ => unreachable!(),
};
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query(
"UPDATE audio_chunks \
SET transcription_status = ?1, \
@@ -720,8 +721,9 @@ impl DatabaseManager {
.bind(status)
.bind(now)
.bind(audio_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -729,6 +731,7 @@ impl DatabaseManager {
// Transient failure: bump attempts. If we'd hit the cap, flip
// to `failed` so the sweep stops re-trying. We do this in one
// UPDATE statement so a concurrent attempt can't double-flip.
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query(
"UPDATE audio_chunks \
SET transcription_attempts = transcription_attempts + 1, \
@@ -744,12 +747,14 @@ impl DatabaseManager {
.bind(&reason)
.bind(MAX_TRANSCRIPTION_ATTEMPTS)
.bind(audio_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
ChunkOutcome::FailedPermanent { reason } => {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query(
"UPDATE audio_chunks \
SET transcription_status = 'failed', \
@@ -761,8 +766,9 @@ impl DatabaseManager {
.bind(now)
.bind(&reason)
.bind(audio_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
}
@@ -777,6 +783,7 @@ impl DatabaseManager {
&self,
audio_chunk_id: i64,
) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query(
"UPDATE audio_chunks \
SET transcription_status = 'pending', \
@@ -786,8 +793,9 @@ impl DatabaseManager {
WHERE id = ?1",
)
.bind(audio_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
diff --git a/crates/screenpipe-db/src/db/frames.rs b/crates/screenpipe-db/src/db/frames.rs
index 0e24ea749..307d46ef2 100644
--- a/crates/screenpipe-db/src/db/frames.rs
+++ b/crates/screenpipe-db/src/db/frames.rs
@@ -812,11 +812,13 @@ impl DatabaseManager {
frame_id: i64,
text_source: &str,
) -> Result<(), anyhow::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE frames SET text_source = ?1 WHERE id = ?2")
.bind(text_source)
.bind(frame_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -1422,11 +1424,13 @@ impl DatabaseManager {
// Add method to update frame names
pub async fn update_frame_name(&self, frame_id: i64, name: &str) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE frames SET name = ?1 WHERE id = ?2")
.bind(name)
.bind(frame_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -1436,11 +1440,13 @@ impl DatabaseManager {
video_chunk_id: i64,
name: &str,
) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE frames SET name = ?1 WHERE video_chunk_id = ?2")
.bind(name)
.bind(video_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
}
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index b12603cf6..5e955f602 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -1292,11 +1292,13 @@ impl DatabaseManager {
chunk_id: i64,
blob_id: &str,
) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE video_chunks SET cloud_blob_id = ?1 WHERE id = ?2")
.bind(blob_id)
.bind(chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -1306,11 +1308,13 @@ impl DatabaseManager {
frame_id: i64,
blob_id: &str,
) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE frames SET cloud_blob_id = ?1 WHERE id = ?2")
.bind(blob_id)
.bind(frame_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -1319,6 +1323,7 @@ impl DatabaseManager {
.acquire_owned()
.await
.map_err(|_| anyhow::anyhow!("SQLite write coordinator closed"))?;
+ let mut conn = self.write_pool.acquire().await?;
debug!("starting aggressive database repair process");
// Force close any pending transactions
@@ -1329,7 +1334,7 @@ impl DatabaseManager {
];
for step in emergency_steps {
- if let Err(e) = sqlx::query(step).execute(&self.pool).await {
+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {
warn!("emergency step failed (continuing anyway): {}", e);
}
}
@@ -1342,7 +1347,7 @@ impl DatabaseManager {
];
for step in wal_cleanup {
- if let Err(e) = sqlx::query(step).execute(&self.pool).await {
+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {
warn!("wal cleanup failed (continuing anyway): {}", e);
}
}
@@ -1361,7 +1366,7 @@ impl DatabaseManager {
for (query, step) in recovery_steps {
debug!("running aggressive recovery step: {}", step);
- match sqlx::query(query).execute(&self.pool).await {
+ match sqlx::query(query).execute(&mut *conn).await {
Ok(_) => debug!("recovery step '{}' succeeded", step),
Err(e) => warn!("recovery step '{}' failed: {}", step, e),
}
@@ -1381,14 +1386,14 @@ impl DatabaseManager {
];
for step in restore_steps {
- if let Err(e) = sqlx::query(step).execute(&self.pool).await {
+ if let Err(e) = sqlx::query(step).execute(&mut *conn).await {
warn!("restore step failed: {}", e);
}
}
// Final verification
match sqlx::query_scalar::<_, String>("PRAGMA quick_check;")
- .fetch_one(&self.pool)
+ .fetch_one(&mut *conn)
.await
{
Ok(result) if result == "ok" => {
@@ -1418,7 +1423,7 @@ impl DatabaseManager {
/// paths still use serialized `TRUNCATE` checkpoints when a physical reset
/// is required.
pub fn start_wal_maintenance(&self) {
- let pool = self.pool.clone();
+ let pool = self.write_pool.clone();
let shutdown = self.close_token.clone();
let write_queue_health = self.write_queue_health.clone();
let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);
@@ -1629,8 +1634,9 @@ impl DatabaseManager {
if self.write_queue_health.is_hard_faulted() {
return Err(SqlxError::PoolClosed);
}
+ let mut conn = self.write_pool.acquire().await?;
let row = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
- .fetch_one(&self.pool)
+ .fetch_one(&mut *conn)
.await?;
Ok((row.get(0), row.get(1), row.get(2)))
}
@@ -1638,9 +1644,14 @@ impl DatabaseManager {
/// Create an atomic backup of the database using `VACUUM INTO`.
/// The destination path must not already exist.
pub async fn backup_to(&self, dest: &str) -> Result<(), sqlx::Error> {
+ let _write_guard = Arc::clone(&self.write_semaphore)
+ .acquire_owned()
+ .await
+ .map_err(|_| SqlxError::PoolClosed)?;
+ let mut conn = self.write_pool.acquire().await?;
sqlx::query("VACUUM INTO ?1")
.bind(dest)
- .execute(&self.pool)
+ .execute(&mut *conn)
.await?;
Ok(())
}
@@ -1664,9 +1675,12 @@ impl DatabaseManager {
/// insufficient disk VACUUM errors (surfaced as 500) without corrupting
/// anything.
pub async fn compact(&self) -> Result<(), sqlx::Error> {
- let _write_guard = self.write_semaphore.acquire().await.ok();
+ let _write_guard = Arc::clone(&self.write_semaphore)
+ .acquire_owned()
+ .await
+ .map_err(|_| SqlxError::PoolClosed)?;
- let mut conn = self.pool.acquire().await?;
+ let mut conn = self.write_pool.acquire().await?;
let _ = sqlx::query("PRAGMA busy_timeout = 60000")
.execute(&mut *conn)
.await;
diff --git a/crates/screenpipe-db/src/db/semantic.rs b/crates/screenpipe-db/src/db/semantic.rs
index 78be9f520..3adc8523c 100644
--- a/crates/screenpipe-db/src/db/semantic.rs
+++ b/crates/screenpipe-db/src/db/semantic.rs
@@ -558,11 +558,13 @@ impl DatabaseManager {
/// separate until an explicit merge chooses the authoritative identity.
pub async fn create_semantic_actor(&self, name: &str) -> Result<SemanticActor, sqlx::Error> {
let name = validated_actor_name(name)?;
+ let mut tx = self.begin_immediate_with_retry().await?;
let id = sqlx::query("INSERT INTO semantic_actors (name) VALUES (?1)")
.bind(name)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?
.last_insert_rowid();
+ tx.commit().await?;
self.get_semantic_actor(id).await
}
@@ -574,6 +576,7 @@ impl DatabaseManager {
name: &str,
) -> Result<SemanticActor, sqlx::Error> {
let name = validated_actor_name(name)?;
+ let mut tx = self.begin_immediate_with_retry().await?;
let updated = sqlx::query(
r#"UPDATE semantic_actors
SET name = ?1,
@@ -582,11 +585,12 @@ impl DatabaseManager {
)
.bind(name)
.bind(actor_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
if updated.rows_affected() != 1 {
return Err(sqlx::Error::RowNotFound);
}
+ tx.commit().await?;
self.get_semantic_actor(actor_id).await
}
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index d24abbb33..e31308c4c 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -268,7 +268,7 @@ impl DatabaseManager {
.await
.map_err(|_| SqlxError::PoolClosed)?;
match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
- .fetch_one(&db_manager.pool)
+ .fetch_one(&db_manager.write_pool)
.await
{
Ok(row) => {
@@ -292,16 +292,17 @@ impl DatabaseManager {
warn!("startup wal checkpoint failed (continuing): {}", e);
}
}
- drop(_checkpoint_guard);
-
- // Run migrations after establishing the connection
- if let Err(error) = Self::run_migrations(&db_manager.pool).await {
+ // Migrations mutate schema and migration bookkeeping, so keep them on
+ // the same serialized writer boundary as application writes.
+ if let Err(error) = Self::run_migrations(&db_manager.write_pool).await {
if crate::sqlite_error::is_sqlite_hard_fault(&error) {
db_manager.write_queue_health.latch_hard_fault(&error);
}
+ drop(_checkpoint_guard);
db_manager.close().await;
return Err(error);
}
+ drop(_checkpoint_guard);
// Surface persistent-file corruption proactively at boot with a recovery
// hint, instead of only discovering it later via worker query errors.
diff --git a/crates/screenpipe-db/src/db/speakers.rs b/crates/screenpipe-db/src/db/speakers.rs
index 0a8b05ef5..40bb80ece 100644
--- a/crates/screenpipe-db/src/db/speakers.rs
+++ b/crates/screenpipe-db/src/db/speakers.rs
@@ -138,11 +138,13 @@ impl DatabaseManager {
embedding: &[f32],
max_stored: usize,
) -> Result<(), SqlxError> {
+ let mut tx = self.begin_immediate_with_retry().await?;
+
// Count existing embeddings for this speaker
let (count,): (i64,) =
sqlx::query_as("SELECT COUNT(*) FROM speaker_embeddings WHERE speaker_id = ?1")
.bind(speaker_id)
- .fetch_one(&self.pool)
+ .fetch_one(&mut **tx.conn())
.await?;
let bytes: &[u8] = embedding.as_bytes();
@@ -154,7 +156,7 @@ impl DatabaseManager {
)
.bind(bytes)
.bind(speaker_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
} else {
// At capacity — replace the most redundant embedding (closest to centroid)
@@ -164,7 +166,7 @@ impl DatabaseManager {
let centroid_blob: Option<(Option<Vec<u8>>,)> =
sqlx::query_as("SELECT centroid FROM speakers WHERE id = ?1")
.bind(speaker_id)
- .fetch_optional(&self.pool)
+ .fetch_optional(&mut **tx.conn())
.await?;
if let Some((Some(centroid_bytes),)) = centroid_blob {
@@ -177,7 +179,7 @@ impl DatabaseManager {
)
.bind(speaker_id)
.bind(&centroid_bytes[..])
- .fetch_optional(&self.pool)
+ .fetch_optional(&mut **tx.conn())
.await?;
if let Some((redundant_id,)) = most_redundant {
@@ -187,7 +189,7 @@ impl DatabaseManager {
)
.bind(bytes)
.bind(redundant_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
debug!(
"speaker {}: rotated embedding {} (closest to centroid) with new sample",
@@ -197,6 +199,7 @@ impl DatabaseManager {
}
}
+ tx.commit().await?;
Ok(())
}
@@ -213,11 +216,13 @@ impl DatabaseManager {
// keeping it responsive to voice drift over time.
const MAX_EFFECTIVE_COUNT: i64 = 50;
+ let mut tx = self.begin_immediate_with_retry().await?;
+
// Get current centroid and count
let row: Option<(Option<Vec<u8>>, i64)> =
sqlx::query_as("SELECT centroid, embedding_count FROM speakers WHERE id = ?1")
.bind(speaker_id)
- .fetch_optional(&self.pool)
+ .fetch_optional(&mut **tx.conn())
.await?;
let (new_centroid, new_count) = match row {
@@ -248,9 +253,10 @@ impl DatabaseManager {
.bind(bytes)
.bind(new_count)
.bind(speaker_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -795,11 +801,13 @@ impl DatabaseManager {
}
pub async fn mark_speaker_as_hallucination(&self, id: i64) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE speakers SET hallucination = TRUE WHERE id = ?")
.bind(id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -857,11 +865,13 @@ impl DatabaseManager {
embedding_id: i64,
to_speaker_id: i64,
) -> Result<(), sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
sqlx::query("UPDATE speaker_embeddings SET speaker_id = ? WHERE id = ?")
.bind(to_speaker_id)
.bind(embedding_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -871,22 +881,27 @@ impl DatabaseManager {
audio_chunk_id: i64,
new_speaker_id: i64,
) -> Result<u64, sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
let result =
sqlx::query("UPDATE audio_transcriptions SET speaker_id = ? WHERE audio_chunk_id = ?")
.bind(new_speaker_id)
.bind(audio_chunk_id)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?;
- Ok(result.rows_affected())
+ let rows_affected = result.rows_affected();
+ tx.commit().await?;
+ Ok(rows_affected)
}
/// Create a new speaker with a name (no embedding)
pub async fn create_speaker_with_name(&self, name: &str) -> Result<Speaker, sqlx::Error> {
+ let mut tx = self.begin_immediate_with_retry().await?;
let id = sqlx::query("INSERT INTO speakers (name) VALUES (?)")
.bind(name)
- .execute(&self.pool)
+ .execute(&mut **tx.conn())
.await?
.last_insert_rowid();
+ tx.commit().await?;
Ok(Speaker {
id,