Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
fix: generate embeddings before chunk-swap transaction to stop SQLite writer-lock starvation during reindex (#355)
Murtaza Nasircbbcb4e5e371
diff --git a/src/services/embeddings.py b/src/services/embeddings.pyindex af23bc95..0422aef4 100644--- a/src/services/embeddings.py+++ b/src/services/embeddings.py@@ -360,26 +360,26 @@ def process_recording_chunks(recording_id):Returns True on full success, False on any failure including the casewhere embedding generation returned fewer vectors than there were- chunks. On failure the transaction is rolled back so the recording's- existing chunks are preserved; this prevents the "old chunks deleted,- new chunks not inserted" silent-failure mode that occurs when the- embedding API blips mid-run.+ chunks. Embeddings are generated before the old chunks are deleted, so+ a failure leaves the recording's existing chunks intact and never holds+ a database write lock across the (potentially slow) embedding API call."""try:recording = db.session.get(Recording, recording_id)if not recording or not recording.transcription:return False- # Delete existing chunks for this recording. The deletion is staged- # in this transaction; if anything below fails we rollback and the- # old chunks survive. Only the final commit makes the swap- # permanent.- TranscriptChunk.query.filter_by(recording_id=recording_id).delete()-- # Create chunks+ # Create chunks and generate their embeddings BEFORE touching the+ # transcript_chunk table. The embeddings call is a network round-trip+ # that can take 30+ seconds against a cold local model; running it+ # inside an open write transaction held SQLite's single writer lock+ # for the whole duration and starved concurrent writers such as the+ # summarize-job enqueue (issue #355). Doing the slow work first keeps+ # the delete + insert + commit window down to milliseconds.chunks = chunk_transcription(recording.transcription)if not chunks:+ TranscriptChunk.query.filter_by(recording_id=recording_id).delete()db.session.commit()return True@@ -388,19 +388,21 @@ def process_recording_chunks(recording_id):# Verify we got one embedding per chunk. _api_embed returns [] on# exhausted retries, and a partial provider response could return- # fewer than expected. Either case is a failure: rolling back keeps- # the recording's old chunks intact so the admin retry pass (or a- # later Re-embed all) can try again.+ # fewer than expected. Either case is a failure; the recording's+ # existing chunks are untouched so the admin retry pass (or a later+ # Re-embed all) can try again.if len(embeddings) != len(chunks):- db.session.rollback()current_app.logger.error(f"Embedding generation returned {len(embeddings)} vectors for "- f"{len(chunks)} chunks on recording {recording_id}; rolling "- f"back to preserve existing chunks."+ f"{len(chunks)} chunks on recording {recording_id}; keeping "+ f"existing chunks.")return False- # Store chunks in database+ # Swap the chunks in one short transaction: the delete is staged and+ # only the final commit makes it permanent, so a failure here still+ # rolls back to the old chunks.+ TranscriptChunk.query.filter_by(recording_id=recording_id).delete()for i, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):chunk = TranscriptChunk(recording_id=recording_id,diff --git a/tests/test_cov_embeddings.py b/tests/test_cov_embeddings.pyindex 987a9b89..cf1f46f2 100644--- a/tests/test_cov_embeddings.py+++ b/tests/test_cov_embeddings.py@@ -628,6 +628,30 @@ def test_process_partial_embedding_count_rolls_back(api_emb):assert TranscriptChunk.query.filter_by(recording_id=rec.id).count() == 0+def test_process_embeds_before_deleting_old_chunks(api_emb):+ # Regression for issue #355: the embeddings call is a slow network+ # round-trip and must run BEFORE the old chunks are deleted, so no+ # write transaction (and SQLite writer lock) is held across it. The+ # old chunks must therefore still be visible at embedding time.+ with app.app_context():+ user = _make_user()+ rec = _make_recording(user, transcription="original text to index")+ with patch.object(api_emb, "generate_embeddings",+ side_effect=lambda texts, user_id=None: _fixed_vectors(texts)):+ assert api_emb.process_recording_chunks(rec.id) is True++ seen = {}++ def spy(texts, user_id=None):+ seen["chunks_at_embed_time"] = TranscriptChunk.query.filter_by(+ recording_id=rec.id).count()+ return _fixed_vectors(texts)++ with patch.object(api_emb, "generate_embeddings", side_effect=spy):+ assert api_emb.process_recording_chunks(rec.id) is True+ assert seen["chunks_at_embed_time"] == 1++def test_process_exception_path_returns_false(api_emb):with app.app_context():user = _make_user()