~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(db): batch queue operations and add retry logic for SQLite lock contention (#5268)
S79a1ed34dc6e
diff --git a/changelog.d/5268.bugfix.md b/changelog.d/5268.bugfix.mdnew file mode 100644index 0000000000..45c703842a--- /dev/null+++ b/changelog.d/5268.bugfix.md@@ -0,0 +1 @@+Fixed database lock contention in SQLite queue processor v2 by batching updates into single transactions and deduplicating per-research updates.diff --git a/src/local_deep_research/web/auth/queue_middleware.py b/src/local_deep_research/web/auth/queue_middleware.pyindex 91001e5679..7da8826ea5 100644--- a/src/local_deep_research/web/auth/queue_middleware.py+++ b/src/local_deep_research/web/auth/queue_middleware.py@@ -54,7 +54,7 @@ def process_pending_queue_operations():if started_count > 0:logger.info(- f"Started {started_count} queued researches for {username}"+ f"Flushed {started_count} queued operations for {username}")except Exception:diff --git a/src/local_deep_research/web/queue/processor_v2.py b/src/local_deep_research/web/queue/processor_v2.pyindex 0cf9da02a3..e10adf2471 100644--- a/src/local_deep_research/web/queue/processor_v2.py+++ b/src/local_deep_research/web/queue/processor_v2.py@@ -699,7 +699,7 @@ class QueueProcessorV2:users_to_remove.append(user_session)except Exception:logger.exception(- f"Error processing queue for {user_session}"+ f"Error processing queue for user {username}")# Don't remove on error - the _process_user_queue method# determines whether to keep checking based on error type@@ -1478,70 +1478,164 @@ class QueueProcessorV2:db_session: Active database session for the userReturns:- Number of operations processed+ Number of operations cleared from the queue"""# Find pending operations for this user (with lock)- operations_to_process = []+ raw_operations = []with self._pending_operations_lock:for op_id, op_data in list(self.pending_operations.items()):if op_data["username"] == username:- operations_to_process.append((op_id, op_data))+ raw_operations.append((op_id, op_data))# Remove immediately to prevent duplicate processingdel self.pending_operations[op_id]- if not operations_to_process:+ if not raw_operations:return 0- processed_count = 0+ # Deduplicate and consolidate operations by research_id to minimize DB locks+ latest_progress = {}+ latest_errors = {}+ other_ops = []++ for op_id, op_data in raw_operations:+ op_type = op_data.get("operation_type")+ rid = op_data.get("research_id")+ ts = op_data.get("timestamp", 0)++ if op_type == "progress_update" and rid:+ if rid not in latest_progress or ts >= latest_progress[rid][+ 1+ ].get("timestamp", 0):+ latest_progress[rid] = (op_id, op_data)+ elif op_type == "error_update" and rid:+ if rid not in latest_errors or ts >= latest_errors[rid][1].get(+ "timestamp", 0+ ):+ latest_errors[rid] = (op_id, op_data)+ else:+ other_ops.append((op_id, op_data))++ operations_to_process = (+ list(latest_progress.values())+ + list(latest_errors.values())+ + other_ops+ )- # Process operations outside the lock (to avoid holding lock during DB operations)- for op_id, op_data in operations_to_process:- try:- operation_type = op_data.get("operation_type")+ from sqlalchemy.exc import (+ OperationalError,+ PendingRollbackError,+ TimeoutError,+ )- if operation_type == "progress_update":- # Update progress in database- from ...database.models import ResearchHistory+ from ...database.models import ResearchHistory- research = (+ max_retries = 3+ backoff = 0.05+ max_requeue_attempts = 10+ # Track the raw count of operations being flushed so the returned+ # count reflects "operations cleared from the queue" rather than+ # "deduplicated batches applied". Set after a successful commit+ # so retries don't double-count.+ processed_count = 0++ for attempt in range(max_retries):+ try:+ # Consolidate DB queries: fetch all relevant ResearchHistory rows in a single batch+ rids = list(+ {+ op_data["research_id"]+ for _, op_data in operations_to_process+ if op_data.get("research_id")+ }+ )+ if rids:+ researches = (db_session.query(ResearchHistory)- .filter_by(id=op_data["research_id"])- .first()+ .filter(ResearchHistory.id.in_(rids))+ .all())- if research:- # Update the progress column directly- research.progress = op_data["progress"]- db_session.commit()- processed_count += 1+ research_map = {r.id: r for r in researches}+ else:+ research_map = {}++ for op_id, op_data in operations_to_process:+ op_type = op_data.get("operation_type")+ rid = op_data.get("research_id")+ research = research_map.get(rid)++ if not research:+ continue++ if op_type == "progress_update":+ research.progress = op_data.get("progress")+ elif op_type == "error_update":+ research.status = op_data.get("status", "failed")+ research.error_message = op_data.get("error_message")+ research.research_meta = op_data.get("metadata")+ research.completed_at = op_data.get("completed_at")+ report_path = op_data.get("report_path")+ if report_path:+ research.report_path = report_path++ # Single commit for the entire batch of consolidated updates.+ # Count is set AFTER the commit succeeds so retries don't+ # double-count the raw operations.+ db_session.commit()+ processed_count = len(raw_operations)+ break- elif operation_type == "error_update":- # Update error status in database- from ...database.models import ResearchHistory+ except (OperationalError, PendingRollbackError, TimeoutError) as e:+ try:+ db_session.rollback()+ except Exception as rollback_err:+ logger.debug(f"Queue flush rollback failed: {rollback_err}")- research = (- db_session.query(ResearchHistory)- .filter_by(id=op_data["research_id"])- .first()+ if attempt < max_retries - 1:+ logger.debug(+ f"Database locked during queue flush for {username} (attempt {attempt + 1}/{max_retries}); retrying in {backoff * 1000:.0f}ms: {type(e).__name__}")- if research:- research.status = op_data["status"]- research.error_message = op_data["error_message"]- research.research_meta = op_data["metadata"]- research.completed_at = op_data["completed_at"]- if op_data.get("report_path"):- research.report_path = op_data["report_path"]- db_session.commit()- processed_count += 1+ time.sleep(backoff)+ backoff *= 2+ else:+ logger.warning(+ f"Failed to commit queue operations for {username} after {max_retries} retries: {type(e).__name__}"+ )+ # Re-queue raw operations so data is not lost, up to max_requeue_attempts+ with self._pending_operations_lock:+ for op_id, op_data in raw_operations:+ retry_count = op_data.get("retry_count", 0) + 1+ if retry_count > max_requeue_attempts:+ logger.exception(+ f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "+ f"after exceeding maximum requeue attempts ({max_requeue_attempts})"+ )+ elif op_id not in self.pending_operations:+ op_data["retry_count"] = retry_count+ self.pending_operations[op_id] = op_data+ processed_count = 0except Exception:- logger.exception(f"Error processing operation {op_id}")- # Rollback to clear the failed transaction state+ logger.exception(+ f"Unexpected error committing queue operations for {username}"+ )try:db_session.rollback()- except Exception:- logger.warning(- f"Failed to rollback after error in operation {op_id}"- )+ except Exception as rollback_err:+ logger.debug(f"Queue flush rollback failed: {rollback_err}")+ # Re-queue raw operations up to max_requeue_attempts+ with self._pending_operations_lock:+ for op_id, op_data in raw_operations:+ retry_count = op_data.get("retry_count", 0) + 1+ if retry_count > max_requeue_attempts:+ logger.exception(+ f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "+ f"after exceeding maximum requeue attempts ({max_requeue_attempts})"+ )+ elif op_id not in self.pending_operations:+ op_data["retry_count"] = retry_count+ self.pending_operations[op_id] = op_data+ processed_count = 0+ breakreturn processed_countdiff --git a/tests/web/queue/test_processor_v2_coverage.py b/tests/web/queue/test_processor_v2_coverage.pyindex c3ce4feef5..abe4c94936 100644--- a/tests/web/queue/test_processor_v2_coverage.py+++ b/tests/web/queue/test_processor_v2_coverage.py@@ -495,8 +495,11 @@ class TestProcessPendingOperationsEdgeCases:}mock_research = MagicMock()+ mock_research.id = "r2"mock_session = MagicMock()- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research+ mock_session.query.return_value.filter.return_value.all.return_value = [+ mock_research+ ]with patch(f"{MODULE}.logger"):count = proc.process_pending_operations_for_user(diff --git a/tests/web/queue/test_processor_v2_lock_contention.py b/tests/web/queue/test_processor_v2_lock_contention.pynew file mode 100644index 0000000000..74285f02b2--- /dev/null+++ b/tests/web/queue/test_processor_v2_lock_contention.py@@ -0,0 +1,190 @@+"""+Tests for processor_v2 handling of SQLite/SQLCipher lock contention, batching, deduplication, and re-queuing.+"""++from unittest.mock import MagicMock, patch+import pytest+from sqlalchemy.exc import OperationalError++from local_deep_research.web.queue.processor_v2 import QueueProcessorV2+from local_deep_research.database.models import ResearchHistory+++def processor():+ """Create a clean QueueProcessorV2 instance for testing."""+ return QueueProcessorV2()+++def test_process_pending_operations_deduplicates_and_batches(processor):+ """Verify that multiple progress updates for the same research_id are deduplicated before batch commit and batch queried."""+ username = "test_user"+ research_id_1 = "res-111"+ research_id_2 = "res-222"++ # Queue multiple progress updates for res-111 and res-222+ for i in range(1, 10):+ processor.queue_progress_update(username, research_id_1, float(i * 10))++ processor.queue_progress_update(username, research_id_2, 50.0)+ processor.queue_progress_update(username, research_id_2, 90.0)++ assert len(processor.pending_operations) == 11++ # Mock DB Session+ mock_session = MagicMock()+ mock_history_1 = MagicMock(spec=ResearchHistory)+ mock_history_1.id = research_id_1+ mock_history_2 = MagicMock(spec=ResearchHistory)+ mock_history_2.id = research_id_2++ filter_mock = MagicMock()+ filter_mock.all.return_value = [mock_history_1, mock_history_2]+ mock_session.query.return_value.filter.return_value = filter_mock++ processed = processor.process_pending_operations_for_user(+ username, mock_session+ )++ # All 11 original operations cleared from queue+ assert processed == 11+ assert len(processor.pending_operations) == 0++ # Verify latest progress values applied+ assert mock_history_1.progress == 90.0+ assert mock_history_2.progress == 90.0++ # Consolidated batch query called exactly once for the batch+ assert mock_session.query.call_count == 1+ # Single commit called for the batch+ assert mock_session.commit.call_count == 1+++def test_process_pending_operations_timestamp_tie_handling(processor):+ """Verify that timestamp tie (identical time.time()) gives precedence to later queued operation."""+ username = "test_user"+ research_id = "res-tie"++ with patch("time.time", return_value=1000.0):+ processor.queue_progress_update(username, research_id, 10.0)+ processor.queue_progress_update(username, research_id, 90.0)++ mock_session = MagicMock()+ mock_history = MagicMock(spec=ResearchHistory)+ mock_history.id = research_id++ filter_mock = MagicMock()+ filter_mock.all.return_value = [mock_history]+ mock_session.query.return_value.filter.return_value = filter_mock++ processed = processor.process_pending_operations_for_user(+ username, mock_session+ )++ assert processed == 2+ assert mock_history.progress == 90.0+++def test_process_pending_operations_retries_on_operational_error(processor):+ """Verify that transient database lock (OperationalError) triggers retry and succeeds."""+ username = "test_user"+ research_id = "res-lock-test"++ processor.queue_progress_update(username, research_id, 75.0)++ mock_session = MagicMock()+ mock_history = MagicMock(spec=ResearchHistory)+ mock_history.id = research_id++ filter_mock = MagicMock()+ filter_mock.all.return_value = [mock_history]+ mock_session.query.return_value.filter.return_value = filter_mock++ # Fail commit on attempt 1 with OperationalError("database is locked"), succeed on attempt 2+ mock_session.commit.side_effect = [+ OperationalError(+ "UPDATE research_history", {}, Exception("database is locked")+ ),+ None,+ ]++ processed = processor.process_pending_operations_for_user(+ username, mock_session+ )++ assert processed == 1+ assert len(processor.pending_operations) == 0+ assert mock_session.commit.call_count == 2+ assert mock_session.rollback.call_count == 1+++def test_process_pending_operations_requeues_on_persistent_lock(processor):+ """Verify that operations are re-queued with incremented retry_count if all commit retries fail."""+ username = "test_user"+ research_id = "res-fail-test"++ processor.queue_progress_update(username, research_id, 33.0)+ assert len(processor.pending_operations) == 1++ mock_session = MagicMock()+ mock_history = MagicMock(spec=ResearchHistory)+ mock_history.id = research_id++ filter_mock = MagicMock()+ filter_mock.all.return_value = [mock_history]+ mock_session.query.return_value.filter.return_value = filter_mock++ # Fail commit on all 3 attempts+ mock_session.commit.side_effect = OperationalError(+ "UPDATE research_history", {}, Exception("database is locked")+ )++ processed = processor.process_pending_operations_for_user(+ username, mock_session+ )++ # 0 processed successfully, operations returned to queue with retry_count set+ assert processed == 0+ assert len(processor.pending_operations) == 1+ requeued_op = next(iter(processor.pending_operations.values()))+ assert requeued_op.get("retry_count") == 1+ assert mock_session.commit.call_count == 3+ assert mock_session.rollback.call_count == 3+++def test_process_pending_operations_requeue_exceeding_max_attempts_drops_op(+ processor,+):+ """Verify that operations exceeding max requeue attempts are dropped (dead-lettered)."""+ username = "test_user"+ research_id = "res-deadletter-test"++ processor.pending_operations["op-dead"] = {+ "username": username,+ "operation_type": "progress_update",+ "research_id": research_id,+ "progress": 50.0,+ "timestamp": 1000.0,+ "retry_count": 10, # Already reached max requeue attempts threshold+ }++ mock_session = MagicMock()+ mock_history = MagicMock(spec=ResearchHistory)+ mock_history.id = research_id++ filter_mock = MagicMock()+ filter_mock.all.return_value = [mock_history]+ mock_session.query.return_value.filter.return_value = filter_mock++ # Fail commit on all retries+ mock_session.commit.side_effect = OperationalError(+ "UPDATE research_history", {}, Exception("database is locked")+ )++ processed = processor.process_pending_operations_for_user(+ username, mock_session+ )++ assert processed == 0+ # Operation should be dropped after exceeding max attempts+ assert len(processor.pending_operations) == 0diff --git a/tests/web/queue/test_processor_v2_real.py b/tests/web/queue/test_processor_v2_real.pyindex 81122d1fb8..c229ddc9c5 100644--- a/tests/web/queue/test_processor_v2_real.py+++ b/tests/web/queue/test_processor_v2_real.py@@ -948,7 +948,10 @@ class TestProcessPendingOperationsForUser:mock_session = MagicMock()mock_research = Mock()- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research+ mock_research.id = "r-200"+ mock_session.query.return_value.filter.return_value.all.return_value = [+ mock_research+ ]count = processor.process_pending_operations_for_user("alice", mock_session@@ -974,7 +977,10 @@ class TestProcessPendingOperationsForUser:mock_session = MagicMock()mock_research = Mock()- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research+ mock_research.id = "r-201"+ mock_session.query.return_value.filter.return_value.all.return_value = [+ mock_research+ ]count = processor.process_pending_operations_for_user("alice", mock_session@@ -1007,7 +1013,10 @@ class TestProcessPendingOperationsForUser:mock_session = MagicMock()mock_research = Mock()- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research+ mock_research.id = "r-301"+ mock_session.query.return_value.filter.return_value.all.return_value = [+ mock_research+ ]count = processor.process_pending_operations_for_user("alice", mock_session@@ -1050,6 +1059,7 @@ class TestProcessPendingOperationsForUser:mock_session = MagicMock()mock_research = Mock(spec=[+ "id","status","error_message","research_meta",@@ -1057,7 +1067,10 @@ class TestProcessPendingOperationsForUser:"report_path",])- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research+ mock_research.id = "r-401"+ mock_session.query.return_value.filter.return_value.all.return_value = [+ mock_research+ ]count = processor.process_pending_operations_for_user("alice", mock_session