Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(db): batch queue operations and add retry logic for SQLite lock contention (#5268)

S79a1ed34dc6e
diff --git a/changelog.d/5268.bugfix.md b/changelog.d/5268.bugfix.md
new file mode 100644
index 0000000000..45c703842a
--- /dev/null
+++ b/changelog.d/5268.bugfix.md
@@ -0,0 +1 @@
+Fixed database lock contention in SQLite queue processor v2 by batching updates into single transactions and deduplicating per-research updates.
diff --git a/src/local_deep_research/web/auth/queue_middleware.py b/src/local_deep_research/web/auth/queue_middleware.py
index 91001e5679..7da8826ea5 100644
--- a/src/local_deep_research/web/auth/queue_middleware.py
+++ b/src/local_deep_research/web/auth/queue_middleware.py
@@ -54,7 +54,7 @@ def process_pending_queue_operations():
if started_count > 0:
logger.info(
- f"Started {started_count} queued researches for {username}"
+ f"Flushed {started_count} queued operations for {username}"
)
except Exception:
diff --git a/src/local_deep_research/web/queue/processor_v2.py b/src/local_deep_research/web/queue/processor_v2.py
index 0cf9da02a3..e10adf2471 100644
--- a/src/local_deep_research/web/queue/processor_v2.py
+++ b/src/local_deep_research/web/queue/processor_v2.py
@@ -699,7 +699,7 @@ class QueueProcessorV2:
users_to_remove.append(user_session)
except Exception:
logger.exception(
- f"Error processing queue for {user_session}"
+ f"Error processing queue for user {username}"
)
# Don't remove on error - the _process_user_queue method
# determines whether to keep checking based on error type
@@ -1478,70 +1478,164 @@ class QueueProcessorV2:
db_session: Active database session for the user
Returns:
- Number of operations processed
+ Number of operations cleared from the queue
"""
# Find pending operations for this user (with lock)
- operations_to_process = []
+ raw_operations = []
with self._pending_operations_lock:
for op_id, op_data in list(self.pending_operations.items()):
if op_data["username"] == username:
- operations_to_process.append((op_id, op_data))
+ raw_operations.append((op_id, op_data))
# Remove immediately to prevent duplicate processing
del self.pending_operations[op_id]
- if not operations_to_process:
+ if not raw_operations:
return 0
- processed_count = 0
+ # Deduplicate and consolidate operations by research_id to minimize DB locks
+ latest_progress = {}
+ latest_errors = {}
+ other_ops = []
+
+ for op_id, op_data in raw_operations:
+ op_type = op_data.get("operation_type")
+ rid = op_data.get("research_id")
+ ts = op_data.get("timestamp", 0)
+
+ if op_type == "progress_update" and rid:
+ if rid not in latest_progress or ts >= latest_progress[rid][
+ 1
+ ].get("timestamp", 0):
+ latest_progress[rid] = (op_id, op_data)
+ elif op_type == "error_update" and rid:
+ if rid not in latest_errors or ts >= latest_errors[rid][1].get(
+ "timestamp", 0
+ ):
+ latest_errors[rid] = (op_id, op_data)
+ else:
+ other_ops.append((op_id, op_data))
+
+ operations_to_process = (
+ list(latest_progress.values())
+ + list(latest_errors.values())
+ + other_ops
+ )
- # Process operations outside the lock (to avoid holding lock during DB operations)
- for op_id, op_data in operations_to_process:
- try:
- operation_type = op_data.get("operation_type")
+ from sqlalchemy.exc import (
+ OperationalError,
+ PendingRollbackError,
+ TimeoutError,
+ )
- if operation_type == "progress_update":
- # Update progress in database
- from ...database.models import ResearchHistory
+ from ...database.models import ResearchHistory
- research = (
+ max_retries = 3
+ backoff = 0.05
+ max_requeue_attempts = 10
+ # Track the raw count of operations being flushed so the returned
+ # count reflects "operations cleared from the queue" rather than
+ # "deduplicated batches applied". Set after a successful commit
+ # so retries don't double-count.
+ processed_count = 0
+
+ for attempt in range(max_retries):
+ try:
+ # Consolidate DB queries: fetch all relevant ResearchHistory rows in a single batch
+ rids = list(
+ {
+ op_data["research_id"]
+ for _, op_data in operations_to_process
+ if op_data.get("research_id")
+ }
+ )
+ if rids:
+ researches = (
db_session.query(ResearchHistory)
- .filter_by(id=op_data["research_id"])
- .first()
+ .filter(ResearchHistory.id.in_(rids))
+ .all()
)
- if research:
- # Update the progress column directly
- research.progress = op_data["progress"]
- db_session.commit()
- processed_count += 1
+ research_map = {r.id: r for r in researches}
+ else:
+ research_map = {}
+
+ for op_id, op_data in operations_to_process:
+ op_type = op_data.get("operation_type")
+ rid = op_data.get("research_id")
+ research = research_map.get(rid)
+
+ if not research:
+ continue
+
+ if op_type == "progress_update":
+ research.progress = op_data.get("progress")
+ elif op_type == "error_update":
+ research.status = op_data.get("status", "failed")
+ research.error_message = op_data.get("error_message")
+ research.research_meta = op_data.get("metadata")
+ research.completed_at = op_data.get("completed_at")
+ report_path = op_data.get("report_path")
+ if report_path:
+ research.report_path = report_path
+
+ # Single commit for the entire batch of consolidated updates.
+ # Count is set AFTER the commit succeeds so retries don't
+ # double-count the raw operations.
+ db_session.commit()
+ processed_count = len(raw_operations)
+ break
- elif operation_type == "error_update":
- # Update error status in database
- from ...database.models import ResearchHistory
+ except (OperationalError, PendingRollbackError, TimeoutError) as e:
+ try:
+ db_session.rollback()
+ except Exception as rollback_err:
+ logger.debug(f"Queue flush rollback failed: {rollback_err}")
- research = (
- db_session.query(ResearchHistory)
- .filter_by(id=op_data["research_id"])
- .first()
+ if attempt < max_retries - 1:
+ logger.debug(
+ f"Database locked during queue flush for {username} (attempt {attempt + 1}/{max_retries}); retrying in {backoff * 1000:.0f}ms: {type(e).__name__}"
)
- if research:
- research.status = op_data["status"]
- research.error_message = op_data["error_message"]
- research.research_meta = op_data["metadata"]
- research.completed_at = op_data["completed_at"]
- if op_data.get("report_path"):
- research.report_path = op_data["report_path"]
- db_session.commit()
- processed_count += 1
+ time.sleep(backoff)
+ backoff *= 2
+ else:
+ logger.warning(
+ f"Failed to commit queue operations for {username} after {max_retries} retries: {type(e).__name__}"
+ )
+ # Re-queue raw operations so data is not lost, up to max_requeue_attempts
+ with self._pending_operations_lock:
+ for op_id, op_data in raw_operations:
+ retry_count = op_data.get("retry_count", 0) + 1
+ if retry_count > max_requeue_attempts:
+ logger.exception(
+ f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "
+ f"after exceeding maximum requeue attempts ({max_requeue_attempts})"
+ )
+ elif op_id not in self.pending_operations:
+ op_data["retry_count"] = retry_count
+ self.pending_operations[op_id] = op_data
+ processed_count = 0
except Exception:
- logger.exception(f"Error processing operation {op_id}")
- # Rollback to clear the failed transaction state
+ logger.exception(
+ f"Unexpected error committing queue operations for {username}"
+ )
try:
db_session.rollback()
- except Exception:
- logger.warning(
- f"Failed to rollback after error in operation {op_id}"
- )
+ except Exception as rollback_err:
+ logger.debug(f"Queue flush rollback failed: {rollback_err}")
+ # Re-queue raw operations up to max_requeue_attempts
+ with self._pending_operations_lock:
+ for op_id, op_data in raw_operations:
+ retry_count = op_data.get("retry_count", 0) + 1
+ if retry_count > max_requeue_attempts:
+ logger.exception(
+ f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "
+ f"after exceeding maximum requeue attempts ({max_requeue_attempts})"
+ )
+ elif op_id not in self.pending_operations:
+ op_data["retry_count"] = retry_count
+ self.pending_operations[op_id] = op_data
+ processed_count = 0
+ break
return processed_count
diff --git a/tests/web/queue/test_processor_v2_coverage.py b/tests/web/queue/test_processor_v2_coverage.py
index c3ce4feef5..abe4c94936 100644
--- a/tests/web/queue/test_processor_v2_coverage.py
+++ b/tests/web/queue/test_processor_v2_coverage.py
@@ -495,8 +495,11 @@ class TestProcessPendingOperationsEdgeCases:
}
mock_research = MagicMock()
+ mock_research.id = "r2"
mock_session = MagicMock()
- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ mock_research
+ ]
with patch(f"{MODULE}.logger"):
count = proc.process_pending_operations_for_user(
diff --git a/tests/web/queue/test_processor_v2_lock_contention.py b/tests/web/queue/test_processor_v2_lock_contention.py
new file mode 100644
index 0000000000..74285f02b2
--- /dev/null
+++ b/tests/web/queue/test_processor_v2_lock_contention.py
@@ -0,0 +1,190 @@
+"""
+Tests for processor_v2 handling of SQLite/SQLCipher lock contention, batching, deduplication, and re-queuing.
+"""
+
+from unittest.mock import MagicMock, patch
+import pytest
+from sqlalchemy.exc import OperationalError
+
+from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
+from local_deep_research.database.models import ResearchHistory
+
+
+def processor():
+ """Create a clean QueueProcessorV2 instance for testing."""
+ return QueueProcessorV2()
+
+
+def test_process_pending_operations_deduplicates_and_batches(processor):
+ """Verify that multiple progress updates for the same research_id are deduplicated before batch commit and batch queried."""
+ username = "test_user"
+ research_id_1 = "res-111"
+ research_id_2 = "res-222"
+
+ # Queue multiple progress updates for res-111 and res-222
+ for i in range(1, 10):
+ processor.queue_progress_update(username, research_id_1, float(i * 10))
+
+ processor.queue_progress_update(username, research_id_2, 50.0)
+ processor.queue_progress_update(username, research_id_2, 90.0)
+
+ assert len(processor.pending_operations) == 11
+
+ # Mock DB Session
+ mock_session = MagicMock()
+ mock_history_1 = MagicMock(spec=ResearchHistory)
+ mock_history_1.id = research_id_1
+ mock_history_2 = MagicMock(spec=ResearchHistory)
+ mock_history_2.id = research_id_2
+
+ filter_mock = MagicMock()
+ filter_mock.all.return_value = [mock_history_1, mock_history_2]
+ mock_session.query.return_value.filter.return_value = filter_mock
+
+ processed = processor.process_pending_operations_for_user(
+ username, mock_session
+ )
+
+ # All 11 original operations cleared from queue
+ assert processed == 11
+ assert len(processor.pending_operations) == 0
+
+ # Verify latest progress values applied
+ assert mock_history_1.progress == 90.0
+ assert mock_history_2.progress == 90.0
+
+ # Consolidated batch query called exactly once for the batch
+ assert mock_session.query.call_count == 1
+ # Single commit called for the batch
+ assert mock_session.commit.call_count == 1
+
+
+def test_process_pending_operations_timestamp_tie_handling(processor):
+ """Verify that timestamp tie (identical time.time()) gives precedence to later queued operation."""
+ username = "test_user"
+ research_id = "res-tie"
+
+ with patch("time.time", return_value=1000.0):
+ processor.queue_progress_update(username, research_id, 10.0)
+ processor.queue_progress_update(username, research_id, 90.0)
+
+ mock_session = MagicMock()
+ mock_history = MagicMock(spec=ResearchHistory)
+ mock_history.id = research_id
+
+ filter_mock = MagicMock()
+ filter_mock.all.return_value = [mock_history]
+ mock_session.query.return_value.filter.return_value = filter_mock
+
+ processed = processor.process_pending_operations_for_user(
+ username, mock_session
+ )
+
+ assert processed == 2
+ assert mock_history.progress == 90.0
+
+
+def test_process_pending_operations_retries_on_operational_error(processor):
+ """Verify that transient database lock (OperationalError) triggers retry and succeeds."""
+ username = "test_user"
+ research_id = "res-lock-test"
+
+ processor.queue_progress_update(username, research_id, 75.0)
+
+ mock_session = MagicMock()
+ mock_history = MagicMock(spec=ResearchHistory)
+ mock_history.id = research_id
+
+ filter_mock = MagicMock()
+ filter_mock.all.return_value = [mock_history]
+ mock_session.query.return_value.filter.return_value = filter_mock
+
+ # Fail commit on attempt 1 with OperationalError("database is locked"), succeed on attempt 2
+ mock_session.commit.side_effect = [
+ OperationalError(
+ "UPDATE research_history", {}, Exception("database is locked")
+ ),
+ None,
+ ]
+
+ processed = processor.process_pending_operations_for_user(
+ username, mock_session
+ )
+
+ assert processed == 1
+ assert len(processor.pending_operations) == 0
+ assert mock_session.commit.call_count == 2
+ assert mock_session.rollback.call_count == 1
+
+
+def test_process_pending_operations_requeues_on_persistent_lock(processor):
+ """Verify that operations are re-queued with incremented retry_count if all commit retries fail."""
+ username = "test_user"
+ research_id = "res-fail-test"
+
+ processor.queue_progress_update(username, research_id, 33.0)
+ assert len(processor.pending_operations) == 1
+
+ mock_session = MagicMock()
+ mock_history = MagicMock(spec=ResearchHistory)
+ mock_history.id = research_id
+
+ filter_mock = MagicMock()
+ filter_mock.all.return_value = [mock_history]
+ mock_session.query.return_value.filter.return_value = filter_mock
+
+ # Fail commit on all 3 attempts
+ mock_session.commit.side_effect = OperationalError(
+ "UPDATE research_history", {}, Exception("database is locked")
+ )
+
+ processed = processor.process_pending_operations_for_user(
+ username, mock_session
+ )
+
+ # 0 processed successfully, operations returned to queue with retry_count set
+ assert processed == 0
+ assert len(processor.pending_operations) == 1
+ requeued_op = next(iter(processor.pending_operations.values()))
+ assert requeued_op.get("retry_count") == 1
+ assert mock_session.commit.call_count == 3
+ assert mock_session.rollback.call_count == 3
+
+
+def test_process_pending_operations_requeue_exceeding_max_attempts_drops_op(
+ processor,
+):
+ """Verify that operations exceeding max requeue attempts are dropped (dead-lettered)."""
+ username = "test_user"
+ research_id = "res-deadletter-test"
+
+ processor.pending_operations["op-dead"] = {
+ "username": username,
+ "operation_type": "progress_update",
+ "research_id": research_id,
+ "progress": 50.0,
+ "timestamp": 1000.0,
+ "retry_count": 10, # Already reached max requeue attempts threshold
+ }
+
+ mock_session = MagicMock()
+ mock_history = MagicMock(spec=ResearchHistory)
+ mock_history.id = research_id
+
+ filter_mock = MagicMock()
+ filter_mock.all.return_value = [mock_history]
+ mock_session.query.return_value.filter.return_value = filter_mock
+
+ # Fail commit on all retries
+ mock_session.commit.side_effect = OperationalError(
+ "UPDATE research_history", {}, Exception("database is locked")
+ )
+
+ processed = processor.process_pending_operations_for_user(
+ username, mock_session
+ )
+
+ assert processed == 0
+ # Operation should be dropped after exceeding max attempts
+ assert len(processor.pending_operations) == 0
diff --git a/tests/web/queue/test_processor_v2_real.py b/tests/web/queue/test_processor_v2_real.py
index 81122d1fb8..c229ddc9c5 100644
--- a/tests/web/queue/test_processor_v2_real.py
+++ b/tests/web/queue/test_processor_v2_real.py
@@ -948,7 +948,10 @@ class TestProcessPendingOperationsForUser:
mock_session = MagicMock()
mock_research = Mock()
- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+ mock_research.id = "r-200"
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ mock_research
+ ]
count = processor.process_pending_operations_for_user(
"alice", mock_session
@@ -974,7 +977,10 @@ class TestProcessPendingOperationsForUser:
mock_session = MagicMock()
mock_research = Mock()
- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+ mock_research.id = "r-201"
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ mock_research
+ ]
count = processor.process_pending_operations_for_user(
"alice", mock_session
@@ -1007,7 +1013,10 @@ class TestProcessPendingOperationsForUser:
mock_session = MagicMock()
mock_research = Mock()
- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+ mock_research.id = "r-301"
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ mock_research
+ ]
count = processor.process_pending_operations_for_user(
"alice", mock_session
@@ -1050,6 +1059,7 @@ class TestProcessPendingOperationsForUser:
mock_session = MagicMock()
mock_research = Mock(
spec=[
+ "id",
"status",
"error_message",
"research_meta",
@@ -1057,7 +1067,10 @@ class TestProcessPendingOperationsForUser:
"report_path",
]
)
- mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+ mock_research.id = "r-401"
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ mock_research
+ ]
count = processor.process_pending_operations_for_user(
"alice", mock_session