~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for processor_v2 handling of SQLite/SQLCipher lock contention, batching, deduplication, and re-queuing.
"""
from unittest.mock import MagicMock, patch, PropertyMock
import pytest
from sqlalchemy.exc import OperationalError
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
from local_deep_research.database.models import ResearchHistory
@pytest.fixture
def processor():
"""Create a clean QueueProcessorV2 instance for testing."""
return QueueProcessorV2()
def test_process_pending_operations_deduplicates_and_batches(processor):
"""Only the latest timestamp per research is applied once."""
username = "test_user"
research_id_1 = "res-111"
research_id_2 = "res-222"
# Deliberately make insertion order disagree with timestamp order. Without
# deduplication, last-write-wins would leave both models at 90.0 and the
# old assertions would still pass.
timestamps = [
100.0,
900.0, # res-111's real latest value is 20.0
200.0,
300.0,
400.0,
500.0,
600.0,
700.0,
800.0,
1000.0, # res-222's real latest value is 50.0
950.0,
]
for i in range(1, 10):
processor.queue_progress_update(username, research_id_1, float(i * 10))
processor.queue_progress_update(username, research_id_2, 50.0)
processor.queue_progress_update(username, research_id_2, 90.0)
assert len(processor.pending_operations) == 11
# Stamp the intended timestamps directly instead of patching ``time.time``
# around the queueing calls. ``queue_progress_update`` also calls
# ``time.time`` indirectly via ``_evict_stale_pending_operations``, so a
# fixed ``side_effect`` list no longer maps 1:1 onto queued operations;
# and feeding it artificially old values makes TTL eviction reap the very
# entries this test is about to assert on. ``pending_operations`` is
# insertion-ordered, so zipping preserves the intended pairing.
for op, ts in zip(processor.pending_operations.values(), timestamps):
op["timestamp"] = ts
# Mock DB Session
mock_session = MagicMock()
mock_history_1 = MagicMock(spec=ResearchHistory)
mock_history_1.id = research_id_1
progress_1 = PropertyMock()
type(mock_history_1).progress = progress_1
mock_history_2 = MagicMock(spec=ResearchHistory)
mock_history_2.id = research_id_2
progress_2 = PropertyMock()
type(mock_history_2).progress = progress_2
filter_mock = MagicMock()
filter_mock.all.return_value = [mock_history_1, mock_history_2]
mock_session.query.return_value.filter.return_value = filter_mock
processed = processor.process_pending_operations_for_user(
username, mock_session
)
# All 11 original operations cleared from queue
assert processed == 11
assert len(processor.pending_operations) == 0
# Dedup must apply exactly one update per research, chosen by timestamp
# rather than insertion order.
progress_1.assert_called_once_with(20.0)
progress_2.assert_called_once_with(50.0)
# Consolidated batch query called exactly once for the batch
assert mock_session.query.call_count == 1
# Single commit called for the batch
assert mock_session.commit.call_count == 1
def test_process_pending_operations_timestamp_tie_handling(processor):
"""Verify that timestamp tie (identical time.time()) gives precedence to later queued operation."""
username = "test_user"
research_id = "res-tie"
with patch("time.time", return_value=1000.0):
processor.queue_progress_update(username, research_id, 10.0)
processor.queue_progress_update(username, research_id, 90.0)
mock_session = MagicMock()
mock_history = MagicMock(spec=ResearchHistory)
mock_history.id = research_id
filter_mock = MagicMock()
filter_mock.all.return_value = [mock_history]
mock_session.query.return_value.filter.return_value = filter_mock
processed = processor.process_pending_operations_for_user(
username, mock_session
)
assert processed == 2
assert mock_history.progress == 90.0
def test_process_pending_operations_retries_on_operational_error(processor):
"""Verify that transient database lock (OperationalError) triggers retry and succeeds."""
username = "test_user"
research_id = "res-lock-test"
processor.queue_progress_update(username, research_id, 75.0)
mock_session = MagicMock()
mock_history = MagicMock(spec=ResearchHistory)
mock_history.id = research_id
filter_mock = MagicMock()
filter_mock.all.return_value = [mock_history]
mock_session.query.return_value.filter.return_value = filter_mock
# Fail commit on attempt 1 with OperationalError("database is locked"), succeed on attempt 2
mock_session.commit.side_effect = [
OperationalError(
"UPDATE research_history", {}, Exception("database is locked")
),
None,
]
processed = processor.process_pending_operations_for_user(
username, mock_session
)
assert processed == 1
assert len(processor.pending_operations) == 0
assert mock_session.commit.call_count == 2
assert mock_session.rollback.call_count == 1
def test_process_pending_operations_requeues_on_persistent_lock(processor):
"""Verify that operations are re-queued with incremented retry_count if all commit retries fail."""
username = "test_user"
research_id = "res-fail-test"
processor.queue_progress_update(username, research_id, 33.0)
assert len(processor.pending_operations) == 1
mock_session = MagicMock()
mock_history = MagicMock(spec=ResearchHistory)
mock_history.id = research_id
filter_mock = MagicMock()
filter_mock.all.return_value = [mock_history]
mock_session.query.return_value.filter.return_value = filter_mock
# Fail commit on all 3 attempts
mock_session.commit.side_effect = OperationalError(
"UPDATE research_history", {}, Exception("database is locked")
)
processed = processor.process_pending_operations_for_user(
username, mock_session
)
# 0 processed successfully, operations returned to queue with retry_count set
assert processed == 0
assert len(processor.pending_operations) == 1
requeued_op = next(iter(processor.pending_operations.values()))
assert requeued_op.get("retry_count") == 1
assert mock_session.commit.call_count == 3
assert mock_session.rollback.call_count == 3
def test_process_pending_operations_requeue_exceeding_max_attempts_drops_op(
processor,
):
"""Verify that operations exceeding max requeue attempts are dropped (dead-lettered)."""
username = "test_user"
research_id = "res-deadletter-test"
processor.pending_operations["op-dead"] = {
"username": username,
"operation_type": "progress_update",
"research_id": research_id,
"progress": 50.0,
"timestamp": 1000.0,
"retry_count": 10, # Already reached max requeue attempts threshold
}
mock_session = MagicMock()
mock_history = MagicMock(spec=ResearchHistory)
mock_history.id = research_id
filter_mock = MagicMock()
filter_mock.all.return_value = [mock_history]
mock_session.query.return_value.filter.return_value = filter_mock
# Fail commit on all retries
mock_session.commit.side_effect = OperationalError(
"UPDATE research_history", {}, Exception("database is locked")
)
processed = processor.process_pending_operations_for_user(
username, mock_session
)
assert processed == 0
# Operation should be dropped after exceeding max attempts
assert len(processor.pending_operations) == 0