Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.6 KB
Raw Blame
"""
Tests for processor_v2 handling of SQLite/SQLCipher lock contention, batching, deduplication, and re-queuing.
"""

from unittest.mock import MagicMock, patch, PropertyMock
import pytest
from sqlalchemy.exc import OperationalError

from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
from local_deep_research.database.models import ResearchHistory


@pytest.fixture
def processor():
    """Create a clean QueueProcessorV2 instance for testing."""
    return QueueProcessorV2()


def test_process_pending_operations_deduplicates_and_batches(processor):
    """Only the latest timestamp per research is applied once."""
    username = "test_user"
    research_id_1 = "res-111"
    research_id_2 = "res-222"

    # Deliberately make insertion order disagree with timestamp order. Without
    # deduplication, last-write-wins would leave both models at 90.0 and the
    # old assertions would still pass.
    timestamps = [
        100.0,
        900.0,  # res-111's real latest value is 20.0
        200.0,
        300.0,
        400.0,
        500.0,
        600.0,
        700.0,
        800.0,
        1000.0,  # res-222's real latest value is 50.0
        950.0,
    ]
    for i in range(1, 10):
        processor.queue_progress_update(username, research_id_1, float(i * 10))
    processor.queue_progress_update(username, research_id_2, 50.0)
    processor.queue_progress_update(username, research_id_2, 90.0)

    assert len(processor.pending_operations) == 11

    # Stamp the intended timestamps directly instead of patching ``time.time``
    # around the queueing calls. ``queue_progress_update`` also calls
    # ``time.time`` indirectly via ``_evict_stale_pending_operations``, so a
    # fixed ``side_effect`` list no longer maps 1:1 onto queued operations;
    # and feeding it artificially old values makes TTL eviction reap the very
    # entries this test is about to assert on. ``pending_operations`` is
    # insertion-ordered, so zipping preserves the intended pairing.
    for op, ts in zip(processor.pending_operations.values(), timestamps):
        op["timestamp"] = ts

    # Mock DB Session
    mock_session = MagicMock()
    mock_history_1 = MagicMock(spec=ResearchHistory)
    mock_history_1.id = research_id_1
    progress_1 = PropertyMock()
    type(mock_history_1).progress = progress_1
    mock_history_2 = MagicMock(spec=ResearchHistory)
    mock_history_2.id = research_id_2
    progress_2 = PropertyMock()
    type(mock_history_2).progress = progress_2

    filter_mock = MagicMock()
    filter_mock.all.return_value = [mock_history_1, mock_history_2]
    mock_session.query.return_value.filter.return_value = filter_mock

    processed = processor.process_pending_operations_for_user(
        username, mock_session
    )

    # All 11 original operations cleared from queue
    assert processed == 11
    assert len(processor.pending_operations) == 0

    # Dedup must apply exactly one update per research, chosen by timestamp
    # rather than insertion order.
    progress_1.assert_called_once_with(20.0)
    progress_2.assert_called_once_with(50.0)

    # Consolidated batch query called exactly once for the batch
    assert mock_session.query.call_count == 1
    # Single commit called for the batch
    assert mock_session.commit.call_count == 1


def test_process_pending_operations_timestamp_tie_handling(processor):
    """Verify that timestamp tie (identical time.time()) gives precedence to later queued operation."""
    username = "test_user"
    research_id = "res-tie"

    with patch("time.time", return_value=1000.0):
        processor.queue_progress_update(username, research_id, 10.0)
        processor.queue_progress_update(username, research_id, 90.0)

    mock_session = MagicMock()
    mock_history = MagicMock(spec=ResearchHistory)
    mock_history.id = research_id

    filter_mock = MagicMock()
    filter_mock.all.return_value = [mock_history]
    mock_session.query.return_value.filter.return_value = filter_mock

    processed = processor.process_pending_operations_for_user(
        username, mock_session
    )

    assert processed == 2
    assert mock_history.progress == 90.0


def test_process_pending_operations_retries_on_operational_error(processor):
    """Verify that transient database lock (OperationalError) triggers retry and succeeds."""
    username = "test_user"
    research_id = "res-lock-test"

    processor.queue_progress_update(username, research_id, 75.0)

    mock_session = MagicMock()
    mock_history = MagicMock(spec=ResearchHistory)
    mock_history.id = research_id

    filter_mock = MagicMock()
    filter_mock.all.return_value = [mock_history]
    mock_session.query.return_value.filter.return_value = filter_mock

    # Fail commit on attempt 1 with OperationalError("database is locked"), succeed on attempt 2
    mock_session.commit.side_effect = [
        OperationalError(
            "UPDATE research_history", {}, Exception("database is locked")
        ),
        None,
    ]

    processed = processor.process_pending_operations_for_user(
        username, mock_session
    )

    assert processed == 1
    assert len(processor.pending_operations) == 0
    assert mock_session.commit.call_count == 2
    assert mock_session.rollback.call_count == 1


def test_process_pending_operations_requeues_on_persistent_lock(processor):
    """Verify that operations are re-queued with incremented retry_count if all commit retries fail."""
    username = "test_user"
    research_id = "res-fail-test"

    processor.queue_progress_update(username, research_id, 33.0)
    assert len(processor.pending_operations) == 1

    mock_session = MagicMock()
    mock_history = MagicMock(spec=ResearchHistory)
    mock_history.id = research_id

    filter_mock = MagicMock()
    filter_mock.all.return_value = [mock_history]
    mock_session.query.return_value.filter.return_value = filter_mock

    # Fail commit on all 3 attempts
    mock_session.commit.side_effect = OperationalError(
        "UPDATE research_history", {}, Exception("database is locked")
    )

    processed = processor.process_pending_operations_for_user(
        username, mock_session
    )

    # 0 processed successfully, operations returned to queue with retry_count set
    assert processed == 0
    assert len(processor.pending_operations) == 1
    requeued_op = next(iter(processor.pending_operations.values()))
    assert requeued_op.get("retry_count") == 1
    assert mock_session.commit.call_count == 3
    assert mock_session.rollback.call_count == 3


def test_process_pending_operations_requeue_exceeding_max_attempts_drops_op(
    processor,
):
    """Verify that operations exceeding max requeue attempts are dropped (dead-lettered)."""
    username = "test_user"
    research_id = "res-deadletter-test"

    processor.pending_operations["op-dead"] = {
        "username": username,
        "operation_type": "progress_update",
        "research_id": research_id,
        "progress": 50.0,
        "timestamp": 1000.0,
        "retry_count": 10,  # Already reached max requeue attempts threshold
    }

    mock_session = MagicMock()
    mock_history = MagicMock(spec=ResearchHistory)
    mock_history.id = research_id

    filter_mock = MagicMock()
    filter_mock.all.return_value = [mock_history]
    mock_session.query.return_value.filter.return_value = filter_mock

    # Fail commit on all retries
    mock_session.commit.side_effect = OperationalError(
        "UPDATE research_history", {}, Exception("database is locked")
    )

    processed = processor.process_pending_operations_for_user(
        username, mock_session
    )

    assert processed == 0
    # Operation should be dropped after exceeding max attempts
    assert len(processor.pending_operations) == 0