Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.6 KB
Raw Blame
from __future__ import annotations

import threading
from datetime import UTC, datetime
from uuid import uuid4

import pytest
from sqlalchemy import func, select

from local_deep_research.constants import ResearchStatus
from local_deep_research.database.models import (
    QueuedResearch,
    QueueStatus,
    ResearchHistory,
    TaskMetadata,
    UserActiveResearch,
)
from local_deep_research.database.queue_service import UserQueueService
from local_deep_research.database.session_context import get_user_db_session
from local_deep_research.exceptions import SystemAtCapacityError
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
from local_deep_research.web.research_state import (
    remove_active_research,
    set_active_research,
)
from tests.connected.queue_test_support import (
    ConnectedUserFixture,
    QueueResearchSeed,
    _queue_state,
    _register_connected_user,
    _seed_queued_research,
)


def test_capacity_rejection_releases_claim_and_restores_queued_state(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    capacity_boundary_reached = threading.Event()

    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(research_id=research_id, position=1),
        )

    def reject_at_capacity(*_args, **_kwargs) -> None:
        capacity_boundary_reached.set()
        raise SystemAtCapacityError("connected queue capacity boundary")

    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.start_research_process",
        reject_at_capacity,
    )

    with get_user_db_session(case.username, case.password) as db_session:
        processor._start_queued_researches(
            db_session,
            UserQueueService(db_session),
            case.username,
            case.password,
            1,
        )

    assert capacity_boundary_reached.is_set()
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, research_id) == (
            ResearchStatus.QUEUED,
            1,
            0,
            "queued",
            0,
            0,
            1,
        )


def test_reclaim_only_resets_rows_without_a_live_research_thread(
    connected_user: ConnectedUserFixture,
) -> None:
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    stranded_id = str(uuid4())
    live_id = str(uuid4())

    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=stranded_id,
                position=1,
                status=ResearchStatus.IN_PROGRESS,
                is_processing=True,
            ),
        )
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=live_id,
                position=2,
                status=ResearchStatus.IN_PROGRESS,
                is_processing=True,
            ),
        )
        db_session.add_all(
            [
                UserActiveResearch(
                    username=case.username,
                    research_id=stranded_id,
                    status=ResearchStatus.IN_PROGRESS,
                    thread_id="stranded-thread",
                    settings_snapshot={},
                ),
                UserActiveResearch(
                    username=case.username,
                    research_id=live_id,
                    status=ResearchStatus.IN_PROGRESS,
                    thread_id=str(threading.get_ident()),
                    settings_snapshot={},
                ),
            ]
        )
        db_session.commit()

    set_active_research(live_id, {"thread": threading.current_thread()})
    try:
        with get_user_db_session(case.username, case.password) as db_session:
            reclaimed = processor._reclaim_stranded_queue_rows(
                db_session,
                case.username,
            )
    finally:
        remove_active_research(live_id)

    assert reclaimed == 1
    with get_user_db_session(case.username, case.password) as db_session:
        stranded_claimed_rows = db_session.execute(
            select(func.count(QueuedResearch.id)).where(
                QueuedResearch.research_id == stranded_id,
                QueuedResearch.is_processing.is_(True),
            )
        ).scalar_one()
        live_claimed_rows = db_session.execute(
            select(func.count(QueuedResearch.id)).where(
                QueuedResearch.research_id == live_id,
                QueuedResearch.is_processing.is_(True),
            )
        ).scalar_one()
        stranded_history_status = db_session.execute(
            select(ResearchHistory.status).where(
                ResearchHistory.id == stranded_id
            )
        ).scalar_one()
        live_history_status = db_session.execute(
            select(ResearchHistory.status).where(ResearchHistory.id == live_id)
        ).scalar_one()
        active_rows = db_session.execute(
            select(func.count(UserActiveResearch.research_id)).where(
                UserActiveResearch.research_id.in_([stranded_id, live_id])
            )
        ).scalar_one()

    assert stranded_claimed_rows == 0
    assert stranded_history_status == ResearchStatus.QUEUED
    assert live_claimed_rows == 1
    assert live_history_status == ResearchStatus.IN_PROGRESS
    assert active_rows == 2


def test_transient_spawn_failure_keeps_task_metadata_queued(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """#6147: a transient spawn failure must revert the TaskMetadata
    claim, otherwise reconcile_research_queue_status under-reports
    queued_tasks and the dispatch loop deregisters the user before the
    row is retried."""
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    spawn_attempted = threading.Event()

    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(research_id=research_id, position=1),
        )

    def fail_first_spawn(*_args, **_kwargs) -> None:
        spawn_attempted.set()
        raise RuntimeError("transient spawn failure")

    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.start_research_process",
        fail_first_spawn,
    )

    with get_user_db_session(case.username, case.password) as db_session:
        processor._start_queued_researches(
            db_session,
            UserQueueService(db_session),
            case.username,
            case.password,
            1,
        )

    assert spawn_attempted.is_set()
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, research_id) == (
            ResearchStatus.QUEUED,
            1,
            0,
            "queued",
            0,
            0,
            1,
        )


def test_reclaim_resets_stranded_task_metadata_to_queued(
    connected_user: ConnectedUserFixture,
) -> None:
    """#6147: crash-reclaim must reset a stranded 'processing'
    TaskMetadata row back to 'queued' so the re-dispatchable row keeps
    counting toward queued_tasks."""
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    stranded_id = str(uuid4())

    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=stranded_id,
                position=1,
                status=ResearchStatus.IN_PROGRESS,
                is_processing=True,
            ),
        )
        # Model the pre-crash claimed state: TaskMetadata flipped to
        # 'processing' and the queue-status counters leaked into
        # active_tasks.
        metadata = (
            db_session.query(TaskMetadata)
            .filter_by(task_id=stranded_id, task_type="research")
            .one()
        )
        metadata.status = "processing"
        metadata.started_at = datetime.now(UTC)
        queue_status = db_session.query(QueueStatus).one()
        queue_status.active_tasks = 1
        queue_status.queued_tasks = 0
        db_session.commit()

    with get_user_db_session(case.username, case.password) as db_session:
        reclaimed = processor._reclaim_stranded_queue_rows(
            db_session,
            case.username,
        )

    assert reclaimed == 1
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, stranded_id) == (
            ResearchStatus.QUEUED,
            1,
            0,
            "queued",
            0,
            0,
            1,
        )
        started_at = db_session.execute(
            select(TaskMetadata.started_at).where(
                TaskMetadata.task_id == stranded_id
            )
        ).scalar_one()
        assert started_at is None


@pytest.mark.parametrize("recovery_commit_fails", [False, True])
def test_polling_recovers_claimed_row_when_queued_counter_is_zero(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
    recovery_commit_fails: bool,
) -> None:
    """A crash or failed recovery commit must not deregister pending work."""
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    spawn_attempted = threading.Event()

    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
        lambda *_args: case.password,
    )
    assert processor._process_user_queue(case.username, "poll-session") is True

    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=research_id,
                position=1,
                status=ResearchStatus.IN_PROGRESS,
                is_processing=True,
            ),
        )
        metadata = (
            db_session.query(TaskMetadata).filter_by(task_id=research_id).one()
        )
        metadata.status = "processing"
        metadata.started_at = datetime.now(UTC)
        queue_status = db_session.query(QueueStatus).one()
        queue_status.active_tasks = 1
        queue_status.queued_tasks = 0
        db_session.commit()

    def reject_at_capacity(*_args, **_kwargs) -> None:
        spawn_attempted.set()
        raise SystemAtCapacityError("connected queue capacity boundary")

    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.start_research_process",
        reject_at_capacity,
    )
    if recovery_commit_fails:
        commit = processor._commit_with_safe_rollback

        def fail_recovery_commit(db_session, operation):
            if operation.startswith("reclaim of stranded rows"):
                db_session.rollback()
                return False
            return commit(db_session, operation)

        monkeypatch.setattr(
            processor, "_commit_with_safe_rollback", fail_recovery_commit
        )
        assert (
            processor._process_user_queue(case.username, "poll-session")
            is False
        )
        assert not spawn_attempted.is_set()
        monkeypatch.setattr(processor, "_commit_with_safe_rollback", commit)

    assert processor._process_user_queue(case.username, "poll-session") is False
    assert spawn_attempted.is_set()
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, research_id) == (
            ResearchStatus.QUEUED,
            1,
            0,
            "queued",
            0,
            0,
            1,
        )


@pytest.mark.parametrize(
    "status",
    [
        ResearchStatus.COMPLETED,
        ResearchStatus.SUSPENDED,
        ResearchStatus.FAILED,
        ResearchStatus.ERROR,
    ],
)
@pytest.mark.parametrize("is_processing", [False, True])
def test_polling_discards_finished_queue_rows(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
    status: ResearchStatus,
    is_processing: bool,
) -> None:
    """A finished parent cannot be re-dispatched or counted as queued work."""
    from unittest.mock import Mock

    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    processor._spawn_retry_counts[research_id] = 2
    spawn = Mock()
    processor.notify_research_failed = Mock()
    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.start_research_process",
        spawn,
    )
    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
        lambda *_args: case.password,
    )
    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=research_id,
                position=1,
                status=status,
                is_processing=is_processing,
            ),
        )
        UserQueueService(db_session).update_task_status(
            research_id, "processing"
        )

    assert processor._process_user_queue(case.username, "poll-session") is True
    assert processor._process_user_queue(case.username, "poll-session") is True
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, research_id) == (
            status,
            0,
            0,
            None,
            0,
            0,
            0,
        )
    assert research_id not in processor._spawn_retry_counts
    spawn.assert_not_called()
    processor.notify_research_failed.assert_not_called()


def test_finished_queue_cleanup_keeps_live_worker_claim(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The worker may set a terminal status before its final bookkeeping."""
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
        lambda *_args: case.password,
    )
    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=research_id,
                position=1,
                status=ResearchStatus.COMPLETED,
                is_processing=True,
            ),
        )
        UserQueueService(db_session).update_task_status(
            research_id, "processing"
        )
    set_active_research(research_id, {"thread": threading.current_thread()})
    try:
        assert (
            processor._process_user_queue(case.username, "poll-session")
            is False
        )
        with get_user_db_session(case.username, case.password) as db_session:
            assert _queue_state(db_session, research_id) == (
                ResearchStatus.COMPLETED,
                1,
                1,
                "processing",
                0,
                1,
                0,
            )
    finally:
        remove_active_research(research_id)
    assert processor._process_user_queue(case.username, "poll-session") is True


def test_finished_queue_cleanup_rolls_back_before_retry(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    research_id = str(uuid4())
    processor._spawn_retry_counts[research_id] = 2
    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
        lambda *_args: case.password,
    )
    with get_user_db_session(case.username, case.password) as db_session:
        _seed_queued_research(
            db_session,
            case.username,
            QueueResearchSeed(
                research_id=research_id,
                position=1,
                status=ResearchStatus.SUSPENDED,
                is_processing=True,
            ),
        )
        UserQueueService(db_session).update_task_status(
            research_id, "processing"
        )
    commit = processor._commit_with_safe_rollback

    def fail_recovery_commit(db_session, operation):
        if operation.startswith("reclaim of stranded rows"):
            db_session.rollback()
            return False
        return commit(db_session, operation)

    monkeypatch.setattr(
        processor, "_commit_with_safe_rollback", fail_recovery_commit
    )
    assert processor._process_user_queue(case.username, "poll-session") is False
    with get_user_db_session(case.username, case.password) as db_session:
        assert _queue_state(db_session, research_id) == (
            ResearchStatus.SUSPENDED,
            1,
            1,
            "processing",
            0,
            1,
            0,
        )
    assert processor._spawn_retry_counts[research_id] == 2
    monkeypatch.setattr(processor, "_commit_with_safe_rollback", commit)
    assert processor._process_user_queue(case.username, "poll-session") is True
    assert research_id not in processor._spawn_retry_counts


@pytest.mark.parametrize("is_processing", [False, True])
def test_finished_queue_row_does_not_starve_waiting_research(
    connected_user: ConnectedUserFixture,
    monkeypatch: pytest.MonkeyPatch,
    is_processing: bool,
) -> None:
    case = connected_user
    _register_connected_user(case)
    processor = QueueProcessorV2()
    finished_id, waiting_id = str(uuid4()), str(uuid4())
    spawned = []
    with get_user_db_session(case.username, case.password) as db_session:
        for seed in (
            QueueResearchSeed(
                research_id=finished_id,
                position=1,
                status=ResearchStatus.COMPLETED,
                is_processing=is_processing,
            ),
            QueueResearchSeed(research_id=waiting_id, position=2),
        ):
            _seed_queued_research(db_session, case.username, seed)
        UserQueueService(db_session).update_task_status(
            finished_id, "completed"
        )

    def spawn(research_id, *_args, **_kwargs):
        spawned.append(research_id)
        return threading.current_thread()

    monkeypatch.setattr(
        "local_deep_research.web.queue.processor_v2.start_research_process",
        spawn,
    )
    for _ in range(2):
        with get_user_db_session(case.username, case.password) as db_session:
            processor._start_queued_researches(
                db_session,
                UserQueueService(db_session),
                case.username,
                case.password,
                1,
            )
    assert spawned == [waiting_id]
    with get_user_db_session(case.username, case.password) as db_session:
        assert (
            db_session.query(ResearchHistory)
            .filter_by(id=finished_id)
            .one()
            .status
            == ResearchStatus.COMPLETED
        )
        assert _queue_state(db_session, waiting_id) == (
            ResearchStatus.IN_PROGRESS,
            0,
            0,
            "processing",
            1,
            1,
            0,
        )