~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
from __future__ import annotations
import threading
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from sqlalchemy import func, select
from local_deep_research.constants import ResearchStatus
from local_deep_research.database.models import (
QueuedResearch,
QueueStatus,
ResearchHistory,
TaskMetadata,
UserActiveResearch,
)
from local_deep_research.database.queue_service import UserQueueService
from local_deep_research.database.session_context import get_user_db_session
from local_deep_research.exceptions import SystemAtCapacityError
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
from local_deep_research.web.research_state import (
remove_active_research,
set_active_research,
)
from tests.connected.queue_test_support import (
ConnectedUserFixture,
QueueResearchSeed,
_queue_state,
_register_connected_user,
_seed_queued_research,
)
def test_capacity_rejection_releases_claim_and_restores_queued_state(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
capacity_boundary_reached = threading.Event()
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(research_id=research_id, position=1),
)
def reject_at_capacity(*_args, **_kwargs) -> None:
capacity_boundary_reached.set()
raise SystemAtCapacityError("connected queue capacity boundary")
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.start_research_process",
reject_at_capacity,
)
with get_user_db_session(case.username, case.password) as db_session:
processor._start_queued_researches(
db_session,
UserQueueService(db_session),
case.username,
case.password,
1,
)
assert capacity_boundary_reached.is_set()
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
ResearchStatus.QUEUED,
1,
0,
"queued",
0,
0,
1,
)
def test_reclaim_only_resets_rows_without_a_live_research_thread(
connected_user: ConnectedUserFixture,
) -> None:
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
stranded_id = str(uuid4())
live_id = str(uuid4())
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=stranded_id,
position=1,
status=ResearchStatus.IN_PROGRESS,
is_processing=True,
),
)
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=live_id,
position=2,
status=ResearchStatus.IN_PROGRESS,
is_processing=True,
),
)
db_session.add_all(
[
UserActiveResearch(
username=case.username,
research_id=stranded_id,
status=ResearchStatus.IN_PROGRESS,
thread_id="stranded-thread",
settings_snapshot={},
),
UserActiveResearch(
username=case.username,
research_id=live_id,
status=ResearchStatus.IN_PROGRESS,
thread_id=str(threading.get_ident()),
settings_snapshot={},
),
]
)
db_session.commit()
set_active_research(live_id, {"thread": threading.current_thread()})
try:
with get_user_db_session(case.username, case.password) as db_session:
reclaimed = processor._reclaim_stranded_queue_rows(
db_session,
case.username,
)
finally:
remove_active_research(live_id)
assert reclaimed == 1
with get_user_db_session(case.username, case.password) as db_session:
stranded_claimed_rows = db_session.execute(
select(func.count(QueuedResearch.id)).where(
QueuedResearch.research_id == stranded_id,
QueuedResearch.is_processing.is_(True),
)
).scalar_one()
live_claimed_rows = db_session.execute(
select(func.count(QueuedResearch.id)).where(
QueuedResearch.research_id == live_id,
QueuedResearch.is_processing.is_(True),
)
).scalar_one()
stranded_history_status = db_session.execute(
select(ResearchHistory.status).where(
ResearchHistory.id == stranded_id
)
).scalar_one()
live_history_status = db_session.execute(
select(ResearchHistory.status).where(ResearchHistory.id == live_id)
).scalar_one()
active_rows = db_session.execute(
select(func.count(UserActiveResearch.research_id)).where(
UserActiveResearch.research_id.in_([stranded_id, live_id])
)
).scalar_one()
assert stranded_claimed_rows == 0
assert stranded_history_status == ResearchStatus.QUEUED
assert live_claimed_rows == 1
assert live_history_status == ResearchStatus.IN_PROGRESS
assert active_rows == 2
def test_transient_spawn_failure_keeps_task_metadata_queued(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""#6147: a transient spawn failure must revert the TaskMetadata
claim, otherwise reconcile_research_queue_status under-reports
queued_tasks and the dispatch loop deregisters the user before the
row is retried."""
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
spawn_attempted = threading.Event()
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(research_id=research_id, position=1),
)
def fail_first_spawn(*_args, **_kwargs) -> None:
spawn_attempted.set()
raise RuntimeError("transient spawn failure")
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.start_research_process",
fail_first_spawn,
)
with get_user_db_session(case.username, case.password) as db_session:
processor._start_queued_researches(
db_session,
UserQueueService(db_session),
case.username,
case.password,
1,
)
assert spawn_attempted.is_set()
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
ResearchStatus.QUEUED,
1,
0,
"queued",
0,
0,
1,
)
def test_reclaim_resets_stranded_task_metadata_to_queued(
connected_user: ConnectedUserFixture,
) -> None:
"""#6147: crash-reclaim must reset a stranded 'processing'
TaskMetadata row back to 'queued' so the re-dispatchable row keeps
counting toward queued_tasks."""
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
stranded_id = str(uuid4())
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=stranded_id,
position=1,
status=ResearchStatus.IN_PROGRESS,
is_processing=True,
),
)
# Model the pre-crash claimed state: TaskMetadata flipped to
# 'processing' and the queue-status counters leaked into
# active_tasks.
metadata = (
db_session.query(TaskMetadata)
.filter_by(task_id=stranded_id, task_type="research")
.one()
)
metadata.status = "processing"
metadata.started_at = datetime.now(UTC)
queue_status = db_session.query(QueueStatus).one()
queue_status.active_tasks = 1
queue_status.queued_tasks = 0
db_session.commit()
with get_user_db_session(case.username, case.password) as db_session:
reclaimed = processor._reclaim_stranded_queue_rows(
db_session,
case.username,
)
assert reclaimed == 1
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, stranded_id) == (
ResearchStatus.QUEUED,
1,
0,
"queued",
0,
0,
1,
)
started_at = db_session.execute(
select(TaskMetadata.started_at).where(
TaskMetadata.task_id == stranded_id
)
).scalar_one()
assert started_at is None
@pytest.mark.parametrize("recovery_commit_fails", [False, True])
def test_polling_recovers_claimed_row_when_queued_counter_is_zero(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
recovery_commit_fails: bool,
) -> None:
"""A crash or failed recovery commit must not deregister pending work."""
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
spawn_attempted = threading.Event()
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
lambda *_args: case.password,
)
assert processor._process_user_queue(case.username, "poll-session") is True
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=research_id,
position=1,
status=ResearchStatus.IN_PROGRESS,
is_processing=True,
),
)
metadata = (
db_session.query(TaskMetadata).filter_by(task_id=research_id).one()
)
metadata.status = "processing"
metadata.started_at = datetime.now(UTC)
queue_status = db_session.query(QueueStatus).one()
queue_status.active_tasks = 1
queue_status.queued_tasks = 0
db_session.commit()
def reject_at_capacity(*_args, **_kwargs) -> None:
spawn_attempted.set()
raise SystemAtCapacityError("connected queue capacity boundary")
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.start_research_process",
reject_at_capacity,
)
if recovery_commit_fails:
commit = processor._commit_with_safe_rollback
def fail_recovery_commit(db_session, operation):
if operation.startswith("reclaim of stranded rows"):
db_session.rollback()
return False
return commit(db_session, operation)
monkeypatch.setattr(
processor, "_commit_with_safe_rollback", fail_recovery_commit
)
assert (
processor._process_user_queue(case.username, "poll-session")
is False
)
assert not spawn_attempted.is_set()
monkeypatch.setattr(processor, "_commit_with_safe_rollback", commit)
assert processor._process_user_queue(case.username, "poll-session") is False
assert spawn_attempted.is_set()
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
ResearchStatus.QUEUED,
1,
0,
"queued",
0,
0,
1,
)
@pytest.mark.parametrize(
"status",
[
ResearchStatus.COMPLETED,
ResearchStatus.SUSPENDED,
ResearchStatus.FAILED,
ResearchStatus.ERROR,
],
)
@pytest.mark.parametrize("is_processing", [False, True])
def test_polling_discards_finished_queue_rows(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
status: ResearchStatus,
is_processing: bool,
) -> None:
"""A finished parent cannot be re-dispatched or counted as queued work."""
from unittest.mock import Mock
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
processor._spawn_retry_counts[research_id] = 2
spawn = Mock()
processor.notify_research_failed = Mock()
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.start_research_process",
spawn,
)
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
lambda *_args: case.password,
)
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=research_id,
position=1,
status=status,
is_processing=is_processing,
),
)
UserQueueService(db_session).update_task_status(
research_id, "processing"
)
assert processor._process_user_queue(case.username, "poll-session") is True
assert processor._process_user_queue(case.username, "poll-session") is True
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
status,
0,
0,
None,
0,
0,
0,
)
assert research_id not in processor._spawn_retry_counts
spawn.assert_not_called()
processor.notify_research_failed.assert_not_called()
def test_finished_queue_cleanup_keeps_live_worker_claim(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The worker may set a terminal status before its final bookkeeping."""
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
lambda *_args: case.password,
)
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=research_id,
position=1,
status=ResearchStatus.COMPLETED,
is_processing=True,
),
)
UserQueueService(db_session).update_task_status(
research_id, "processing"
)
set_active_research(research_id, {"thread": threading.current_thread()})
try:
assert (
processor._process_user_queue(case.username, "poll-session")
is False
)
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
ResearchStatus.COMPLETED,
1,
1,
"processing",
0,
1,
0,
)
finally:
remove_active_research(research_id)
assert processor._process_user_queue(case.username, "poll-session") is True
def test_finished_queue_cleanup_rolls_back_before_retry(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
research_id = str(uuid4())
processor._spawn_retry_counts[research_id] = 2
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.session_password_store.get_session_password",
lambda *_args: case.password,
)
with get_user_db_session(case.username, case.password) as db_session:
_seed_queued_research(
db_session,
case.username,
QueueResearchSeed(
research_id=research_id,
position=1,
status=ResearchStatus.SUSPENDED,
is_processing=True,
),
)
UserQueueService(db_session).update_task_status(
research_id, "processing"
)
commit = processor._commit_with_safe_rollback
def fail_recovery_commit(db_session, operation):
if operation.startswith("reclaim of stranded rows"):
db_session.rollback()
return False
return commit(db_session, operation)
monkeypatch.setattr(
processor, "_commit_with_safe_rollback", fail_recovery_commit
)
assert processor._process_user_queue(case.username, "poll-session") is False
with get_user_db_session(case.username, case.password) as db_session:
assert _queue_state(db_session, research_id) == (
ResearchStatus.SUSPENDED,
1,
1,
"processing",
0,
1,
0,
)
assert processor._spawn_retry_counts[research_id] == 2
monkeypatch.setattr(processor, "_commit_with_safe_rollback", commit)
assert processor._process_user_queue(case.username, "poll-session") is True
assert research_id not in processor._spawn_retry_counts
@pytest.mark.parametrize("is_processing", [False, True])
def test_finished_queue_row_does_not_starve_waiting_research(
connected_user: ConnectedUserFixture,
monkeypatch: pytest.MonkeyPatch,
is_processing: bool,
) -> None:
case = connected_user
_register_connected_user(case)
processor = QueueProcessorV2()
finished_id, waiting_id = str(uuid4()), str(uuid4())
spawned = []
with get_user_db_session(case.username, case.password) as db_session:
for seed in (
QueueResearchSeed(
research_id=finished_id,
position=1,
status=ResearchStatus.COMPLETED,
is_processing=is_processing,
),
QueueResearchSeed(research_id=waiting_id, position=2),
):
_seed_queued_research(db_session, case.username, seed)
UserQueueService(db_session).update_task_status(
finished_id, "completed"
)
def spawn(research_id, *_args, **_kwargs):
spawned.append(research_id)
return threading.current_thread()
monkeypatch.setattr(
"local_deep_research.web.queue.processor_v2.start_research_process",
spawn,
)
for _ in range(2):
with get_user_db_session(case.username, case.password) as db_session:
processor._start_queued_researches(
db_session,
UserQueueService(db_session),
case.username,
case.password,
1,
)
assert spawned == [waiting_id]
with get_user_db_session(case.username, case.password) as db_session:
assert (
db_session.query(ResearchHistory)
.filter_by(id=finished_id)
.one()
.status
== ResearchStatus.COMPLETED
)
assert _queue_state(db_session, waiting_id) == (
ResearchStatus.IN_PROGRESS,
0,
0,
"processing",
1,
1,
0,
)