~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
from datetime import UTC, datetime
from unittest.mock import MagicMock, Mock, patch
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from local_deep_research.database.models.queue import QueueStatus, TaskMetadata
from local_deep_research.database.models.queued_research import QueuedResearch
from local_deep_research.database.models.research import ResearchHistory
from local_deep_research.web.queue import cleanup_queued_research_state
PROCESSOR_MODULE = "local_deep_research.web.queue.processor_v2"
SETTINGS_MANAGER = "local_deep_research.settings.manager.SettingsManager"
@pytest.fixture()
def db_session() -> Session:
engine = create_engine("sqlite:///:memory:")
QueuedResearch.__table__.create(engine)
QueueStatus.__table__.create(engine)
TaskMetadata.__table__.create(engine)
ResearchHistory.__table__.create(engine)
session = sessionmaker(bind=engine)()
yield session
session.close()
engine.dispose()
def _queued_research(research_id: str, position: int) -> QueuedResearch:
return QueuedResearch(
username="alice",
research_id=research_id,
query=research_id,
mode="quick",
position=position,
)
def _task(
task_id: str, status: str, task_type: str = "research"
) -> TaskMetadata:
return TaskMetadata(task_id=task_id, status=status, task_type=task_type)
def test_cleanup_counts_only_research_tasks_and_refreshes_last_checked(
db_session: Session,
) -> None:
# Given: research and indexing work with a stale research-capacity counter.
last_checked = datetime(2020, 1, 1, tzinfo=UTC)
db_session.add_all(
[
_queued_research("remove", 1),
_task("remove", "queued"),
_task("queued-research", "queued"),
_task("active-research", "processing"),
_task("indexing", "processing", "indexing"),
_task("benchmark", "queued", "benchmark"),
QueueStatus(
active_tasks=99, queued_tasks=99, last_checked=last_checked
),
]
)
db_session.commit()
# When: lifecycle cleanup recomputes queue capacity.
cleanup_queued_research_state(db_session, ["remove"])
db_session.commit()
# Then: only research metadata consumes research queue capacity.
status = db_session.query(QueueStatus).one()
assert status.queued_tasks == 1
assert status.active_tasks == 1
assert status.last_checked > last_checked
def test_cleanup_compacts_equal_positions_by_position_then_id(
db_session: Session,
) -> None:
# Given: surviving rows with the same queue position.
db_session.add_all(
[
_queued_research("remove", 1),
_queued_research("first", 4),
_queued_research("second", 4),
]
)
db_session.commit()
statements: list[str] = []
def capture_statement(
_connection, _cursor, statement, _parameters, _context, _many
):
statements.append(statement)
engine = db_session.get_bind()
event.listen(engine, "before_cursor_execute", capture_statement)
# When: cleanup compacts the affected user's queue.
try:
cleanup_queued_research_state(db_session, ["remove"])
finally:
event.remove(engine, "before_cursor_execute", capture_statement)
# Then: SQL ordering supplies a deterministic tie-breaker.
assert any(
"ORDER BY queued_researches.position, queued_researches.id" in statement
for statement in statements
)
def test_missing_parent_sweep_cleans_queue_and_metadata_only_orphans(
db_session: Session,
) -> None:
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
# Given: one orphaned queue row and one orphaned research metadata row.
db_session.add_all(
[
_queued_research("queue-only", 1),
_task("queue-only", "queued"),
_task("metadata-only", "processing"),
QueueStatus(active_tasks=7, queued_tasks=8),
]
)
db_session.commit()
processor = QueueProcessorV2()
processor._spawn_retry_counts.update({"queue-only": 1, "metadata-only": 2})
# When: legacy reconciliation runs for the user.
result = processor._sweep_missing_parent_queue_rows(db_session, "alice")
# Then: every orphan is removed and its retry state is discarded.
assert result.can_dispatch is True
assert result.cleaned_ids == frozenset({"queue-only", "metadata-only"})
assert db_session.query(QueuedResearch).count() == 0
assert db_session.query(TaskMetadata).count() == 0
assert processor._spawn_retry_counts == {}
def test_sweep_commit_failure_stops_dispatch_before_reclaim_or_claim(
db_session: Session,
) -> None:
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
# Given: an orphan whose cleanup transaction cannot commit.
db_session.add(_queued_research("missing-parent", 1))
db_session.commit()
processor = QueueProcessorV2()
processor._reclaim_stranded_queue_rows = Mock()
processor._start_research = Mock()
processor.notify_research_failed = Mock()
# When: dispatch reaches a failed reconciliation commit.
with patch.object(
processor, "_commit_with_safe_rollback", return_value=False
):
processor._start_queued_researches(db_session, Mock(), "alice", "pw", 1)
# Then: no subsequent reclaim, claim, retry, or failure notification runs.
processor._reclaim_stranded_queue_rows.assert_not_called()
processor._start_research.assert_not_called()
processor.notify_research_failed.assert_not_called()
@pytest.mark.parametrize(
("remaining_queue_row", "expected_empty"),
[(None, True), ((1,), False)],
ids=["drained", "persisted-row"],
)
def test_process_user_queue_sweeps_before_stale_status_capacity_gates(
remaining_queue_row: tuple[int] | None, expected_empty: bool
) -> None:
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
# Given: stale full-and-empty counters that reconciliation refreshes.
processor = QueueProcessorV2()
reconciled = False
call_order: list[str] = []
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_session.query.return_value.filter_by.return_value.count.return_value = 0
mock_session.query.return_value.filter_by.return_value.first.return_value = remaining_queue_row
queue_service = Mock()
def sweep(_db_session: Session, _username: str) -> Mock:
nonlocal reconciled
reconciled = True
call_order.append("sweep")
return Mock(can_dispatch=True)
def queue_status() -> dict[str, int]:
call_order.append("status")
if not reconciled:
return {"active_tasks": 1, "queued_tasks": 0}
if call_order.count("status") == 1:
return {"active_tasks": 0, "queued_tasks": 1}
return {"active_tasks": 0, "queued_tasks": 0}
processor._sweep_missing_parent_queue_rows = Mock(side_effect=sweep)
processor._start_queued_researches = Mock()
queue_service.get_queue_status.side_effect = queue_status
settings_manager = Mock()
settings_manager.get_setting.return_value = 1
# When: periodic queue processing begins.
with (
patch(f"{PROCESSOR_MODULE}.session_password_store") as passwords,
patch(f"{PROCESSOR_MODULE}.db_manager") as database_manager,
patch(
f"{PROCESSOR_MODULE}.get_user_db_session", return_value=mock_session
),
patch(
f"{PROCESSOR_MODULE}.UserQueueService", return_value=queue_service
),
patch(SETTINGS_MANAGER, return_value=settings_manager),
):
passwords.get_session_password.return_value = "pw"
database_manager.open_user_database.return_value = Mock()
result = processor._process_user_queue("alice", "session")
# Then: refreshed counters admit the queued research despite stale gates.
assert call_order[:2] == ["sweep", "status"]
processor._start_queued_researches.assert_called_once_with(
mock_session, queue_service, "alice", "pw", 1
)
# A zero counter after dispatch permits deregistration only when no
# persisted queue row remains for the next polling attempt.
assert result is expected_empty