~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""``POST /library/api/queue-all-undownloaded`` -- what it queues and what it
counts as skipped.
Ported from ``tests/research_library/routes/test_library_routes_extra_coverage.py``
(``TestQueueAllUndownloadedEdgeCases``) and
``test_library_routes_coverage.py``/``_deep_coverage.py``
(``TestQueueAllUndownloaded``), all deleted in the Flask->FastAPI migration.
The handler has ZERO behavioural coverage on the branch: the only occurrence
of the path anywhere under ``tests/`` is one line in
``tests/security/test_unauthenticated_reachability_census.py`` asserting it
401s when logged out. Everything the route decides -- which resources are
queued, which are skipped, and what the reset UPDATE writes -- could be
deleted outright without turning a single test red.
The counts are not cosmetic: ``queued`` and ``skipped`` are the numbers the
"Queue all" button reports back, and the three skip paths are different
answers to "why did nothing happen". Conflating "your retry policy refused
this" with "this resource has no URL" leaves a user with an unexplained zero.
The claim-safety half of this route (its ``status != PROCESSING`` reset
guard, #4691) lives in ``test_download_queue_claim_routes_4691.py``, which
drives it against a real database.
"""
import json
from contextlib import contextmanager
from unittest.mock import MagicMock, Mock, patch
import pytest
from starlette.requests import Request
from local_deep_research.database.models.library import DocumentStatus
from local_deep_research.web.routers import library as library_module
from local_deep_research.web.routers.library import queue_all_undownloaded
# ---------------------------------------------------------------------------
# Harness
# ---------------------------------------------------------------------------
def _request():
return Request(
{
"type": "http",
"method": "POST",
"path": "/library/api/queue-all-undownloaded",
"headers": [(b"content-type", b"application/json")],
"query_string": b"",
"session": {"session_id": "sid"},
},
lambda: {"type": "http.request", "body": b"{}", "more_body": False},
)
def _resource(rid, url="https://arxiv.org/abs/2301.00001", research_id="r1"):
"""One projected ``(id, url, research_id)`` row from the scan query."""
row = Mock()
row.id = rid
row.url = url
row.research_id = research_id
return row
class _ResourceQuery:
"""The whole-table scan: ``query(...).outerjoin(...).filter(...).all()``."""
def __init__(self, rows):
self._rows = rows
def outerjoin(self, *args, **kwargs):
return self
def join(self, *args, **kwargs):
# The lookup of resources whose document awaits its PDF
# (``_one_resource_per_awaiting_document``) finds none here.
return _ResourceQuery([])
def filter(self, *args, **kwargs):
return self
def all(self):
return self._rows
class _QueueQuery:
"""The per-resource ``LibraryDownloadQueue`` lookup and reset UPDATE."""
def __init__(self, existing=None, update_result=1):
self._existing = existing
self._update_result = update_result
self.updates = []
def filter_by(self, **kwargs):
return self
def filter(self, *args, **kwargs):
return self
def first(self):
return self._existing
def update(self, values, **kwargs):
self.updates.append(values)
return self._update_result
class _Session:
"""Routes the scan query and the queue query to their own doubles."""
def __init__(self, resource_query, queue_query):
self.resource_query = resource_query
self.queue_query = queue_query
self.added = []
def query(self, *entities):
if entities and entities[0] is library_module.LibraryDownloadQueue:
return self.queue_query
return self.resource_query
def add(self, obj):
self.added.append(obj)
def commit(self):
pass
def _resource_filter(results, permanently_failed=0, temporarily_failed=0):
instance = MagicMock()
instance.filter_downloadable_resources.return_value = results
summary = MagicMock()
summary.to_dict.return_value = {"total": len(results)}
summary.permanently_failed_count = permanently_failed
summary.temporarily_failed_count = temporarily_failed
instance.get_filter_summary.return_value = summary
instance.get_skipped_resources_info.return_value = []
return instance
def _filter_result(resource_id, can_retry=True, reason="ok"):
result = Mock()
result.resource_id = resource_id
result.can_retry = can_retry
result.reason = reason
return result
def _run(session, filter_results):
@contextmanager
def fake_db_session(*args, **kwargs):
yield session
with (
patch.object(
library_module, "get_user_db_session", side_effect=fake_db_session
),
patch.object(
library_module, "get_authenticated_user_password", return_value="pw"
),
patch.object(
library_module,
"ResourceFilter",
return_value=_resource_filter(filter_results),
),
):
return queue_all_undownloaded(_request(), username="alice")
# ===========================================================================
# The three skip paths
# ===========================================================================
def test_a_resource_with_no_url_is_skipped_not_queued():
"""``ResearchResource.url`` is nullable. A row with no URL cannot be
downloaded, so it must be counted as skipped -- queueing it would create
a queue row nothing can ever resolve.
Two guards cooperate here (the explicit ``if not resource.url`` fast path
and ``is_downloadable_domain(None) -> False``), so the assertion is on
the OUTCOME rather than on either one: it goes red only when both are
gone, which is the state that actually queues an unresolvable row."""
session = _Session(_ResourceQuery([_resource(20, url=None)]), _QueueQuery())
result = _run(session, [_filter_result(20)])
assert result["queued"] == 0, result
assert result["skipped"] == 1, result
assert session.added == [], "a URL-less resource must not be queued"
def test_a_resource_the_retry_policy_refuses_is_skipped():
"""``can_retry=False`` -- the resource has failed permanently often
enough that the filter says stop. Ignoring it re-queues a resource that
will fail again on every single "Queue all"."""
session = _Session(_ResourceQuery([_resource(21)]), _QueueQuery())
result = _run(session, [_filter_result(21, can_retry=False)])
assert result["queued"] == 0, result
assert result["skipped"] == 1, result
assert session.added == []
def test_a_resource_with_no_filter_result_at_all_is_skipped():
"""The lookup is ``filter_results_by_id.get(resource.id)``, so a resource
the filter did not report on comes back ``None``. Treating a missing
verdict as permission would queue exactly the resources the filter chose
not to vouch for."""
session = _Session(_ResourceQuery([_resource(24)]), _QueueQuery())
result = _run(session, [])
assert result["queued"] == 0, result
assert result["skipped"] == 1, result
assert session.added == []
# ===========================================================================
# The queue path
# ===========================================================================
def test_a_new_resource_is_queued_as_a_pending_row():
"""Positive control for all three skip tests: the same harness with a
passing filter result and a real URL must actually queue something, so
"skip everything" would not satisfy them."""
session = _Session(
_ResourceQuery([_resource(1)]), _QueueQuery(existing=None)
)
result = _run(session, [_filter_result(1)])
assert result["success"] is True
assert result["queued"] == 1, result
assert result["skipped"] == 0, result
assert result["research_ids"] == ["r1"]
assert len(session.added) == 1
queued = session.added[0]
assert isinstance(queued, library_module.LibraryDownloadQueue)
assert queued.resource_id == 1
assert queued.research_id == "r1"
assert queued.status is DocumentStatus.PENDING
def test_an_existing_row_is_reset_to_pending_and_its_completion_cleared():
"""The retry path. ``completed_at`` must be cleared alongside the status:
a row left carrying a completion timestamp while sitting at PENDING is a
contradiction the download manager renders as "already downloaded".
"""
existing = Mock()
existing.id = 7
queue_query = _QueueQuery(existing=existing, update_result=1)
session = _Session(_ResourceQuery([_resource(21)]), queue_query)
result = _run(session, [_filter_result(21)])
assert result["queued"] == 1, result
assert session.added == [], "an existing row must be updated, not re-added"
assert queue_query.updates, "no reset UPDATE was issued"
values = {col.key: val for col, val in queue_query.updates[0].items()}
assert values["status"] is DocumentStatus.PENDING
assert values["completed_at"] is None
def test_a_row_that_is_already_pending_is_still_reported_as_queued():
"""The UPDATE matches nothing for a row already PENDING (its filter
excludes PENDING), yet the resource IS queued for download. Counting only
the rows the UPDATE touched would tell a user who pressed the button
twice that nothing is queued."""
existing = Mock()
existing.id = 7
queue_query = _QueueQuery(existing=existing, update_result=0)
session = _Session(_ResourceQuery([_resource(22)]), queue_query)
result = _run(session, [_filter_result(22)])
assert result["queued"] == 1, (
f"a row that was already pending is still queued for download: {result}"
)
assert result["skipped"] == 0, result
# ===========================================================================
# The response shape the UI reads
# ===========================================================================
def test_the_response_carries_the_counts_and_the_filter_summary():
session = _Session(
_ResourceQuery([_resource(1), _resource(2, url=None)]), _QueueQuery()
)
result = _run(session, [_filter_result(1), _filter_result(2)])
assert set(result) == {
"success",
"queued",
"research_ids",
"total_undownloaded",
"skipped",
"filter_summary",
"skipped_details",
}
assert result["total_undownloaded"] == 2
assert result["queued"] == 1
assert result["skipped"] == 1
# Serialisable: this is returned straight to the browser as JSON.
json.dumps(result)
@pytest.mark.parametrize("count", [0, 3])
def test_total_undownloaded_reports_the_scan_size_not_the_queued_count(count):
"""``total_undownloaded`` is the size of the scan, independent of how
many were queued -- it is what tells the user how much of their library
the filter looked at."""
rows = [_resource(i, url=None) for i in range(count)]
session = _Session(_ResourceQuery(rows), _QueueQuery())
result = _run(session, [])
assert result["total_undownloaded"] == count
assert result["queued"] == 0