Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.1 KB
Raw Blame
"""``POST /library/api/queue-all-undownloaded`` -- what it queues and what it
counts as skipped.

Ported from ``tests/research_library/routes/test_library_routes_extra_coverage.py``
(``TestQueueAllUndownloadedEdgeCases``) and
``test_library_routes_coverage.py``/``_deep_coverage.py``
(``TestQueueAllUndownloaded``), all deleted in the Flask->FastAPI migration.

The handler has ZERO behavioural coverage on the branch: the only occurrence
of the path anywhere under ``tests/`` is one line in
``tests/security/test_unauthenticated_reachability_census.py`` asserting it
401s when logged out. Everything the route decides -- which resources are
queued, which are skipped, and what the reset UPDATE writes -- could be
deleted outright without turning a single test red.

The counts are not cosmetic: ``queued`` and ``skipped`` are the numbers the
"Queue all" button reports back, and the three skip paths are different
answers to "why did nothing happen". Conflating "your retry policy refused
this" with "this resource has no URL" leaves a user with an unexplained zero.

The claim-safety half of this route (its ``status != PROCESSING`` reset
guard, #4691) lives in ``test_download_queue_claim_routes_4691.py``, which
drives it against a real database.
"""

import json
from contextlib import contextmanager
from unittest.mock import MagicMock, Mock, patch

import pytest
from starlette.requests import Request

from local_deep_research.database.models.library import DocumentStatus
from local_deep_research.web.routers import library as library_module
from local_deep_research.web.routers.library import queue_all_undownloaded


# ---------------------------------------------------------------------------
# Harness
# ---------------------------------------------------------------------------


def _request():
    return Request(
        {
            "type": "http",
            "method": "POST",
            "path": "/library/api/queue-all-undownloaded",
            "headers": [(b"content-type", b"application/json")],
            "query_string": b"",
            "session": {"session_id": "sid"},
        },
        lambda: {"type": "http.request", "body": b"{}", "more_body": False},
    )


def _resource(rid, url="https://arxiv.org/abs/2301.00001", research_id="r1"):
    """One projected ``(id, url, research_id)`` row from the scan query."""
    row = Mock()
    row.id = rid
    row.url = url
    row.research_id = research_id
    return row


class _ResourceQuery:
    """The whole-table scan: ``query(...).outerjoin(...).filter(...).all()``."""

    def __init__(self, rows):
        self._rows = rows

    def outerjoin(self, *args, **kwargs):
        return self

    def join(self, *args, **kwargs):
        # The lookup of resources whose document awaits its PDF
        # (``_one_resource_per_awaiting_document``) finds none here.
        return _ResourceQuery([])

    def filter(self, *args, **kwargs):
        return self

    def all(self):
        return self._rows


class _QueueQuery:
    """The per-resource ``LibraryDownloadQueue`` lookup and reset UPDATE."""

    def __init__(self, existing=None, update_result=1):
        self._existing = existing
        self._update_result = update_result
        self.updates = []

    def filter_by(self, **kwargs):
        return self

    def filter(self, *args, **kwargs):
        return self

    def first(self):
        return self._existing

    def update(self, values, **kwargs):
        self.updates.append(values)
        return self._update_result


class _Session:
    """Routes the scan query and the queue query to their own doubles."""

    def __init__(self, resource_query, queue_query):
        self.resource_query = resource_query
        self.queue_query = queue_query
        self.added = []

    def query(self, *entities):
        if entities and entities[0] is library_module.LibraryDownloadQueue:
            return self.queue_query
        return self.resource_query

    def add(self, obj):
        self.added.append(obj)

    def commit(self):
        pass


def _resource_filter(results, permanently_failed=0, temporarily_failed=0):
    instance = MagicMock()
    instance.filter_downloadable_resources.return_value = results
    summary = MagicMock()
    summary.to_dict.return_value = {"total": len(results)}
    summary.permanently_failed_count = permanently_failed
    summary.temporarily_failed_count = temporarily_failed
    instance.get_filter_summary.return_value = summary
    instance.get_skipped_resources_info.return_value = []
    return instance


def _filter_result(resource_id, can_retry=True, reason="ok"):
    result = Mock()
    result.resource_id = resource_id
    result.can_retry = can_retry
    result.reason = reason
    return result


def _run(session, filter_results):
    @contextmanager
    def fake_db_session(*args, **kwargs):
        yield session

    with (
        patch.object(
            library_module, "get_user_db_session", side_effect=fake_db_session
        ),
        patch.object(
            library_module, "get_authenticated_user_password", return_value="pw"
        ),
        patch.object(
            library_module,
            "ResourceFilter",
            return_value=_resource_filter(filter_results),
        ),
    ):
        return queue_all_undownloaded(_request(), username="alice")


# ===========================================================================
# The three skip paths
# ===========================================================================


def test_a_resource_with_no_url_is_skipped_not_queued():
    """``ResearchResource.url`` is nullable. A row with no URL cannot be
    downloaded, so it must be counted as skipped -- queueing it would create
    a queue row nothing can ever resolve.

    Two guards cooperate here (the explicit ``if not resource.url`` fast path
    and ``is_downloadable_domain(None) -> False``), so the assertion is on
    the OUTCOME rather than on either one: it goes red only when both are
    gone, which is the state that actually queues an unresolvable row."""
    session = _Session(_ResourceQuery([_resource(20, url=None)]), _QueueQuery())

    result = _run(session, [_filter_result(20)])

    assert result["queued"] == 0, result
    assert result["skipped"] == 1, result
    assert session.added == [], "a URL-less resource must not be queued"


def test_a_resource_the_retry_policy_refuses_is_skipped():
    """``can_retry=False`` -- the resource has failed permanently often
    enough that the filter says stop. Ignoring it re-queues a resource that
    will fail again on every single "Queue all"."""
    session = _Session(_ResourceQuery([_resource(21)]), _QueueQuery())

    result = _run(session, [_filter_result(21, can_retry=False)])

    assert result["queued"] == 0, result
    assert result["skipped"] == 1, result
    assert session.added == []


def test_a_resource_with_no_filter_result_at_all_is_skipped():
    """The lookup is ``filter_results_by_id.get(resource.id)``, so a resource
    the filter did not report on comes back ``None``. Treating a missing
    verdict as permission would queue exactly the resources the filter chose
    not to vouch for."""
    session = _Session(_ResourceQuery([_resource(24)]), _QueueQuery())

    result = _run(session, [])

    assert result["queued"] == 0, result
    assert result["skipped"] == 1, result
    assert session.added == []


# ===========================================================================
# The queue path
# ===========================================================================


def test_a_new_resource_is_queued_as_a_pending_row():
    """Positive control for all three skip tests: the same harness with a
    passing filter result and a real URL must actually queue something, so
    "skip everything" would not satisfy them."""
    session = _Session(
        _ResourceQuery([_resource(1)]), _QueueQuery(existing=None)
    )

    result = _run(session, [_filter_result(1)])

    assert result["success"] is True
    assert result["queued"] == 1, result
    assert result["skipped"] == 0, result
    assert result["research_ids"] == ["r1"]
    assert len(session.added) == 1
    queued = session.added[0]
    assert isinstance(queued, library_module.LibraryDownloadQueue)
    assert queued.resource_id == 1
    assert queued.research_id == "r1"
    assert queued.status is DocumentStatus.PENDING


def test_an_existing_row_is_reset_to_pending_and_its_completion_cleared():
    """The retry path. ``completed_at`` must be cleared alongside the status:
    a row left carrying a completion timestamp while sitting at PENDING is a
    contradiction the download manager renders as "already downloaded".
    """
    existing = Mock()
    existing.id = 7
    queue_query = _QueueQuery(existing=existing, update_result=1)
    session = _Session(_ResourceQuery([_resource(21)]), queue_query)

    result = _run(session, [_filter_result(21)])

    assert result["queued"] == 1, result
    assert session.added == [], "an existing row must be updated, not re-added"
    assert queue_query.updates, "no reset UPDATE was issued"
    values = {col.key: val for col, val in queue_query.updates[0].items()}
    assert values["status"] is DocumentStatus.PENDING
    assert values["completed_at"] is None


def test_a_row_that_is_already_pending_is_still_reported_as_queued():
    """The UPDATE matches nothing for a row already PENDING (its filter
    excludes PENDING), yet the resource IS queued for download. Counting only
    the rows the UPDATE touched would tell a user who pressed the button
    twice that nothing is queued."""
    existing = Mock()
    existing.id = 7
    queue_query = _QueueQuery(existing=existing, update_result=0)
    session = _Session(_ResourceQuery([_resource(22)]), queue_query)

    result = _run(session, [_filter_result(22)])

    assert result["queued"] == 1, (
        f"a row that was already pending is still queued for download: {result}"
    )
    assert result["skipped"] == 0, result


# ===========================================================================
# The response shape the UI reads
# ===========================================================================


def test_the_response_carries_the_counts_and_the_filter_summary():
    session = _Session(
        _ResourceQuery([_resource(1), _resource(2, url=None)]), _QueueQuery()
    )

    result = _run(session, [_filter_result(1), _filter_result(2)])

    assert set(result) == {
        "success",
        "queued",
        "research_ids",
        "total_undownloaded",
        "skipped",
        "filter_summary",
        "skipped_details",
    }
    assert result["total_undownloaded"] == 2
    assert result["queued"] == 1
    assert result["skipped"] == 1
    # Serialisable: this is returned straight to the browser as JSON.
    json.dumps(result)


@pytest.mark.parametrize("count", [0, 3])
def test_total_undownloaded_reports_the_scan_size_not_the_queued_count(count):
    """``total_undownloaded`` is the size of the scan, independent of how
    many were queued -- it is what tells the user how much of their library
    the filter looked at."""
    rows = [_resource(i, url=None) for i in range(count)]
    session = _Session(_ResourceQuery(rows), _QueueQuery())

    result = _run(session, [])

    assert result["total_undownloaded"] == count
    assert result["queued"] == 0