Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.2 KB
Raw Blame
"""Caps for user-supplied string-list parameters on library routes.

``_string_list_error`` is the shared validation seam for the
``urls``/``research_ids``/``document_ids`` list parameters accepted by
``/api/check-downloads``, ``/api/download-bulk``, and
``/api/mark-redownload``. It checked shape only — a list of non-empty
strings — with no count or per-item length bound.

Only ``check-downloads`` feeds its list into a SQLAlchemy ``.in_(...)``
filter; ``download-bulk`` and ``mark-redownload`` loop over the list
per item instead. The routes' flat 16 MB JSON body allowance fits
millions of short items, while the shipped SQLite build accepts at
most 250,000 bind parameters in one query (a Debian compile-time
value — upstream SQLite defaults to 32,766). An authenticated user
could therefore post an oversized list and either turn a
``check-downloads`` request into an unhandled
``sqlalchemy.exc.OperationalError`` ("too many SQL variables", HTTP
500), or drive unbounded per-item query/session churn on the other two
routes. Real flows never approach the cap: a research run has at most
a few hundred sources and bulk selections are dozens.

These tests pin the caps: overlong lists and overlong items are
rejected with a 400 before any query runs, while honest lists keep
passing.
"""

import asyncio
import json
from contextlib import contextmanager
from unittest.mock import MagicMock, Mock, patch

from fastapi.responses import JSONResponse
from starlette.requests import Request

from local_deep_research.web.routers import library as library_module
from local_deep_research.web.routers.library import (
    _string_list_error,
    check_downloads,
    download_bulk,
    mark_for_redownload,
)

#: Contract values. 1000 items is far above any real bulk selection and
#: far below SQLite's 250,000 bind-parameter ceiling on this build.
CONTRACT_MAX_ITEMS = 1000
#: 2048 characters covers any URL or id the UI ever sends.
CONTRACT_MAX_ITEM_LENGTH = 2048


def test_oversized_list_is_rejected():
    """More items than the cap must 400, not reach the query layer."""
    error = _string_list_error(["x"] * (CONTRACT_MAX_ITEMS + 1), "urls")

    assert isinstance(error, JSONResponse), (
        "oversized urls list passed validation and would reach the "
        "route's query layer unbounded"
    )
    assert error.status_code == 400
    assert "at most" in json.loads(error.body)["error"]


def test_oversized_item_is_rejected():
    """Per-item length is bounded alongside the count."""
    error = _string_list_error(["x" * (CONTRACT_MAX_ITEM_LENGTH + 1)], "urls")

    assert isinstance(error, JSONResponse), (
        "an over-long list item passed validation unbounded"
    )
    assert error.status_code == 400


def test_honest_lists_still_pass():
    """Lists within both caps keep validating exactly as before."""
    # Empty-list rejection lives in the routes (``not urls``) before
    # the helper runs; the helper itself accepts [] — unchanged.
    assert _string_list_error([], "urls") is None
    assert _string_list_error(["a", ""], "urls") is not None
    assert _string_list_error(["a", "b"], "urls") is None
    assert _string_list_error(["u"] * CONTRACT_MAX_ITEMS, "urls") is None


def test_caps_stay_under_the_sqlite_bind_limit():
    """The implementation's cap must remain a safe margin under 250k.

    Structural bound so the invariant survives future cap tuning; the
    constant only exists once the fix lands, hence the local import.
    """
    from local_deep_research.web.routers.library import (  # noqa: PLC0415
        _MAX_STRING_LIST_ITEMS,
    )

    assert _MAX_STRING_LIST_ITEMS <= 250_000 // 100, (
        "item cap crept toward the SQLite bind-parameter ceiling; "
        "keep it a real bound, not a rounding error away from 500s"
    )


class TestBulkCeilingPreservesDownloadAll:
    """download-bulk's higher ceiling: the download-all contract holds.

    The locked contract in ``test_library_api_response_shapes`` requires
    ``download_bulk`` to stream lists above one thousand items; the
    manual-selection cap must not apply there. These pins keep the bulk
    ceiling honest from both sides: above the functional floor, far
    below the bind-parameter crash threshold.
    """

    def test_bulk_lists_above_the_manual_cap_validate(self):
        """1001 ids — the download-all contract size — must pass."""
        from local_deep_research.web.routers.library import (  # noqa: PLC0415
            _MAX_BULK_LIST_ITEMS,
        )

        assert (
            _string_list_error(
                ["r"] * 1001,
                "research_ids",
                max_items=_MAX_BULK_LIST_ITEMS,
            )
            is None
        )

    def test_bulk_ceiling_is_above_the_contract_floor(self):
        """>1000 streaming is structural, not incidental."""
        from local_deep_research.web.routers.library import (  # noqa: PLC0415
            _MAX_BULK_LIST_ITEMS,
        )

        assert _MAX_BULK_LIST_ITEMS > 1000, (
            "download-all contract requires >1000 items to stream; the "
            "bulk ceiling dropped to or below the functional floor"
        )

    def test_bulk_ceiling_stays_under_the_bind_limit(self):
        """Even the raised cap keeps a real margin under 250k binds."""
        from local_deep_research.web.routers.library import (  # noqa: PLC0415
            _MAX_BULK_LIST_ITEMS,
        )

        assert _MAX_BULK_LIST_ITEMS <= 250_000 // 2, (
            "bulk item cap crept toward the SQLite bind-parameter "
            "ceiling; keep it a real bound, not a rounding error "
            "away from 500s"
        )

    def test_oversized_bulk_list_is_still_rejected(self):
        from local_deep_research.web.routers.library import (  # noqa: PLC0415
            _MAX_BULK_LIST_ITEMS,
        )

        # The payload below scales with _MAX_BULK_LIST_ITEMS, so on its
        # own this test can never catch the constant moving -- "one more
        # than the ceiling" is rejected by construction no matter what
        # the ceiling is. Pin the actual contract value: 100_000 is the
        # deliberate ceiling, not just whatever the constant currently
        # says, so a change to it (e.g. loosening to 125_000) must also
        # update this literal.
        assert _MAX_BULK_LIST_ITEMS == 100_000, (
            "the bulk ceiling is a deliberate contract value; changing "
            "it requires updating this pin"
        )

        error = _string_list_error(
            ["r"] * (_MAX_BULK_LIST_ITEMS + 1),
            "research_ids",
            max_items=_MAX_BULK_LIST_ITEMS,
        )

        assert isinstance(error, JSONResponse), (
            "an oversized download-bulk list passed validation and "
            "would drive an unbounded per-item query loop"
        )
        assert error.status_code == 400


# ===========================================================================
# Route-level coverage of the max_items= wiring
# ===========================================================================
#
# Every test above calls ``_string_list_error`` directly, so nothing
# catches a route being handed the wrong ``max_items=`` -- e.g.
# ``check-downloads`` losing its default 1000-item ceiling by picking
# up ``_MAX_BULK_LIST_ITEMS``, or ``download-bulk`` losing its higher
# ceiling by falling back to the 1000-item default and breaking the
# locked download-all contract. These call the route handlers
# themselves, the same way
# ``tests/web/routers/test_library_api_response_shapes.py`` does (see
# its ``_post_json_request`` and ``_patched`` helpers), so a
# ``max_items=`` wiring change at the call site shows up here.


def _post_json_request(payload, path="/library/api/x"):
    """Build a POST request carrying *payload* as its JSON body.

    Adapted from the same-named helper in
    ``tests/web/routers/test_library_api_response_shapes.py``, which
    calls these route handlers directly the same way.
    """
    body = json.dumps(payload).encode("utf-8")

    async def receive():
        return {"type": "http.request", "body": body, "more_body": False}

    return Request(
        {
            "type": "http",
            "method": "POST",
            "path": path,
            "headers": [(b"content-type", b"application/json")],
            "query_string": b"",
            "session": {"session_id": "sid"},
        },
        receive,
    )


@contextmanager
def _fake_db_session(*_args, **_kwargs):
    yield MagicMock()


class TestRouteLevelCapWiring:
    def test_check_downloads_rejects_one_thousand_and_one_urls(self):
        """check-downloads must keep the 1000-item manual-selection cap.

        Fails if the route's ``_string_list_error(urls, "urls")`` call
        is ever changed to pass a higher (or no) ``max_items``, which
        would let more than 1000 urls through to the query layer.
        """
        with patch.object(
            library_module,
            "get_user_db_session",
            side_effect=_fake_db_session,
        ) as get_session:
            response = asyncio.run(
                check_downloads(
                    _post_json_request(
                        {"research_id": "r1", "urls": ["u"] * 1001},
                        path="/library/api/check-downloads",
                    ),
                    username="alice",
                )
            )

        assert response.status_code == 400
        assert json.loads(response.body) == {
            "error": "urls must contain at most 1000 items"
        }
        get_session.assert_not_called()

    def test_mark_redownload_rejects_one_thousand_and_one_document_ids(self):
        """mark-redownload must keep the 1000-item manual-selection cap.

        Fails if the route's
        ``_string_list_error(document_ids, "document_ids")`` call is
        ever changed to pass a higher (or no) ``max_items``, which
        would let more than 1000 document_ids through to the service.
        """
        service = Mock()
        run_db_sync = Mock(side_effect=AssertionError("database work started"))

        with (
            patch.object(
                library_module, "LibraryService", return_value=service
            ),
            patch.object(library_module, "run_db_sync", run_db_sync),
        ):
            response = asyncio.run(
                mark_for_redownload(
                    _post_json_request(
                        {"document_ids": ["d"] * 1001},
                        path="/library/api/mark-redownload",
                    ),
                    username="alice",
                )
            )

        assert response.status_code == 400
        assert json.loads(response.body) == {
            "error": "document_ids must contain at most 1000 items"
        }
        run_db_sync.assert_not_called()
        service.mark_for_redownload.assert_not_called()

    def test_download_bulk_accepts_one_thousand_and_one_research_ids(self):
        """download-bulk must keep its higher bulk ceiling.

        Fails if the route's call is ever repointed at the 1000-item
        manual-selection default: the locked download-all contract
        (``test_library_api_response_shapes``) requires lists above
        one thousand to stream (200, SSE), not 400.
        """
        response = asyncio.run(
            download_bulk(
                _post_json_request(
                    {"research_ids": ["r"] * 1001},
                    path="/library/api/download-bulk",
                ),
                username="alice",
            )
        )

        assert response.status_code == 200
        assert response.media_type == "text/event-stream"

    def test_download_bulk_rejects_one_hundred_thousand_and_one_research_ids(
        self,
    ):
        """download-bulk must still reject past its own (higher) ceiling.

        Fails if the route's call ever drops ``max_items=`` (unbounded)
        or is repointed at a looser constant than the 100,000-item
        bulk ceiling.
        """
        response = asyncio.run(
            download_bulk(
                _post_json_request(
                    {"research_ids": ["r"] * 100_001},
                    path="/library/api/download-bulk",
                ),
                username="alice",
            )
        )

        assert response.status_code == 400
        assert json.loads(response.body) == {
            "error": "research_ids must contain at most 100000 items"
        }