~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): bound library string-list parameters (1000 items, 2048 chars; 100,000 for download-bulk) (#6428)
LearningCircuit702cdb3e9a10
diff --git a/changelog.d/6428.security.md b/changelog.d/6428.security.mdnew file mode 100644index 0000000000..7c204a304e--- /dev/null+++ b/changelog.d/6428.security.md@@ -0,0 +1 @@+The `urls`/`research_ids`/`document_ids` list parameters on `/api/check-downloads`, `/api/download-bulk`, and `/api/mark-redownload` are now capped (1000 items and 2048 characters per item; download-bulk keeps a higher 100,000-item ceiling so the download-all flow still streams libraries above one thousand entries). The shared shape-only validator let an oversized list through the routes' flat 16 MB JSON body allowance, which fits millions of short items. On `check-downloads`, that reached the `.in_()` SQL filter and — on the shipped build, where SQLite accepts at most 250,000 bind parameters (a Debian compile-time value; upstream SQLite defaults to 32,766) — turned the request into an unhandled `sqlalchemy.exc.OperationalError` ("too many SQL variables", HTTP 500). `download-bulk` and `mark-redownload` don't use `.in_()`; there, the same oversized list instead drove an unbounded per-item query/session loop. Oversized input now returns a 400 before any query runs; honest lists are unaffected.diff --git a/src/local_deep_research/web/routers/library.py b/src/local_deep_research/web/routers/library.pyindex 38dbad2314..652b6700cf 100644--- a/src/local_deep_research/web/routers/library.py+++ b/src/local_deep_research/web/routers/library.py@@ -56,15 +56,69 @@ from typing import Annotatedrouter = APIRouter(prefix="/library", tags=["library"])-def _string_list_error(value, field_name: str) -> JSONResponse | None:- """Return a 400 response unless *value* is a non-empty-string list."""- if not isinstance(value, list) or any(- not isinstance(item, str) or not item.strip() for item in value- ):+#: Upper bound for user-supplied id/url lists on the manual-selection+#: routes (check-downloads, mark-redownload). Far above any real bulk+#: selection and a safe margin under SQLite's 250,000 bind-parameter+#: ceiling, so an oversized list can never reach the .in_() filters.+_MAX_STRING_LIST_ITEMS = 1000+#: Higher bound for download-bulk only: the locked download-all+#: contract (test_library_api_response_shapes) requires lists above+#: one thousand to stream, so the functional floor sits above the+#: manual cap. download_bulk never binds these ids into a SQL query —+#: it loops per id, opening a session and queueing downloads for each+#: — so the real cost axis is per-item work, not a bind-parameter+#: ceiling. 100,000 bounds that per-item work to a size the "queue+#: everything undownloaded" flow (queue-all-undownloaded's+#: research_ids) can legitimately reach.+_MAX_BULK_LIST_ITEMS = 100_000+#: Upper bound per item; 2048 characters covers any URL or id the+#: UI ever sends while bounding per-request memory.+_MAX_STRING_LIST_ITEM_LENGTH = 2048+++def _string_list_error(+ value, field_name: str, max_items: int = _MAX_STRING_LIST_ITEMS+) -> JSONResponse | None:+ """Return a 400 response unless *value* is a bounded non-empty-string list.++ Caps both the item count and per-item length. Only+ ``check-downloads`` feeds the list into a SQLAlchemy ``.in_()``+ filter, whose bind-parameter ceiling this helper's default+ ``max_items`` stays a safe margin under; ``download-bulk`` and+ ``mark-redownload`` loop over the list per item instead, so their+ exposure is unbounded per-item work rather than a bind-parameter+ crash. Routes that must preserve larger lists (download-all) pass+ a higher ``max_items`` sized to that per-item cost instead of the+ bind ceiling.++ The count check runs before the per-item type/length sweeps so an+ oversized list 400s in O(1) instead of paying an O(n) scan first.+ """+ if not isinstance(value, list):+ return JSONResponse(+ {"error": f"{field_name} must be a list of non-empty strings"},+ status_code=400,+ )+ if len(value) > max_items:+ return JSONResponse(+ {"error": (f"{field_name} must contain at most {max_items} items")},+ status_code=400,+ )+ if any(not isinstance(item, str) or not item.strip() for item in value):return JSONResponse({"error": f"{field_name} must be a list of non-empty strings"},status_code=400,)+ if any(len(item) > _MAX_STRING_LIST_ITEM_LENGTH for item in value):+ return JSONResponse(+ {+ "error": (+ f"{field_name} items must be at most"+ f" {_MAX_STRING_LIST_ITEM_LENGTH} characters"+ )+ },+ status_code=400,+ )return None@@ -1033,7 +1087,15 @@ async def download_bulk(return JSONResponse({"error": "No research IDs provided"}, status_code=400)- if error := _string_list_error(research_ids, "research_ids"):+ # Download-all selections legitimately exceed the manual-selection+ # cap (locked contract: lists above one thousand must stream), so+ # this route uses the higher bulk ceiling — bounded to a size the+ # "queue everything undownloaded" flow can legitimately reach, not+ # to a SQL bind-parameter limit (this route loops per id rather+ # than binding the list into a query).+ if error := _string_list_error(+ research_ids, "research_ids", max_items=_MAX_BULK_LIST_ITEMS+ ):return errorif not isinstance(mode, str) or mode not in {"pdf", "text_only"}:return JSONResponse(diff --git a/tests/security/test_string_list_route_caps.py b/tests/security/test_string_list_route_caps.pynew file mode 100644index 0000000000..cbc43b091d--- /dev/null+++ b/tests/security/test_string_list_route_caps.py@@ -0,0 +1,328 @@+"""Caps for user-supplied string-list parameters on library routes.++``_string_list_error`` is the shared validation seam for the+``urls``/``research_ids``/``document_ids`` list parameters accepted by+``/api/check-downloads``, ``/api/download-bulk``, and+``/api/mark-redownload``. It checked shape only — a list of non-empty+strings — with no count or per-item length bound.++Only ``check-downloads`` feeds its list into a SQLAlchemy ``.in_(...)``+filter; ``download-bulk`` and ``mark-redownload`` loop over the list+per item instead. The routes' flat 16 MB JSON body allowance fits+millions of short items, while the shipped SQLite build accepts at+most 250,000 bind parameters in one query (a Debian compile-time+value — upstream SQLite defaults to 32,766). An authenticated user+could therefore post an oversized list and either turn a+``check-downloads`` request into an unhandled+``sqlalchemy.exc.OperationalError`` ("too many SQL variables", HTTP+500), or drive unbounded per-item query/session churn on the other two+routes. Real flows never approach the cap: a research run has at most+a few hundred sources and bulk selections are dozens.++These tests pin the caps: overlong lists and overlong items are+rejected with a 400 before any query runs, while honest lists keep+passing.+"""++import asyncio+import json+from contextlib import contextmanager+from unittest.mock import MagicMock, Mock, patch++from fastapi.responses import JSONResponse+from starlette.requests import Request++from local_deep_research.web.routers import library as library_module+from local_deep_research.web.routers.library import (+ _string_list_error,+ check_downloads,+ download_bulk,+ mark_for_redownload,+)++#: Contract values. 1000 items is far above any real bulk selection and+#: far below SQLite's 250,000 bind-parameter ceiling on this build.+CONTRACT_MAX_ITEMS = 1000+#: 2048 characters covers any URL or id the UI ever sends.+CONTRACT_MAX_ITEM_LENGTH = 2048+++def test_oversized_list_is_rejected():+ """More items than the cap must 400, not reach the query layer."""+ error = _string_list_error(["x"] * (CONTRACT_MAX_ITEMS + 1), "urls")++ assert isinstance(error, JSONResponse), (+ "oversized urls list passed validation and would reach the "+ "route's query layer unbounded"+ )+ assert error.status_code == 400+ assert "at most" in json.loads(error.body)["error"]+++def test_oversized_item_is_rejected():+ """Per-item length is bounded alongside the count."""+ error = _string_list_error(["x" * (CONTRACT_MAX_ITEM_LENGTH + 1)], "urls")++ assert isinstance(error, JSONResponse), (+ "an over-long list item passed validation unbounded"+ )+ assert error.status_code == 400+++def test_honest_lists_still_pass():+ """Lists within both caps keep validating exactly as before."""+ # Empty-list rejection lives in the routes (``not urls``) before+ # the helper runs; the helper itself accepts [] — unchanged.+ assert _string_list_error([], "urls") is None+ assert _string_list_error(["a", ""], "urls") is not None+ assert _string_list_error(["a", "b"], "urls") is None+ assert _string_list_error(["u"] * CONTRACT_MAX_ITEMS, "urls") is None+++def test_caps_stay_under_the_sqlite_bind_limit():+ """The implementation's cap must remain a safe margin under 250k.++ Structural bound so the invariant survives future cap tuning; the+ constant only exists once the fix lands, hence the local import.+ """+ from local_deep_research.web.routers.library import ( # noqa: PLC0415+ _MAX_STRING_LIST_ITEMS,+ )++ assert _MAX_STRING_LIST_ITEMS <= 250_000 // 100, (+ "item cap crept toward the SQLite bind-parameter ceiling; "+ "keep it a real bound, not a rounding error away from 500s"+ )+++class TestBulkCeilingPreservesDownloadAll:+ """download-bulk's higher ceiling: the download-all contract holds.++ The locked contract in ``test_library_api_response_shapes`` requires+ ``download_bulk`` to stream lists above one thousand items; the+ manual-selection cap must not apply there. These pins keep the bulk+ ceiling honest from both sides: above the functional floor, far+ below the bind-parameter crash threshold.+ """++ def test_bulk_lists_above_the_manual_cap_validate(self):+ """1001 ids — the download-all contract size — must pass."""+ from local_deep_research.web.routers.library import ( # noqa: PLC0415+ _MAX_BULK_LIST_ITEMS,+ )++ assert (+ _string_list_error(+ ["r"] * 1001,+ "research_ids",+ max_items=_MAX_BULK_LIST_ITEMS,+ )+ is None+ )++ def test_bulk_ceiling_is_above_the_contract_floor(self):+ """>1000 streaming is structural, not incidental."""+ from local_deep_research.web.routers.library import ( # noqa: PLC0415+ _MAX_BULK_LIST_ITEMS,+ )++ assert _MAX_BULK_LIST_ITEMS > 1000, (+ "download-all contract requires >1000 items to stream; the "+ "bulk ceiling dropped to or below the functional floor"+ )++ def test_bulk_ceiling_stays_under_the_bind_limit(self):+ """Even the raised cap keeps a real margin under 250k binds."""+ from local_deep_research.web.routers.library import ( # noqa: PLC0415+ _MAX_BULK_LIST_ITEMS,+ )++ assert _MAX_BULK_LIST_ITEMS <= 250_000 // 2, (+ "bulk item cap crept toward the SQLite bind-parameter "+ "ceiling; keep it a real bound, not a rounding error "+ "away from 500s"+ )++ def test_oversized_bulk_list_is_still_rejected(self):+ from local_deep_research.web.routers.library import ( # noqa: PLC0415+ _MAX_BULK_LIST_ITEMS,+ )++ # The payload below scales with _MAX_BULK_LIST_ITEMS, so on its+ # own this test can never catch the constant moving -- "one more+ # than the ceiling" is rejected by construction no matter what+ # the ceiling is. Pin the actual contract value: 100_000 is the+ # deliberate ceiling, not just whatever the constant currently+ # says, so a change to it (e.g. loosening to 125_000) must also+ # update this literal.+ assert _MAX_BULK_LIST_ITEMS == 100_000, (+ "the bulk ceiling is a deliberate contract value; changing "+ "it requires updating this pin"+ )++ error = _string_list_error(+ ["r"] * (_MAX_BULK_LIST_ITEMS + 1),+ "research_ids",+ max_items=_MAX_BULK_LIST_ITEMS,+ )++ assert isinstance(error, JSONResponse), (+ "an oversized download-bulk list passed validation and "+ "would drive an unbounded per-item query loop"+ )+ assert error.status_code == 400+++# ===========================================================================+# Route-level coverage of the max_items= wiring+# ===========================================================================+#+# Every test above calls ``_string_list_error`` directly, so nothing+# catches a route being handed the wrong ``max_items=`` -- e.g.+# ``check-downloads`` losing its default 1000-item ceiling by picking+# up ``_MAX_BULK_LIST_ITEMS``, or ``download-bulk`` losing its higher+# ceiling by falling back to the 1000-item default and breaking the+# locked download-all contract. These call the route handlers+# themselves, the same way+# ``tests/web/routers/test_library_api_response_shapes.py`` does (see+# its ``_post_json_request`` and ``_patched`` helpers), so a+# ``max_items=`` wiring change at the call site shows up here.+++def _post_json_request(payload, path="/library/api/x"):+ """Build a POST request carrying *payload* as its JSON body.++ Adapted from the same-named helper in+ ``tests/web/routers/test_library_api_response_shapes.py``, which+ calls these route handlers directly the same way.+ """+ body = json.dumps(payload).encode("utf-8")++ async def receive():+ return {"type": "http.request", "body": body, "more_body": False}++ return Request(+ {+ "type": "http",+ "method": "POST",+ "path": path,+ "headers": [(b"content-type", b"application/json")],+ "query_string": b"",+ "session": {"session_id": "sid"},+ },+ receive,+ )+++@contextmanager+def _fake_db_session(*_args, **_kwargs):+ yield MagicMock()+++class TestRouteLevelCapWiring:+ def test_check_downloads_rejects_one_thousand_and_one_urls(self):+ """check-downloads must keep the 1000-item manual-selection cap.++ Fails if the route's ``_string_list_error(urls, "urls")`` call+ is ever changed to pass a higher (or no) ``max_items``, which+ would let more than 1000 urls through to the query layer.+ """+ with patch.object(+ library_module,+ "get_user_db_session",+ side_effect=_fake_db_session,+ ) as get_session:+ response = asyncio.run(+ check_downloads(+ _post_json_request(+ {"research_id": "r1", "urls": ["u"] * 1001},+ path="/library/api/check-downloads",+ ),+ username="alice",+ )+ )++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "error": "urls must contain at most 1000 items"+ }+ get_session.assert_not_called()++ def test_mark_redownload_rejects_one_thousand_and_one_document_ids(self):+ """mark-redownload must keep the 1000-item manual-selection cap.++ Fails if the route's+ ``_string_list_error(document_ids, "document_ids")`` call is+ ever changed to pass a higher (or no) ``max_items``, which+ would let more than 1000 document_ids through to the service.+ """+ service = Mock()+ run_db_sync = Mock(side_effect=AssertionError("database work started"))++ with (+ patch.object(+ library_module, "LibraryService", return_value=service+ ),+ patch.object(library_module, "run_db_sync", run_db_sync),+ ):+ response = asyncio.run(+ mark_for_redownload(+ _post_json_request(+ {"document_ids": ["d"] * 1001},+ path="/library/api/mark-redownload",+ ),+ username="alice",+ )+ )++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "error": "document_ids must contain at most 1000 items"+ }+ run_db_sync.assert_not_called()+ service.mark_for_redownload.assert_not_called()++ def test_download_bulk_accepts_one_thousand_and_one_research_ids(self):+ """download-bulk must keep its higher bulk ceiling.++ Fails if the route's call is ever repointed at the 1000-item+ manual-selection default: the locked download-all contract+ (``test_library_api_response_shapes``) requires lists above+ one thousand to stream (200, SSE), not 400.+ """+ response = asyncio.run(+ download_bulk(+ _post_json_request(+ {"research_ids": ["r"] * 1001},+ path="/library/api/download-bulk",+ ),+ username="alice",+ )+ )++ assert response.status_code == 200+ assert response.media_type == "text/event-stream"++ def test_download_bulk_rejects_one_hundred_thousand_and_one_research_ids(+ self,+ ):+ """download-bulk must still reject past its own (higher) ceiling.++ Fails if the route's call ever drops ``max_items=`` (unbounded)+ or is repointed at a looser constant than the 100,000-item+ bulk ceiling.+ """+ response = asyncio.run(+ download_bulk(+ _post_json_request(+ {"research_ids": ["r"] * 100_001},+ path="/library/api/download-bulk",+ ),+ username="alice",+ )+ )++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "error": "research_ids must contain at most 100000 items"+ }