Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): bound library string-list parameters (1000 items, 2048 chars; 100,000 for download-bulk) (#6428)

LearningCircuit702cdb3e9a10
diff --git a/changelog.d/6428.security.md b/changelog.d/6428.security.md
new file mode 100644
index 0000000000..7c204a304e
--- /dev/null
+++ b/changelog.d/6428.security.md
@@ -0,0 +1 @@
+The `urls`/`research_ids`/`document_ids` list parameters on `/api/check-downloads`, `/api/download-bulk`, and `/api/mark-redownload` are now capped (1000 items and 2048 characters per item; download-bulk keeps a higher 100,000-item ceiling so the download-all flow still streams libraries above one thousand entries). The shared shape-only validator let an oversized list through the routes' flat 16 MB JSON body allowance, which fits millions of short items. On `check-downloads`, that reached the `.in_()` SQL filter and — on the shipped build, where SQLite accepts at most 250,000 bind parameters (a Debian compile-time value; upstream SQLite defaults to 32,766) — turned the request into an unhandled `sqlalchemy.exc.OperationalError` ("too many SQL variables", HTTP 500). `download-bulk` and `mark-redownload` don't use `.in_()`; there, the same oversized list instead drove an unbounded per-item query/session loop. Oversized input now returns a 400 before any query runs; honest lists are unaffected.
diff --git a/src/local_deep_research/web/routers/library.py b/src/local_deep_research/web/routers/library.py
index 38dbad2314..652b6700cf 100644
--- a/src/local_deep_research/web/routers/library.py
+++ b/src/local_deep_research/web/routers/library.py
@@ -56,15 +56,69 @@ from typing import Annotated
router = APIRouter(prefix="/library", tags=["library"])
-def _string_list_error(value, field_name: str) -> JSONResponse | None:
- """Return a 400 response unless *value* is a non-empty-string list."""
- if not isinstance(value, list) or any(
- not isinstance(item, str) or not item.strip() for item in value
- ):
+#: Upper bound for user-supplied id/url lists on the manual-selection
+#: routes (check-downloads, mark-redownload). Far above any real bulk
+#: selection and a safe margin under SQLite's 250,000 bind-parameter
+#: ceiling, so an oversized list can never reach the .in_() filters.
+_MAX_STRING_LIST_ITEMS = 1000
+#: Higher bound for download-bulk only: the locked download-all
+#: contract (test_library_api_response_shapes) requires lists above
+#: one thousand to stream, so the functional floor sits above the
+#: manual cap. download_bulk never binds these ids into a SQL query —
+#: it loops per id, opening a session and queueing downloads for each
+#: — so the real cost axis is per-item work, not a bind-parameter
+#: ceiling. 100,000 bounds that per-item work to a size the "queue
+#: everything undownloaded" flow (queue-all-undownloaded's
+#: research_ids) can legitimately reach.
+_MAX_BULK_LIST_ITEMS = 100_000
+#: Upper bound per item; 2048 characters covers any URL or id the
+#: UI ever sends while bounding per-request memory.
+_MAX_STRING_LIST_ITEM_LENGTH = 2048
+
+
+def _string_list_error(
+ value, field_name: str, max_items: int = _MAX_STRING_LIST_ITEMS
+) -> JSONResponse | None:
+ """Return a 400 response unless *value* is a bounded non-empty-string list.
+
+ Caps both the item count and per-item length. Only
+ ``check-downloads`` feeds the list into a SQLAlchemy ``.in_()``
+ filter, whose bind-parameter ceiling this helper's default
+ ``max_items`` stays a safe margin under; ``download-bulk`` and
+ ``mark-redownload`` loop over the list per item instead, so their
+ exposure is unbounded per-item work rather than a bind-parameter
+ crash. Routes that must preserve larger lists (download-all) pass
+ a higher ``max_items`` sized to that per-item cost instead of the
+ bind ceiling.
+
+ The count check runs before the per-item type/length sweeps so an
+ oversized list 400s in O(1) instead of paying an O(n) scan first.
+ """
+ if not isinstance(value, list):
+ return JSONResponse(
+ {"error": f"{field_name} must be a list of non-empty strings"},
+ status_code=400,
+ )
+ if len(value) > max_items:
+ return JSONResponse(
+ {"error": (f"{field_name} must contain at most {max_items} items")},
+ status_code=400,
+ )
+ if any(not isinstance(item, str) or not item.strip() for item in value):
return JSONResponse(
{"error": f"{field_name} must be a list of non-empty strings"},
status_code=400,
)
+ if any(len(item) > _MAX_STRING_LIST_ITEM_LENGTH for item in value):
+ return JSONResponse(
+ {
+ "error": (
+ f"{field_name} items must be at most"
+ f" {_MAX_STRING_LIST_ITEM_LENGTH} characters"
+ )
+ },
+ status_code=400,
+ )
return None
@@ -1033,7 +1087,15 @@ async def download_bulk(
return JSONResponse(
{"error": "No research IDs provided"}, status_code=400
)
- if error := _string_list_error(research_ids, "research_ids"):
+ # Download-all selections legitimately exceed the manual-selection
+ # cap (locked contract: lists above one thousand must stream), so
+ # this route uses the higher bulk ceiling — bounded to a size the
+ # "queue everything undownloaded" flow can legitimately reach, not
+ # to a SQL bind-parameter limit (this route loops per id rather
+ # than binding the list into a query).
+ if error := _string_list_error(
+ research_ids, "research_ids", max_items=_MAX_BULK_LIST_ITEMS
+ ):
return error
if not isinstance(mode, str) or mode not in {"pdf", "text_only"}:
return JSONResponse(
diff --git a/tests/security/test_string_list_route_caps.py b/tests/security/test_string_list_route_caps.py
new file mode 100644
index 0000000000..cbc43b091d
--- /dev/null
+++ b/tests/security/test_string_list_route_caps.py
@@ -0,0 +1,328 @@
+"""Caps for user-supplied string-list parameters on library routes.
+
+``_string_list_error`` is the shared validation seam for the
+``urls``/``research_ids``/``document_ids`` list parameters accepted by
+``/api/check-downloads``, ``/api/download-bulk``, and
+``/api/mark-redownload``. It checked shape only — a list of non-empty
+strings — with no count or per-item length bound.
+
+Only ``check-downloads`` feeds its list into a SQLAlchemy ``.in_(...)``
+filter; ``download-bulk`` and ``mark-redownload`` loop over the list
+per item instead. The routes' flat 16 MB JSON body allowance fits
+millions of short items, while the shipped SQLite build accepts at
+most 250,000 bind parameters in one query (a Debian compile-time
+value — upstream SQLite defaults to 32,766). An authenticated user
+could therefore post an oversized list and either turn a
+``check-downloads`` request into an unhandled
+``sqlalchemy.exc.OperationalError`` ("too many SQL variables", HTTP
+500), or drive unbounded per-item query/session churn on the other two
+routes. Real flows never approach the cap: a research run has at most
+a few hundred sources and bulk selections are dozens.
+
+These tests pin the caps: overlong lists and overlong items are
+rejected with a 400 before any query runs, while honest lists keep
+passing.
+"""
+
+import asyncio
+import json
+from contextlib import contextmanager
+from unittest.mock import MagicMock, Mock, patch
+
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+from local_deep_research.web.routers import library as library_module
+from local_deep_research.web.routers.library import (
+ _string_list_error,
+ check_downloads,
+ download_bulk,
+ mark_for_redownload,
+)
+
+#: Contract values. 1000 items is far above any real bulk selection and
+#: far below SQLite's 250,000 bind-parameter ceiling on this build.
+CONTRACT_MAX_ITEMS = 1000
+#: 2048 characters covers any URL or id the UI ever sends.
+CONTRACT_MAX_ITEM_LENGTH = 2048
+
+
+def test_oversized_list_is_rejected():
+ """More items than the cap must 400, not reach the query layer."""
+ error = _string_list_error(["x"] * (CONTRACT_MAX_ITEMS + 1), "urls")
+
+ assert isinstance(error, JSONResponse), (
+ "oversized urls list passed validation and would reach the "
+ "route's query layer unbounded"
+ )
+ assert error.status_code == 400
+ assert "at most" in json.loads(error.body)["error"]
+
+
+def test_oversized_item_is_rejected():
+ """Per-item length is bounded alongside the count."""
+ error = _string_list_error(["x" * (CONTRACT_MAX_ITEM_LENGTH + 1)], "urls")
+
+ assert isinstance(error, JSONResponse), (
+ "an over-long list item passed validation unbounded"
+ )
+ assert error.status_code == 400
+
+
+def test_honest_lists_still_pass():
+ """Lists within both caps keep validating exactly as before."""
+ # Empty-list rejection lives in the routes (``not urls``) before
+ # the helper runs; the helper itself accepts [] — unchanged.
+ assert _string_list_error([], "urls") is None
+ assert _string_list_error(["a", ""], "urls") is not None
+ assert _string_list_error(["a", "b"], "urls") is None
+ assert _string_list_error(["u"] * CONTRACT_MAX_ITEMS, "urls") is None
+
+
+def test_caps_stay_under_the_sqlite_bind_limit():
+ """The implementation's cap must remain a safe margin under 250k.
+
+ Structural bound so the invariant survives future cap tuning; the
+ constant only exists once the fix lands, hence the local import.
+ """
+ from local_deep_research.web.routers.library import ( # noqa: PLC0415
+ _MAX_STRING_LIST_ITEMS,
+ )
+
+ assert _MAX_STRING_LIST_ITEMS <= 250_000 // 100, (
+ "item cap crept toward the SQLite bind-parameter ceiling; "
+ "keep it a real bound, not a rounding error away from 500s"
+ )
+
+
+class TestBulkCeilingPreservesDownloadAll:
+ """download-bulk's higher ceiling: the download-all contract holds.
+
+ The locked contract in ``test_library_api_response_shapes`` requires
+ ``download_bulk`` to stream lists above one thousand items; the
+ manual-selection cap must not apply there. These pins keep the bulk
+ ceiling honest from both sides: above the functional floor, far
+ below the bind-parameter crash threshold.
+ """
+
+ def test_bulk_lists_above_the_manual_cap_validate(self):
+ """1001 ids — the download-all contract size — must pass."""
+ from local_deep_research.web.routers.library import ( # noqa: PLC0415
+ _MAX_BULK_LIST_ITEMS,
+ )
+
+ assert (
+ _string_list_error(
+ ["r"] * 1001,
+ "research_ids",
+ max_items=_MAX_BULK_LIST_ITEMS,
+ )
+ is None
+ )
+
+ def test_bulk_ceiling_is_above_the_contract_floor(self):
+ """>1000 streaming is structural, not incidental."""
+ from local_deep_research.web.routers.library import ( # noqa: PLC0415
+ _MAX_BULK_LIST_ITEMS,
+ )
+
+ assert _MAX_BULK_LIST_ITEMS > 1000, (
+ "download-all contract requires >1000 items to stream; the "
+ "bulk ceiling dropped to or below the functional floor"
+ )
+
+ def test_bulk_ceiling_stays_under_the_bind_limit(self):
+ """Even the raised cap keeps a real margin under 250k binds."""
+ from local_deep_research.web.routers.library import ( # noqa: PLC0415
+ _MAX_BULK_LIST_ITEMS,
+ )
+
+ assert _MAX_BULK_LIST_ITEMS <= 250_000 // 2, (
+ "bulk item cap crept toward the SQLite bind-parameter "
+ "ceiling; keep it a real bound, not a rounding error "
+ "away from 500s"
+ )
+
+ def test_oversized_bulk_list_is_still_rejected(self):
+ from local_deep_research.web.routers.library import ( # noqa: PLC0415
+ _MAX_BULK_LIST_ITEMS,
+ )
+
+ # The payload below scales with _MAX_BULK_LIST_ITEMS, so on its
+ # own this test can never catch the constant moving -- "one more
+ # than the ceiling" is rejected by construction no matter what
+ # the ceiling is. Pin the actual contract value: 100_000 is the
+ # deliberate ceiling, not just whatever the constant currently
+ # says, so a change to it (e.g. loosening to 125_000) must also
+ # update this literal.
+ assert _MAX_BULK_LIST_ITEMS == 100_000, (
+ "the bulk ceiling is a deliberate contract value; changing "
+ "it requires updating this pin"
+ )
+
+ error = _string_list_error(
+ ["r"] * (_MAX_BULK_LIST_ITEMS + 1),
+ "research_ids",
+ max_items=_MAX_BULK_LIST_ITEMS,
+ )
+
+ assert isinstance(error, JSONResponse), (
+ "an oversized download-bulk list passed validation and "
+ "would drive an unbounded per-item query loop"
+ )
+ assert error.status_code == 400
+
+
+# ===========================================================================
+# Route-level coverage of the max_items= wiring
+# ===========================================================================
+#
+# Every test above calls ``_string_list_error`` directly, so nothing
+# catches a route being handed the wrong ``max_items=`` -- e.g.
+# ``check-downloads`` losing its default 1000-item ceiling by picking
+# up ``_MAX_BULK_LIST_ITEMS``, or ``download-bulk`` losing its higher
+# ceiling by falling back to the 1000-item default and breaking the
+# locked download-all contract. These call the route handlers
+# themselves, the same way
+# ``tests/web/routers/test_library_api_response_shapes.py`` does (see
+# its ``_post_json_request`` and ``_patched`` helpers), so a
+# ``max_items=`` wiring change at the call site shows up here.
+
+
+def _post_json_request(payload, path="/library/api/x"):
+ """Build a POST request carrying *payload* as its JSON body.
+
+ Adapted from the same-named helper in
+ ``tests/web/routers/test_library_api_response_shapes.py``, which
+ calls these route handlers directly the same way.
+ """
+ body = json.dumps(payload).encode("utf-8")
+
+ async def receive():
+ return {"type": "http.request", "body": body, "more_body": False}
+
+ return Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": path,
+ "headers": [(b"content-type", b"application/json")],
+ "query_string": b"",
+ "session": {"session_id": "sid"},
+ },
+ receive,
+ )
+
+
+@contextmanager
+def _fake_db_session(*_args, **_kwargs):
+ yield MagicMock()
+
+
+class TestRouteLevelCapWiring:
+ def test_check_downloads_rejects_one_thousand_and_one_urls(self):
+ """check-downloads must keep the 1000-item manual-selection cap.
+
+ Fails if the route's ``_string_list_error(urls, "urls")`` call
+ is ever changed to pass a higher (or no) ``max_items``, which
+ would let more than 1000 urls through to the query layer.
+ """
+ with patch.object(
+ library_module,
+ "get_user_db_session",
+ side_effect=_fake_db_session,
+ ) as get_session:
+ response = asyncio.run(
+ check_downloads(
+ _post_json_request(
+ {"research_id": "r1", "urls": ["u"] * 1001},
+ path="/library/api/check-downloads",
+ ),
+ username="alice",
+ )
+ )
+
+ assert response.status_code == 400
+ assert json.loads(response.body) == {
+ "error": "urls must contain at most 1000 items"
+ }
+ get_session.assert_not_called()
+
+ def test_mark_redownload_rejects_one_thousand_and_one_document_ids(self):
+ """mark-redownload must keep the 1000-item manual-selection cap.
+
+ Fails if the route's
+ ``_string_list_error(document_ids, "document_ids")`` call is
+ ever changed to pass a higher (or no) ``max_items``, which
+ would let more than 1000 document_ids through to the service.
+ """
+ service = Mock()
+ run_db_sync = Mock(side_effect=AssertionError("database work started"))
+
+ with (
+ patch.object(
+ library_module, "LibraryService", return_value=service
+ ),
+ patch.object(library_module, "run_db_sync", run_db_sync),
+ ):
+ response = asyncio.run(
+ mark_for_redownload(
+ _post_json_request(
+ {"document_ids": ["d"] * 1001},
+ path="/library/api/mark-redownload",
+ ),
+ username="alice",
+ )
+ )
+
+ assert response.status_code == 400
+ assert json.loads(response.body) == {
+ "error": "document_ids must contain at most 1000 items"
+ }
+ run_db_sync.assert_not_called()
+ service.mark_for_redownload.assert_not_called()
+
+ def test_download_bulk_accepts_one_thousand_and_one_research_ids(self):
+ """download-bulk must keep its higher bulk ceiling.
+
+ Fails if the route's call is ever repointed at the 1000-item
+ manual-selection default: the locked download-all contract
+ (``test_library_api_response_shapes``) requires lists above
+ one thousand to stream (200, SSE), not 400.
+ """
+ response = asyncio.run(
+ download_bulk(
+ _post_json_request(
+ {"research_ids": ["r"] * 1001},
+ path="/library/api/download-bulk",
+ ),
+ username="alice",
+ )
+ )
+
+ assert response.status_code == 200
+ assert response.media_type == "text/event-stream"
+
+ def test_download_bulk_rejects_one_hundred_thousand_and_one_research_ids(
+ self,
+ ):
+ """download-bulk must still reject past its own (higher) ceiling.
+
+ Fails if the route's call ever drops ``max_items=`` (unbounded)
+ or is repointed at a looser constant than the 100,000-item
+ bulk ceiling.
+ """
+ response = asyncio.run(
+ download_bulk(
+ _post_json_request(
+ {"research_ids": ["r"] * 100_001},
+ path="/library/api/download-bulk",
+ ),
+ username="alice",
+ )
+ )
+
+ assert response.status_code == 400
+ assert json.loads(response.body) == {
+ "error": "research_ids must contain at most 100000 items"
+ }