~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(research): bound request-body values in the start-research log lines (#6756)
Sebastian Cao73bfacab61bb
diff --git a/changelog.d/+research-log-preview-bound.security.md b/changelog.d/+research-log-preview-bound.security.mdnew file mode 100644index 0000000000..ceea3d74ca--- /dev/null+++ b/changelog.d/+research-log-preview-bound.security.md@@ -0,0 +1 @@+The research router's start-research log lines, and the `save_research_strategy` service call they make, now bound the request-body values they include (the body-supplied `model`, the posted key list and the other raw fields logged on that path) through the log-value preview #6191 added for the notes router, instead of building their full `str()` on every request. Four of those lines are emitted at the default INFO level, so this also caps what reaches the log sinks.diff --git a/src/local_deep_research/web/routers/research.py b/src/local_deep_research/web/routers/research.pyindex 6e764adafc..6ba5ddc212 100644--- a/src/local_deep_research/web/routers/research.py+++ b/src/local_deep_research/web/routers/research.py@@ -19,6 +19,9 @@ from ..dependencies.threadpool import ()from ..template_config import templates+# Bounded preview for untrusted request-body values in log lines (#6191).+from .notes import _log_value_preview+import asyncioimport jsonimport re@@ -206,7 +209,13 @@ def _extract_research_params(data, settings_manager):f"No model_provider in request, using database setting: {model_provider}")else:- logger.debug(f"Using model_provider from request: {model_provider}")+ # Raw request-body value: log it through the bounded preview so a+ # wide dict/list posted for it does not cost a full repr per+ # request (#6305, same shape as #6191). Same for ``model`` below.+ logger.debug(+ "Using model_provider from request: {}",+ _log_value_preview(model_provider),+ )# Normalize provider to lowercase canonical form (main fix #3348) —# the uppercase comparisons below would otherwise never match the# lowercase values stored in settings / sent by the UI, silently@@ -218,7 +227,7 @@ def _extract_research_params(data, settings_manager):model = settings_manager.get_setting("llm.model", None)logger.debug(f"No model in request, using database setting: {model}")else:- logger.debug(f"Using model from request: {model}")+ logger.debug("Using model from request: {}", _log_value_preview(model))custom_endpoint = Noneif model_provider == "openai_endpoint":@@ -799,14 +808,17 @@ async def start_research(def _start_research_sync(data: dict, username: str, base_url: str, session_id: str | None):- # Debug logging to trace model parameter- logger.debug(f"Request data keys: {list(data.keys())}")+ # Debug logging to trace model parameter. The key list is request+ # input too (a body may carry any number of keys of any length), so+ # it goes through the same bounded preview as the values below.+ logger.debug("Request data keys: {}", _log_value_preview(list(data.keys())))# Check if this is a news searchmetadata = data.get("metadata", {})if metadata.get("is_news_search"):logger.info(- f"News search request received: triggered_by={metadata.get('triggered_by', 'unknown')}"+ "News search request received: triggered_by={}",+ _log_value_preview(metadata.get("triggered_by", "unknown")),)query = data.get("query")@@ -890,17 +902,35 @@ def _start_research_sync(# (and, under rollback-journal SQLite, a shared database lock).db_session.rollback()- # Debug logging for model parameter specifically+ # Debug logging for model parameter specifically. ``model`` is an+ # unvalidated request-body value, so it must not be interpolated+ # eagerly: an f-string builds the full repr on every request+ # regardless of log level. The bounded preview keeps the cost and+ # the logged size capped (#6305, same shape as #6191).logger.debug(- f"Extracted model value: '{model}' (type: {type(model).__name__})"+ "Extracted model value: {} (type: {})",+ _log_value_preview(model),+ type(model).__name__,)- # Log the selections for troubleshooting+ # Log the selections for troubleshooting. These two lines are emitted+ # at the default INFO level, so the request-body values in them go+ # through the same bounded preview. ``max_results`` and ``time_period``+ # are validated at the request boundary (``validate_search_overrides``)+ # and are passed as they are.logger.info(- f"Starting research with provider: {model_provider}, model: {model}, search engine: {search_engine}"+ "Starting research with provider: {}, model: {}, search engine: {}",+ _log_value_preview(model_provider),+ _log_value_preview(model),+ _log_value_preview(search_engine),)logger.info(- f"Additional parameters: max_results={max_results}, time_period={time_period}, iterations={iterations}, questions={questions_per_iteration}, strategy={strategy}"+ "Additional parameters: max_results={}, time_period={}, iterations={}, questions={}, strategy={}",+ max_results,+ time_period,+ _log_value_preview(iterations),+ _log_value_preview(questions_per_iteration),+ _log_value_preview(strategy),)if not query:@@ -952,7 +982,8 @@ def _start_research_sync(if not model:logger.error(- f"No model specified or configured. Provider: {model_provider}"+ "No model specified or configured. Provider: {}",+ _log_value_preview(model_provider),)return JSONResponse({diff --git a/src/local_deep_research/web/services/research_service.py b/src/local_deep_research/web/services/research_service.pyindex c8f2e2a830..717b3b9aa4 100644--- a/src/local_deep_research/web/services/research_service.py+++ b/src/local_deep_research/web/services/research_service.py@@ -354,9 +354,20 @@ def save_research_strategy(research_id, strategy_name, *, username):the Flask session user (or fail off-request-context), socallers must state whose DB they mean."""+ # ``strategy_name`` is the raw request-body ``strategy`` (the research+ # router passes it through unvalidated), so it goes through the bounded+ # preview (#6191) here and on the info line below instead of an eager+ # f-string (#6305). Imported at call time, not at module level, so this+ # service does not pull the notes router (and FastAPI) into the CLI /+ # MCP import tree; same pattern as the rag router import in+ # research_library/services/download_service.py.+ from ..routers.notes import _log_value_preview+try:logger.debug(- f"save_research_strategy called with research_id={research_id}, strategy_name={strategy_name}"+ "save_research_strategy called with research_id={}, strategy_name={}",+ research_id,+ _log_value_preview(strategy_name),)with get_user_db_session(username) as session:# Check if a strategy already exists for this research@@ -384,7 +395,9 @@ def save_research_strategy(research_id, strategy_name, *, username):session.commit()logger.info(- f"Saved strategy '{strategy_name}' for research {research_id}"+ "Saved strategy {} for research {}",+ _log_value_preview(strategy_name),+ research_id,)except Exception:logger.exception("Error saving research strategy")diff --git a/tests/web/routers/test_research_model_log_preview_bound.py b/tests/web/routers/test_research_model_log_preview_bound.pynew file mode 100644index 0000000000..1bc95244ef--- /dev/null+++ b/tests/web/routers/test_research_model_log_preview_bound.py@@ -0,0 +1,404 @@+"""Regression tests for #6305: request-body values in the research router's+log lines must go through the bounded preview helper from #6191, not an+eager f-string.++``_extract_research_params`` and ``_start_research_sync``+(``web/routers/research.py``) interpolated the body-supplied ``model``+into eager f-strings at three points of one request (``Using model from+request``, ``Extracted model value`` and the ``Starting research with+provider ...`` info line), and the other raw body values logged on that+path (``model_provider``, ``search_engine``, ``iterations``,+``questions_per_iteration``, ``strategy``, the news-search+``triggered_by``, the body's key list on the ``Request data keys`` line)+had the same shape. So did ``save_research_strategy``+(``web/services/research_service.py``), which ``_start_research_sync``+calls synchronously with the same raw ``strategy`` and which f-stringed+it at DEBUG and INFO. An f-string is evaluated before loguru sees it, so+the FULL ``str()`` of the value was built on every request regardless of+log level, and the info lines are emitted at the default level into+every sink. For a wide dict/list posted as ``{"model": {...}}`` that is+full serialisation work per request, the same shape #6191 bounded for+the notes router's log-value previews.++Same policy as ``tests/notes/test_log_value_preview_dict_bound.py``:+``isinstance(preview, str)`` / ``len(preview) <= 100`` style assertions+are NOT regression guards (``reprlib`` bounds the output even when it+does the full work), so the tests below are written to fail under these+weakenings instead:++ (W1) un-wiring the fix at any call site: back to an eager f-string in+ ``_extract_research_params``, ``_start_research_sync`` or+ ``save_research_strategy``, or any other eager stringification+ of a body value added next to the bounded call (e.g.+ ``_audit = repr(search_engine)``)+ (W2) re-pointing the imported name at a naive helper (e.g.+ ``repr(value)[:100]``), which touches every element again+ (W3) settings router only: reintroducing the eager+ ``f"Converting checkbox ... {value}"`` message that #6201+ replaced with the value-free key/type/length line++No HTTP client / DB needed: the source guards run against the imported+modules directly, and the call-path probes mock the per-user session and+the settings manager at their import seams (the idiom of+``test_start_research_ssrf.py``), so everything stays fast and+deterministic. Every call-path probe feeds a wide value to EVERY body+field the function logs, so (W1) at any one of those sites, not just the+named ones, fails the probe and not only a source guard.+"""++import inspect+import json+from contextlib import contextmanager+from unittest.mock import MagicMock, patch++import pytest++from local_deep_research.web.routers import notes as notes_module+from local_deep_research.web.routers import research as research_module+from local_deep_research.web.routers import settings as settings_module+from local_deep_research.web.services import (+ research_service as research_service_module,+)++# Local (inside-function) import in ``_start_research_sync``: patch it at+# its source module, as ``test_start_research_ssrf.py`` does.+_SETTINGS_MANAGER = "local_deep_research.settings.SettingsManager"++# The eager f-string logs of request-body values the two functions used to+# carry (by their opening text), and the bounded calls that replaced them.+_EAGER_BODY_VALUE_LOGS = {+ "_extract_research_params": (+ 'f"Using model_provider from request',+ 'f"Using model from request',+ ),+ "_start_research_sync": (+ 'f"Request data keys',+ 'f"News search request received',+ 'f"Extracted model value',+ 'f"Starting research with provider',+ 'f"Additional parameters',+ 'f"No model specified or configured',+ ),+ "save_research_strategy": (+ 'f"save_research_strategy called',+ 'f"Saved strategy',+ ),+}+_BOUNDED_BODY_VALUE_CALLS = {+ "_extract_research_params": (+ "_log_value_preview(model_provider)",+ "_log_value_preview(model)",+ ),+ "_start_research_sync": (+ "_log_value_preview(list(data.keys()))",+ '_log_value_preview(metadata.get("triggered_by", "unknown"))',+ "_log_value_preview(model)",+ "_log_value_preview(model_provider)",+ "_log_value_preview(search_engine)",+ "_log_value_preview(iterations)",+ "_log_value_preview(questions_per_iteration)",+ "_log_value_preview(strategy)",+ ),+ "save_research_strategy": ("_log_value_preview(strategy_name)",),+}+++class CountingValue:+ """Dict/list element that counts how many times it is repr-ed.++ A direct *work* bound with no timing flakiness: an eager f-string (or+ a naive ``repr(value)[:100]``) stringifies every element, while the+ bounded preview walk hands ``reprlib`` at most 11 of them.+ """++ reprs = 0++ def __init__(self, n: int) -> None:+ self.n = n++ def __repr__(self) -> str:+ type(self).reprs += 1+ return f"V{self.n}"+++def _wide(n: int = 50_000) -> dict:+ """A wide dict body value: 50,000 entries, each counting its reprs."""+ return {i: CountingValue(i) for i in range(n)}+++class WideNameValue(dict):+ """A wide dict body value for the two fields the code treats as names.++ ``model_provider`` goes through ``normalize_provider`` (``.lower()``)+ and ``search_engine`` through the collection prechecks+ (``.startswith``) BEFORE either reaches its log lines, so a plain wide+ dict posted for them raises first and never exercises the log sites.+ This double duck-types the two calls the path makes on a name (as+ ``None``-ish results) so the real code carries it to the log lines+ unchanged. It is still a ``dict`` to the preview's bounding walk, which+ copies it into a plain 11-entry dict before ``reprlib`` sees it, so+ only an eager ``repr``/f-string of the value stringifies every entry.+ """++ def lower(self):+ return self++ def startswith(self, prefix):+ return False+++def _wide_name(n: int = 50_000) -> WideNameValue:+ return WideNameValue((i, CountingValue(i)) for i in range(n))+++def _settings_manager():+ """SettingsManager mock backed by a lookup table.++ Same idiom as ``test_start_research_ssrf.py``. The snapshot is NOT a+ dict, so ``_precheck_engine_policy`` skips (its documented behaviour+ for test doubles); the egress-policy precheck has its own suite.+ """+ lookup = {+ "llm.provider": "ollama",+ "llm.model": "gpt-4",+ "llm.ollama.url": "http://localhost:11434",+ "search.tool": "searxng",+ "search.iterations": 5,+ "search.questions_per_iteration": 5,+ "search.search_strategy": "source-based",+ }+ sm = MagicMock()+ sm.get_setting.side_effect = lambda key, default=None: lookup.get(+ key, default+ )+ sm.get_settings_snapshot.return_value = MagicMock()+ return sm+++class TestBodyValueLogSites:+ @pytest.mark.parametrize(+ "function_name",+ [+ "_extract_research_params",+ "_start_research_sync",+ "save_research_strategy",+ ],+ )+ def test_body_values_are_logged_through_the_bounded_preview(+ self, function_name+ ):+ """Catches (W1) at each call site: the wiring itself.++ Same style as ``test_async_handlers_offload.py``: the function's+ source must route every request-body value it logs through the+ helper, and none of the eager f-strings may come back.+ """+ module = (+ research_service_module+ if function_name == "save_research_strategy"+ else research_module+ )+ source = inspect.getsource(getattr(module, function_name))+ for eager in _EAGER_BODY_VALUE_LOGS[function_name]:+ assert eager not in source, (+ f"{function_name} interpolates a request-body value into an "+ f"eager f-string again ({eager}...) (#6305)"+ )+ for bounded in _BOUNDED_BODY_VALUE_CALLS[function_name]:+ assert bounded in source, (+ f"{function_name} no longer routes a request-body value "+ f"through the bounded log preview ({bounded}) (#6305)"+ )++ def test_research_binds_the_audited_notes_helper(self):+ """Catches (W2) via identity.++ The name research.py calls must BE the #6191 helper (whose bounds+ ``tests/notes/test_log_value_preview_dict_bound.py`` proves), not+ a lookalike local shim.+ """+ assert (+ research_module._log_value_preview+ is notes_module._log_value_preview+ )++ def test_wide_dict_model_value_is_not_fully_reprd(self):+ """Catches (W2): a work bound through the research-bound name.++ Measured: 10 element reprs with the bounded preview (reprlib+ renders one bounded 11-entry copy), 50,000 under an eager+ f-string or a naive ``repr(value)[:100]``.+ """+ wide = _wide()+ CountingValue.reprs = 0+ preview = research_module._log_value_preview(wide)+ assert CountingValue.reprs < 100+ # The preview must still record that entries were omitted.+ assert preview.endswith(", ...}")+ assert len(preview) <= 100++ def test_large_list_model_value_is_not_fully_reprd(self):+ """Catches (W2) for the list shape of the same body value."""+ big = [CountingValue(i) for i in range(50_000)]+ CountingValue.reprs = 0+ preview = research_module._log_value_preview(big)+ assert CountingValue.reprs < 100+ assert preview.endswith(", ...]")+ assert len(preview) <= 100++ def test_extract_research_params_does_not_repr_every_model_entry(self):+ """Catches (W1) and (W2) on the extraction path, through the real+ call.++ Measured: 10 element reprs with the bounded preview, 50,000 with+ the eager ``Using model from request`` f-string.+ """+ wide = _wide()+ CountingValue.reprs = 0+ params = research_module._extract_research_params(+ {"model": wide}, _settings_manager()+ )+ assert params["model"] is wide+ assert CountingValue.reprs < 100++ def test_start_research_sync_does_not_repr_every_body_entry(self):+ """Catches (W1) and (W2) at every ``_start_research_sync`` value+ site at once, through the real call path.++ The body carries a 50,000-entry dict for EVERY raw value the+ function logs (``model``, logged three times along the path,+ ``model_provider`` twice, ``search_engine``, the three unvalidated+ ``Additional parameters`` values and the news-search+ ``triggered_by``; the two name fields as ``WideNameValue`` so the+ name-shaped code before the log lines lets them through) and no+ ``query``, so the call runs every log line and then returns the+ 400 for the missing query. Measured: 100 element reprs with the+ bounded preview (10 per log of a wide value), 450,000 with the+ eager f-strings; one eager stringification of any single one of+ those values (e.g. ``_audit = repr(search_engine)``) adds 50,000.+ """+ body = {+ "model_provider": _wide_name(),+ "model": _wide(),+ "search_engine": _wide_name(),+ "iterations": _wide(),+ "questions_per_iteration": _wide(),+ "strategy": _wide(),+ "metadata": {"is_news_search": True, "triggered_by": _wide()},+ }++ @contextmanager+ def _session_ctx(*args, **kwargs):+ yield MagicMock()++ CountingValue.reprs = 0+ with (+ patch.object(research_module, "get_user_db_session", _session_ctx),+ patch(_SETTINGS_MANAGER, return_value=_settings_manager()),+ ):+ response = research_module._start_research_sync(+ body, "testuser", "http://testserver/", None+ )++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "status": "error",+ "message": "Query is required",+ }+ assert CountingValue.reprs < 200++ def test_start_research_sync_does_not_repr_every_body_key(self):+ """Catches (W1) and (W2) at the ``Request data keys`` line, the+ first line of ``_start_research_sync``, through the real call.++ The body has 50,000 keys (each counting its reprs) and no+ ``query``; the key list is the only wide thing in it, so the count+ isolates that one line. Measured: 10 element reprs with the+ bounded preview, 50,000 with the eager+ ``f"Request data keys: {list(data.keys())}"``.+ """+ body = {CountingValue(i): None for i in range(50_000)}++ @contextmanager+ def _session_ctx(*args, **kwargs):+ yield MagicMock()++ CountingValue.reprs = 0+ with (+ patch.object(research_module, "get_user_db_session", _session_ctx),+ patch(_SETTINGS_MANAGER, return_value=_settings_manager()),+ ):+ response = research_module._start_research_sync(+ body, "testuser", "http://testserver/", None+ )++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "status": "error",+ "message": "Query is required",+ }+ assert CountingValue.reprs < 100++ def test_save_research_strategy_does_not_repr_every_strategy_entry(+ self, loguru_caplog+ ):+ """Catches (W1) and (W2) at both ``save_research_strategy`` sites,+ through the real call.++ ``_start_research_sync`` hands this function the same raw body+ ``strategy`` it just logged, so a wide dict reaches its DEBUG and+ INFO lines. The per-user session and the ORM row are patched at+ the module seams (the idiom of the research_service suites). The+ function swallows every exception, so the captured ``Saved+ strategy`` message is what proves both log lines actually ran.+ Measured: 20 element reprs with the bounded preview (10 per log),+ 100,000 with the eager f-strings.+ """+ wide = _wide()+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ None+ )++ @contextmanager+ def _session_ctx(*args, **kwargs):+ yield session++ CountingValue.reprs = 0+ with (+ loguru_caplog.at_level("DEBUG"),+ patch.object(+ research_service_module, "get_user_db_session", _session_ctx+ ),+ patch.object(research_service_module, "ResearchStrategy"),+ ):+ research_service_module.save_research_strategy(+ "research-1", wide, username="testuser"+ )++ session.commit.assert_called_once()+ assert "Saved strategy" in loguru_caplog.text+ assert "Error saving research strategy" not in loguru_caplog.text+ assert CountingValue.reprs < 100+++class TestSettingsCheckboxLogSite:+ def test_checkbox_conversion_log_stays_value_free(self):+ """Catches (W3): locks in the already-landed settings fix.++ #6305's other named site. ``_save_all_settings_sync``'s checkbox+ conversion message must keep logging only key, type name and+ length (via loguru brace args), never the submitted value: a+ checkbox row can carry a credential-bearing container, so even a+ bounded preview of the VALUE would disclose too much here (#6201).+ """+ source = inspect.getsource(settings_module._save_all_settings_sync)+ assert 'f"Converting checkbox' not in source, (+ "_save_all_settings_sync interpolates the submitted checkbox "+ "value into an eager f-string again (#6305)"+ )+ assert "Converting checkbox {} from {} (len={}) to bool" in source, (+ "the value-free checkbox conversion message changed; make sure "+ "any replacement still logs only key/type/length, via loguru "+ "brace args (#6201, #6305)"+ )