Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(logging): omit queries from empty search events (#5646)

LearningCircuit762b02f94c3d
diff --git a/.pre-commit-hooks/check-sensitive-logging.py b/.pre-commit-hooks/check-sensitive-logging.py
index a194d73557..8f2dbc0c1e 100755
--- a/.pre-commit-hooks/check-sensitive-logging.py
+++ b/.pre-commit-hooks/check-sensitive-logging.py
@@ -10,7 +10,9 @@ wraps user-facing exceptions should omit ``from e`` to break the chain.
"""
import ast
+import io
import sys
+import tokenize
from pathlib import Path
from typing import List, Optional
@@ -125,6 +127,87 @@ ALLOWED_LOGGING = [
]
+# ---------------------------------------------------------------------------
+# Search-query logging (#5646)
+# ---------------------------------------------------------------------------
+# The user's search query is free text they typed: it is the most identifying
+# thing that flows through the search stack, and every log sink (stderr, the
+# log database, the frontend log view) is shared across research sessions.
+# ``BaseSearchEngine.run()`` no longer interpolates it into its empty-result
+# message, but ``run()`` only reaches that line *after* calling
+# ``_get_previews()`` — so an engine that logs the query on entry, or in its
+# own empty-result warning, undoes the fix for that engine.
+#
+# Scope is the whole ``web_search_engines/`` package, at shared log levels.
+# An earlier version of this rule keyed off the *lexically enclosing function
+# name* (``_get_previews`` / ``_get_search_results``) plus the
+# production-visible levels. That is defeated by adding a frame: the engines'
+# preview implementations call ``_optimize_query_for_*``, ``_simplify_query``,
+# ``_adaptive_search`` and friends, and every one of those helpers was exempt
+# by construction while sitting squarely on the empty-search path. Scoping by
+# call graph is not something an AST-per-file hook can do, so the scope is the
+# package instead.
+#
+# Query text is never needed here: the engine name and a result count say what
+# happened, and the query itself is still persisted per search by
+# ``SearchTracker.record_search`` → ``SearchCall`` in the metrics database.
+#
+# What this rule does *not* catch, deliberately or otherwise:
+#
+# * derived values reached through a *call* — ``len(query)``,
+# ``query.split()``. ``len(query)`` is genuinely non-reproducing;
+# ``query.split()`` reproduces the query modulo whitespace. This matcher
+# deliberately leaves generic calls unclassified; it does not attempt
+# to determine which calls preserve query text. Truncated slices
+# (``query[:100]``) *are* flagged: a Subscript over a query name is only
+# ever a piece of the user's text, and the shapes that would be false
+# positives — ``len(query)`` is a Call, ``query_counts[engine]``
+# subscripts a non-query name — are unaffected.
+# * the query reached through anything other than a bare ``query`` /
+# ``*_query`` name — ``params["q"]``, ``self.query``, ``str(query)``, or a
+# local called ``expanded`` / ``simplified`` / ``query_terms``. Exact-name
+# matching is what keeps ``query_url`` / ``query_params`` /
+# ``query_count`` out of the results.
+# * ``logger.trace()`` and ``logger.success()``: the shared
+# ``_is_logger_call`` level set this rule reuses does not list the two
+# loguru-only levels, so neither is inspected by any rule in this file.
+# * query logging outside ``web_search_engines/`` (~35 sites in
+# ``advanced_search_system/``, ``api.py`` and the news pipeline).
+SEARCH_ENGINE_DIRS = ("src/local_deep_research/web_search_engines/",)
+
+# Per-line escape hatch. The marker must open a real Python comment on the
+# physical line carrying the logging call's method name, and be followed by a
+# written justification of at least QUERY_LOG_REASON_MIN_CHARS non-space
+# characters, e.g.::
+#
+# logger.warning( # allow-search-query-log: operators match this against
+# # the upstream engine's own rejection log
+# f"Engine rejected query: {query}"
+# )
+QUERY_LOG_SUPPRESSION = "allow-search-query-log:"
+# The marker must open the comment (``# allow-search-query-log: ...``) and be
+# followed by a real sentence. A substring match anywhere in the comment let
+# ``# TODO: drop the allow-search-query-log: marker below`` — the comment a
+# developer writes while cleaning these up — silence the rule, and a
+# one-character reason such as ``.`` satisfied a bare "non-empty" test.
+QUERY_LOG_REASON_MIN_CHARS = 8
+
+# PEP 701 (Python 3.12+) lets a comment live inside the replacement field of a
+# multi-line f-string; tokenize reports it as a COMMENT even though it is part
+# of a string literal, so the f-string nesting level has to be tracked to keep
+# "marker text inside a string is not an exemption" true. Absent on older
+# interpreters, where no such token is ever emitted.
+FSTRING_START = getattr(tokenize, "FSTRING_START", -1)
+FSTRING_END = getattr(tokenize, "FSTRING_END", -2)
+
+# Levels every logger rule inspects. ``_is_logger_call`` and the
+# ``logger.bind(...)`` chain check must use the same set: when they drifted,
+# ``logger.bind(query=query).debug(...)`` was the one shape no rule saw.
+LOGGER_LEVEL_METHODS = frozenset(
+ {"info", "debug", "warning", "error", "critical", "exception", "log"}
+)
+
+
# The only sanctioned logger import inside SECURE_LOGGING_DIRS. Matched
# exactly (module + level), never by suffix, so lookalike modules such as
# ``notsecurity.secure_logging`` cannot spoof it.
@@ -183,7 +266,7 @@ WRAPPER_CHAIN_METHODS = {"bind"}
class SensitiveLoggingChecker(ast.NodeVisitor):
"""AST visitor to detect sensitive data in logging statements."""
- def __init__(self, filename: str):
+ def __init__(self, filename: str, source_lines: Optional[List[str]] = None):
self.filename = filename
self.errors: List[str] = []
# Track exception variable names from enclosing except handlers
@@ -199,6 +282,43 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
self.in_secure_dir = any(
d in filename.replace("\\", "/") for d in SECURE_LOGGING_DIRS
)
+ self.in_search_engine_dir = any(
+ d in filename.replace("\\", "/") for d in SEARCH_ENGINE_DIRS
+ )
+ # Only actual Python comments can justify a query-log exemption.
+ # source_lines must use physical LF boundaries, matching ast.parse.
+ # Tokenize once so marker text inside strings cannot silence a call.
+ self.query_log_suppression_lines: set[int] = set()
+ if self.in_search_engine_dir and source_lines:
+ try:
+ tokens = tokenize.generate_tokens(
+ io.StringIO("\n".join(source_lines)).readline
+ )
+ fstring_depth = 0
+ for token in tokens:
+ if token.type == FSTRING_START:
+ fstring_depth += 1
+ continue
+ if token.type == FSTRING_END:
+ fstring_depth = max(0, fstring_depth - 1)
+ continue
+ if token.type != tokenize.COMMENT or fstring_depth:
+ continue
+ # The marker has to open the comment, not merely appear
+ # somewhere in it, and the reason has to be written out.
+ text = token.string.lstrip("#").strip()
+ if not text.startswith(QUERY_LOG_SUPPRESSION):
+ continue
+ reason = text[len(QUERY_LOG_SUPPRESSION) :]
+ if (
+ len("".join(reason.split()))
+ < QUERY_LOG_REASON_MIN_CHARS
+ ):
+ continue
+ self.query_log_suppression_lines.add(token.start[0])
+ except (tokenize.TokenError, IndentationError):
+ # Incomplete source cannot justify an exemption.
+ self.query_log_suppression_lines.clear()
# Per-file wrapper-import hint. Depth is counted from the
# local_deep_research package root so the leading-dot count matches
# the file's actual position in the tree.
@@ -1133,11 +1253,11 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
# production-visible — run the exception-variable check.
if (
isinstance(func, ast.Attribute)
- and func.attr
- in {"info", "warning", "error", "critical", "exception", "log"}
+ and func.attr in LOGGER_LEVEL_METHODS
and self._is_wrapper_chain(func.value)
):
self._check_exception_var_in_log(node)
+ self._check_search_query_in_log(node)
def _is_wrapper_chain(self, expr: ast.AST) -> bool:
"""True for bind() call chains rooted at the name 'logger'."""
@@ -1158,6 +1278,7 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
self._check_sensitive_logging(node)
self._check_exc_info_on_warning(node)
self._check_exception_var_in_log(node)
+ self._check_search_query_in_log(node)
if self.in_secure_dir:
self._check_secure_dir_call(node)
@@ -1169,15 +1290,7 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
if isinstance(node.func, ast.Attribute):
# Check for logger.info, logger.debug, etc.
attr_name = node.func.attr
- if attr_name in {
- "info",
- "debug",
- "warning",
- "error",
- "critical",
- "exception",
- "log",
- }:
+ if attr_name in LOGGER_LEVEL_METHODS:
if isinstance(node.func.value, ast.Name):
return node.func.value.id == "logger"
if isinstance(node.func.value, ast.Attribute):
@@ -1224,6 +1337,175 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
f"use logger.exception() or logger.debug(..., exc_info=True) instead"
)
+ @staticmethod
+ def _is_search_query_name(expr: ast.AST) -> bool:
+ """True for the raw search query variable or a rewritten variant.
+
+ Matched by exact name, not substring: ``query``, ``optimized_query``,
+ ``simplified_query`` and friends carry the user's text, while
+ ``query_url``, ``query_params`` or ``query_count`` do not.
+ """
+ return isinstance(expr, ast.Name) and (
+ expr.id == "query" or expr.id.endswith("_query")
+ )
+
+ @classmethod
+ def _reproduced_query_name(cls, expr: ast.AST) -> Optional[str]:
+ """Name of the query this expression reproduces the text of, if any.
+
+ A bare ``query`` reproduces all of it; ``query[:100]`` reproduces the
+ first hundred characters, which is still the text the user typed.
+ Genuinely non-reproducing derivations stay out: ``len(query)`` and
+ ``query.split()`` are Calls, not Subscripts, and ``query_counts[...]``
+ subscripts a name that is not a query name to begin with.
+ """
+ if cls._is_search_query_name(expr):
+ return expr.id # type: ignore[attr-defined]
+ if isinstance(expr, ast.Subscript) and cls._is_search_query_name(
+ expr.value
+ ):
+ return expr.value.id # type: ignore[attr-defined]
+ return None
+
+ def _interpolated_query_names(self, expr: ast.AST) -> List[str]:
+ """Query names whose *value* this argument puts into the message.
+
+ Interpolating the name itself counts, and so does interpolating a
+ slice of it. ``len(query)`` and ``query.split()`` are not flagged --
+ the first does not reproduce the text at all, the second is a Call
+ the matcher does not look inside.
+ """
+ found = []
+ name = self._reproduced_query_name(expr)
+ if name:
+ found.append(name)
+ for sub in ast.walk(expr):
+ if isinstance(sub, ast.JoinedStr):
+ for value in sub.values:
+ if isinstance(value, ast.FormattedValue):
+ name = self._reproduced_query_name(value.value)
+ if name:
+ found.append(name)
+ elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Mod):
+ # "... %s" % query / "... %s %s" % (query, other)
+ operands = (
+ sub.right.elts
+ if isinstance(sub.right, ast.Tuple)
+ else [sub.right]
+ )
+ found.extend(
+ name
+ for name in map(self._reproduced_query_name, operands)
+ if name
+ )
+ elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Add):
+ # "... " + query (string concatenation)
+ found.extend(
+ name
+ for name in map(
+ self._reproduced_query_name, (sub.left, sub.right)
+ )
+ if name
+ )
+ elif (
+ isinstance(sub, ast.Call)
+ and isinstance(sub.func, ast.Attribute)
+ and sub.func.attr == "format"
+ ):
+ # "... {}".format(query) / .format(q=query)
+ found.extend(
+ name
+ for name in map(self._reproduced_query_name, sub.args)
+ if name
+ )
+ found.extend(
+ name
+ for name in map(
+ self._reproduced_query_name,
+ [k.value for k in sub.keywords],
+ )
+ if name
+ )
+ return found
+
+ def _bound_query_names(self, node: ast.Call) -> List[str]:
+ """Query names attached via a ``logger.bind(...)`` chain.
+
+ ``logger.bind(query=query).warning("empty")`` never puts the query in
+ the message, but loguru copies bind kwargs into ``record["extra"]``,
+ which structured sinks serialise — so it is a live sink and the check
+ has to look past the outer call it is invoked on.
+ """
+ found: List[str] = []
+ expr = node.func
+ expr = expr.value if isinstance(expr, ast.Attribute) else None
+ while (
+ isinstance(expr, ast.Call)
+ and isinstance(expr.func, ast.Attribute)
+ and expr.func.attr in WRAPPER_CHAIN_METHODS
+ ):
+ for arg in expr.args:
+ found.extend(self._interpolated_query_names(arg))
+ for keyword in expr.keywords:
+ found.extend(self._interpolated_query_names(keyword.value))
+ expr = expr.func.value
+ return found
+
+ def _query_log_suppressed(self, node: ast.Call) -> bool:
+ """True if the call carries a justified suppression comment.
+
+ The marker counts only on the physical line the call's method name
+ sits on. Accepting it anywhere in the call's span let a comment
+ written about an unrelated argument exempt the message: a reason
+ given for a ``logger.bind(engine=engine, # ...)`` keyword silenced
+ the ``.warning(f"... {query}")`` several lines below it, and the
+ package has eleven ``logger.bind(policy_audit=True).warning(`` sites
+ of exactly that shape.
+ """
+ func = node.func
+ lineno = node.lineno
+ if isinstance(func, ast.Attribute):
+ lineno = getattr(func, "end_lineno", None) or node.lineno
+ return lineno in self.query_log_suppression_lines
+
+ def _check_search_query_in_log(self, node: ast.Call) -> None:
+ """Flag search queries logged on the empty-search path (#5646).
+
+ See the SEARCH_ENGINE_DIRS block above for why the scope is the whole
+ ``web_search_engines/`` package at the shared log levels, rather than the
+ preview-path functions at the production-visible levels only.
+ """
+ if not self.in_search_engine_dir:
+ return
+
+ level = self._get_log_level(node)
+
+ names = []
+ for arg in node.args:
+ names.extend(self._interpolated_query_names(arg))
+ for keyword in node.keywords:
+ names.extend(self._interpolated_query_names(keyword.value))
+ names.extend(self._bound_query_names(node))
+ if not names:
+ return
+ if self._query_log_suppressed(node):
+ return
+
+ for name in sorted(set(names)):
+ self.errors.append(
+ f"{self.filename}:{node.lineno}: search query '{name}' "
+ f"interpolated into logger.{level}() under "
+ f"web_search_engines/ "
+ f"- BaseSearchEngine.run() no longer logs the query, so this "
+ f"puts it straight back into the log. Log the engine name and "
+ f"a result count instead; the query itself is still persisted "
+ f"per search by SearchTracker.record_search. If the query "
+ f"really is required here, open a comment on the "
+ f"logger.{level}() line with "
+ f"'# {QUERY_LOG_SUPPRESSION} <reason>' and a reason of at "
+ f"least {QUERY_LOG_REASON_MIN_CHARS} non-space characters"
+ )
+
def _check_exception_var_in_log(self, node: ast.Call) -> None:
"""Flag logging calls that interpolate the exception variable.
@@ -1517,7 +1799,9 @@ def check_file(filepath: Path) -> List[str]:
content = f.read()
tree = ast.parse(content, filename=str(filepath))
- checker = SensitiveLoggingChecker(str(filepath))
+ checker = SensitiveLoggingChecker(
+ str(filepath), source_lines=content.split("\n")
+ )
checker.visit(tree)
return checker.errors
diff --git a/changelog.d/5646.bugfix.md b/changelog.d/5646.bugfix.md
new file mode 100644
index 0000000000..892990ccfb
--- /dev/null
+++ b/changelog.d/5646.bugfix.md
@@ -0,0 +1,64 @@
+Search-engine entry, empty-result and query-rewriting messages now omit the
+explicit search text.
+`BaseSearchEngine.run()` already left the query out of its "returned no preview
+results" message, but it only reaches that line after the engine's own
+`_get_previews()` has run — and 20 engines logged the raw query on entry,
+several again in their own empty-result warning, so the same empty search wrote
+the query twice immediately before the sanitised message. The query-rewriting
+helpers those implementations call one frame down (PubMed query simplification
+and optimisation, Semantic Scholar and Guardian and Wikinews query
+optimisation, the Paperless LLM query expansion, the GitHub query builder,
+Wayback URL extraction) logged it too, on exactly the same path. All of those
+messages now carry the engine name, a result count and query lengths only, as
+do the two `Cannot save search metrics` warnings from the metrics recorder (one
+of which paired the query with your username).
+
+The same sweep covers the sites that reached the query through something other
+than a bare `query` variable, which the static rule below cannot see: PubMed's
+`Simplified query: {simplified}` (INFO, on the empty-result retry path), NASA
+ADS's `{str(params['q'])[:100]}` (INFO, in the same `_get_previews()` the entry
+log was fixed in), and Paperless's `{expanded[:500]}` / `{expanded[:200]}`
+expansion previews and its `query.lower().split()` term list. PubMed's
+historical-focus check logged the LLM's raw answer, and the prompt it answers
+embeds the query, so a model that ignored the "answer ONLY yes/no" instruction
+could echo the query back into the log; it now logs the derived boolean.
+Queries are still
+recorded per search in the metrics database, so "why did this search return
+nothing" stays answerable.
+
+A `check-sensitive-logging` pre-commit rule rejects recognized query expressions
+in log calls under `web_search_engines/`, at the shared logging levels and in
+any function — including via `"{}".format(query)`, `"..." + query`, a brace
+placeholder filled from a keyword argument, and `logger.bind(query=query)`
+chains at the same levels as direct calls, `debug` included — all of which
+loguru writes to a sink — and truncated slices such as `query[:100]`, which
+still reproduce the first hundred characters of what the user typed.
+
+Still not covered by that rule: logging that reaches the query through
+something other than a bare `query` / `*_query` variable (a dict entry, an
+attribute, `str(query)`, `query.split()` or any other call, a differently
+named local), `logger.trace()` / `logger.success()` — the two loguru-only
+levels the hook's shared level set omits — and the roughly 35
+query-interpolating log calls elsewhere in the codebase, mostly
+`advanced_search_system/`, `api.py` and the news pipeline, which this change
+does not touch. Three call sites in `web_search_engines/` did use those
+shapes and are fixed here: Semantic Scholar's last empty-result fallback
+logged `longest_word`, a differently named local holding the longest word of
+the query, Guardian overrides `run()` and logged a `self._search_metadata`
+dict whose `original_query` / `optimized_query` entries carried the query on
+every search that returned results, and PubChem's per-compound exception
+handler logged `name` — which, on the autocomplete-empty fallback, is the
+raw query itself, so a failure processing it wrote the query straight to the
+log. The static rule still cannot see a derived local or an attribute-carried
+value, which is why all three sites now also have a runtime test asserting
+that no word of a probe query reaches a captured log message.
+
+Exemptions require an actual Python comment that opens with the marker, gives
+a reason of at least eight non-space characters, and sits on the physical line
+carrying the logging call's method name — a reason written about one part of a
+multi-line call does not exempt the rest of it. Marker text inside a string is
+not an exemption, including a PEP 701 comment inside an f-string replacement
+field.
+This change removes explicit query logging at the covered sites; provider
+responses, result content and exceptions are not subject to complete query
+tracking by this static rule.
diff --git a/src/local_deep_research/metrics/search_tracker.py b/src/local_deep_research/metrics/search_tracker.py
index d5a3ad1451..ee52b41ecb 100644
--- a/src/local_deep_research/metrics/search_tracker.py
+++ b/src/local_deep_research/metrics/search_tracker.py
@@ -70,8 +70,8 @@ class SearchTracker:
username = context.get("username")
if not username:
logger.warning(
- f"Cannot save search metrics - no username in research context. "
- f"Search: {engine_name} for '{query}'"
+ "Cannot save search metrics - no username in research "
+ f"context. Search engine: {engine_name}"
)
return
@@ -79,8 +79,9 @@ class SearchTracker:
password = context.get("user_password")
if not password:
logger.warning(
- f"Cannot save search metrics - no password in research context. "
- f"Search: {engine_name} for '{query}', username: {username}"
+ "Cannot save search metrics - no password in research "
+ f"context. Search engine: {engine_name}, "
+ f"username: {username}"
)
return
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
index d941edc347..a8fc69fd8b 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
@@ -394,7 +394,7 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
logger.info(
f"Collection '{self.collection_name}' search returned "
- f"{len(results)} results for query: {query[:50]}..."
+ f"{len(results)} results"
)
return results
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
index 312eebaf00..8f1a6d687b 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
@@ -380,9 +380,7 @@ class ElasticsearchSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(
- f"Getting document previews from Elasticsearch with query: {query}"
- )
+ logger.info("Getting document previews from Elasticsearch")
try:
# Build the search query
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_github.py b/src/local_deep_research/web_search_engines/engines/search_engine_github.py
index b948afde39..6547ba56c0 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_github.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_github.py
@@ -168,7 +168,8 @@ class GitHubSearchEngine(BaseSearchEngine):
# Validate the optimized query
if optimized_query and len(optimized_query) > 0:
logger.info(
- f"LLM optimized query from '{query}' to '{optimized_query}'"
+ f"LLM optimized GitHub query "
+ f"({len(query)} -> {len(optimized_query)} chars)"
)
return optimized_query
logger.warning("LLM returned empty query, using original")
@@ -195,7 +196,7 @@ class GitHubSearchEngine(BaseSearchEngine):
# Optimize GitHub query using LLM
github_query = self._optimize_github_query(query)
- logger.info(f"Final GitHub query: {github_query}")
+ logger.info(f"Final GitHub query built ({len(github_query)} chars)")
# Construct search parameters
params = {
@@ -502,7 +503,7 @@ class GitHubSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting GitHub previews for query: {query}")
+ logger.info("Getting GitHub previews")
# For contribution-focused queries, automatically adjust search type and add filters
if any(
@@ -567,9 +568,7 @@ class GitHubSearchEngine(BaseSearchEngine):
keywords[:5]
) # Add up to 5 keywords
- logger.info(
- f"Using specialized contribution query: {specialized_query}"
- )
+ logger.info("Using specialized contribution query")
# Perform GitHub search with specialized query
results = self._search_github(specialized_query)
@@ -582,7 +581,7 @@ class GitHubSearchEngine(BaseSearchEngine):
results = self._search_github(query)
if not results:
- logger.warning(f"No GitHub results found for query: {query}")
+ logger.warning("No GitHub results found")
return []
# Format results as previews
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
index 2e67e95b1f..6f661c7494 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
@@ -228,18 +228,16 @@ class GooglePSESearchEngine(BaseSearchEngine):
self.retry_delay * (2 ** (attempt - 1)) * jitter
)
logger.info(
- "Retry attempt {} / {} for query '{}'. Waiting {} s",
+ "Retry attempt {} / {} for Google PSE query. Waiting {} s",
attempt + 1,
self.max_retries,
- query,
f"{sleep_time:.2f}",
)
time.sleep(sleep_time)
# Make the request
logger.debug(
- "Making request to Google PSE API: {} (start_index={})",
- query,
+ "Making request to Google PSE API (start_index={})",
start_index,
)
# Apply rate limiting before request
@@ -365,7 +363,5 @@ class GooglePSESearchEngine(BaseSearchEngine):
logger.warning(f"Error getting search results: {safe_msg}")
break
- logger.info(
- "Retrieved {} search results for query: '{}'", len(results), query
- )
+ logger.info("Retrieved {} search results", len(results))
return results
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_guardian.py b/src/local_deep_research/web_search_engines/engines/search_engine_guardian.py
index 18d4a03657..dcb48af1c8 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_guardian.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_guardian.py
@@ -105,7 +105,8 @@ class GuardianSearchEngine(BaseSearchEngine):
if len(query) > 150:
simple_query = " ".join(query.split()[:10])
logger.info(
- f"Query too long ({len(query)} chars), truncating to: {simple_query}"
+ f"Query too long ({len(query)} chars), truncated to "
+ f"{len(simple_query)} chars"
)
query = simple_query
@@ -163,8 +164,10 @@ Return ONLY the extremely brief search query.
):
optimized_query = optimized_query[1:-1]
- logger.info(f"Original query: '{query}'")
- logger.info(f"Optimized for Guardian: '{optimized_query}'")
+ logger.info(
+ f"Optimized query for Guardian "
+ f"({len(query)} -> {len(optimized_query)} chars)"
+ )
return optimized_query
@@ -351,7 +354,7 @@ ONE WORD ONLY:"""
)
# Log full parameters for debugging
- logger.info(f"Guardian API search query: '{query}'")
+ logger.info(f"Guardian API search request ({len(query)} chars)")
logger.info(
f"Guardian API date range: {self.from_date} to {self.to_date}"
)
@@ -374,6 +377,10 @@ ONE WORD ONLY:"""
# Log the complete request parameters (except API key)
log_params = params.copy()
log_params["api-key"] = "REDACTED"
+ # The user's search text is the "q" param — keep it out of the log
+ # the same way the API key is (#5646). The engine name and the
+ # other request parameters are all that's useful here.
+ log_params["q"] = "REDACTED"
logger.info(f"Guardian API request parameters: {log_params}")
# Apply rate limiting before request
@@ -451,9 +458,7 @@ ONE WORD ONLY:"""
Returns:
List of preview dictionaries
"""
- logger.info(
- f"Getting articles from The Guardian API for query: {query}"
- )
+ logger.info("Getting articles from The Guardian API")
# Step 1: Optimize the query using LLM
optimized_query = self._optimize_query_for_guardian(query)
@@ -464,10 +469,13 @@ ONE WORD ONLY:"""
# Step 3: Perform adaptive search
articles, strategy = self._adaptive_search(optimized_query)
- # Store search metadata for debugging
+ # Store search metadata for debugging. The two query strings this
+ # used to carry reached logger.info() in run() below, which is the
+ # success path of every Guardian search (#5646) — keep the lengths,
+ # which answer "was the query rewritten", and drop the text itself.
self._search_metadata = {
- "original_query": query,
- "optimized_query": optimized_query,
+ "original_query_length": len(query or ""),
+ "optimized_query_length": len(optimized_query or ""),
"strategy": strategy,
"from_date": self.from_date,
"to_date": self.to_date,
@@ -559,7 +567,7 @@ ONE WORD ONLY:"""
[w for w in query.split() if len(w) > 3][:3]
)
logger.warning(
- f"No Guardian articles found, trying simplified query: {simple_query}"
+ "No Guardian articles found, retrying with a simplified query"
)
previews = self._get_previews(simple_query)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_gutenberg.py b/src/local_deep_research/web_search_engines/engines/search_engine_gutenberg.py
index 27c3de9e1e..0c1b3d94cb 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_gutenberg.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_gutenberg.py
@@ -218,7 +218,7 @@ class GutenbergSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Gutenberg previews for query: {query}")
+ logger.info("Getting Gutenberg previews")
# Apply rate limiting
self._last_wait_time = self.rate_tracker.apply_rate_limit(
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
index 427292e079..dd92e880ea 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
@@ -517,9 +517,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
results.append(result)
- logger.info(
- f"Library RAG search returned {len(results)} results for query: {query}"
- )
+ logger.info(f"Library RAG search returned {len(results)} results")
return results
except Exception as e:
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
index c6fd53ed6f..e8f5927455 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
@@ -102,7 +102,7 @@ class MojeekSearchEngine(BaseSearchEngine):
Returns:
List of search result dicts
"""
- logger.info(f"Mojeek running search for query: {query}")
+ logger.info("Mojeek running search")
try:
params = {
@@ -169,9 +169,7 @@ class MojeekSearchEngine(BaseSearchEngine):
if results:
logger.info(f"Mojeek returned {len(results)} valid results")
else:
- logger.warning(
- f"Mojeek returned no valid results for query: {query}"
- )
+ logger.warning("Mojeek returned no valid results")
return results
@@ -194,12 +192,12 @@ class MojeekSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Mojeek previews for query: {query}")
+ logger.info("Getting Mojeek previews")
results = self._get_search_results(query)
if not results:
- logger.warning(f"No Mojeek results found for query: {query}")
+ logger.warning("No Mojeek results found")
return []
previews = []
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
index 9cd8ee0cd7..6bd45c6789 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_nasa_ads.py
@@ -126,7 +126,7 @@ class NasaAdsSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Searching NASA ADS for: {query}")
+ logger.info("Searching NASA ADS")
# Build the search query - NASA ADS has good natural language support
# We can use the query directly or enhance it slightly
@@ -187,7 +187,8 @@ class NasaAdsSearchEngine(BaseSearchEngine):
# Make the API request
logger.info(
- f"Making NASA ADS API request with query: {str(params['q'])[:100]}..."
+ f"Making NASA ADS API request "
+ f"({len(str(params['q']))} query chars)"
)
response = safe_get(
f"{self.api_base}/search/query",
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
index 4a97e06d92..ce46894339 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_openalex.py
@@ -300,7 +300,7 @@ class OpenAlexSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Searching OpenAlex for: {query}")
+ logger.info("Searching OpenAlex")
# Build the search URL with parameters
params = {
@@ -350,7 +350,9 @@ class OpenAlexSearchEngine(BaseSearchEngine):
# results: it retries keylessly once and drops the key for
# this engine's lifetime. The same helper backs the DOI
# enrichment pass and the research-library downloader.
- logger.info(f"Making OpenAlex API request with params: {params}")
+ logger.info(
+ f"Making OpenAlex API request with params: {sorted(params.keys())}"
+ )
# Read once. ``self.api_key`` is shared mutable state (another
# thread on this instance can drop it), and the retry decision
# below and the OpenAlexAuthError gate further down must agree
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_openlibrary.py b/src/local_deep_research/web_search_engines/engines/search_engine_openlibrary.py
index c401ea5dd9..7199c04be1 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_openlibrary.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_openlibrary.py
@@ -119,7 +119,7 @@ class OpenLibrarySearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Open Library previews for query: {query}")
+ logger.info("Getting Open Library previews")
# Apply rate limiting
self._last_wait_time = self.rate_tracker.apply_rate_limit(
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_paperless.py b/src/local_deep_research/web_search_engines/engines/search_engine_paperless.py
index 2a3664dc8e..1834322db2 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_paperless.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_paperless.py
@@ -139,7 +139,9 @@ class PaperlessSearchEngine(BaseSearchEngine):
url = urljoin(self.api_url or "", endpoint)
logger.debug(f"Making request to: {redact_url_for_log(url)}")
- logger.debug(f"Request params: {params}")
+ logger.debug(
+ f"Request params: {sorted(params.keys()) if params else None}"
+ )
logger.debug(
f"Headers: {self.headers.keys() if self.headers else 'None'}"
)
@@ -182,7 +184,8 @@ class PaperlessSearchEngine(BaseSearchEngine):
safe_msg = self._scrub_error(e)
logger.warning(f"Error making request to Paperless-ngx: {safe_msg}")
logger.debug(
- f"Failed URL: {redact_url_for_log(url)}, params: {params}"
+ f"Failed URL: {redact_url_for_log(url)}, "
+ f"params: {sorted(params.keys()) if params else None}"
)
return {}
@@ -197,9 +200,7 @@ class PaperlessSearchEngine(BaseSearchEngine):
Expanded query with keywords
"""
if not self.llm:
- logger.info(
- f"No LLM available for query expansion, using original: '{query}'"
- )
+ logger.info("No LLM available for query expansion, using original")
return query
try:
@@ -214,7 +215,7 @@ Include synonyms, plural forms, and technical terms.
IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
logger.debug(
- f"Sending query expansion prompt to LLM for: '{query}'"
+ f"Sending query expansion prompt to LLM ({len(query)} chars)"
)
response = self.llm.invoke(prompt)
expanded = (
@@ -223,9 +224,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
else str(response)
).strip()
- logger.debug(
- f"Raw LLM response (first 500 chars): {expanded[:500]}"
- )
+ logger.debug(f"Raw LLM response received ({len(expanded)} chars)")
# Clean up the response - remove any explanatory text
if "\n" in expanded:
@@ -234,11 +233,10 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
# Always trust the LLM's expansion - it knows better than hard-coded rules
logger.info(
- f"LLM expanded query from '{query}' to {len(expanded)} chars with {expanded.count('OR')} ORs"
- )
- logger.debug(
- f"Expanded query preview (first 200 chars): {expanded[:200]}..."
+ f"LLM expanded query from {len(query)} chars to {len(expanded)} "
+ f"chars with {expanded.count('OR')} ORs"
)
+ logger.debug(f"Expanded query is {len(expanded)} chars")
return expanded
except Exception as e:
@@ -256,7 +254,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
Returns:
Combined and deduplicated results
"""
- logger.info(f"Starting multi-pass search for query: '{query}'")
+ logger.info("Starting multi-pass search")
all_results = {} # Use dict to deduplicate by doc_id
# Pass 1: Original query
@@ -267,7 +265,8 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
}
logger.info(
- f"Pass 1 - Original query: '{query}' (max_results={self.max_results})"
+ f"Pass 1 - Original query ({len(query)} chars, "
+ f"max_results={self.max_results})"
)
response = self._make_request("/api/documents/", params=params)
@@ -296,9 +295,6 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
logger.info(
f"Pass 2 - Using expanded query with {expanded_query.count('OR')} ORs"
)
- logger.debug(
- f"Pass 2 - Full expanded query (first 500 chars): '{expanded_query[:500]}...'"
- )
logger.info(
f"Pass 2 - Max results set to: {params['page_size']}"
)
@@ -376,9 +372,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
else:
previews.append(doc_previews)
- logger.info(
- f"Found {len(previews)} documents in Paperless-ngx for query: {query}"
- )
+ logger.info(f"Found {len(previews)} documents in Paperless-ngx")
return previews
except Exception as e:
@@ -504,7 +498,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
query_terms = query.lower().split()
content_lower = content.lower()
logger.debug(
- f"Searching for query terms in content: {query_terms}"
+ f"Searching content for {len(query_terms)} query terms"
)
# Find first occurrence of any query term
@@ -514,7 +508,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
if pos != -1 and (best_pos == -1 or pos < best_pos):
best_pos = pos
logger.debug(
- f"Found term '{term}' at position {pos}"
+ f"Found a query term at position {pos}"
)
if best_pos != -1:
@@ -751,7 +745,7 @@ IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
if not filtered_previews:
logger.info(
f"LLM relevance filter returned no results "
- f"from {len(previews)} previews for query: {query}"
+ f"from {len(previews)} previews"
)
else:
filtered_previews = previews
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubchem.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubchem.py
index c73cd967ff..e212923584 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubchem.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubchem.py
@@ -248,7 +248,7 @@ class PubChemSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting PubChem previews for query: {query}")
+ logger.info("Getting PubChem previews")
# Apply rate limiting
self._last_wait_time = self.rate_tracker.apply_rate_limit(
@@ -271,7 +271,7 @@ class PubChemSearchEngine(BaseSearchEngine):
previews: list[dict[str, Any]] = []
seen_cids = set()
- for name in compound_names:
+ for index, name in enumerate(compound_names):
if len(previews) >= self.max_results:
break
@@ -356,9 +356,13 @@ class PubChemSearchEngine(BaseSearchEngine):
except RateLimitError:
raise
except Exception as e:
+ # `name` is a compound name and, on the autocomplete-empty
+ # fallback path (compound_names = [query] above), can be the
+ # raw user query — log its position instead (#5646).
safe_msg = self._scrub_error(e)
logger.exception(
- f"Error processing PubChem compound: {name} ({type(e).__name__}): {safe_msg}"
+ f"Error processing PubChem compound {index + 1}/"
+ f"{len(compound_names)} ({type(e).__name__}): {safe_msg}"
)
continue
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
index f531e76ebd..4164a4d562 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
@@ -157,9 +157,7 @@ class PubMedSearchEngine(BaseSearchEngine):
data = response.json()
count = int(data["esearchresult"]["count"])
- logger.info(
- "Query '{}' has {} total results in PubMed", query, count
- )
+ logger.info("PubMed reports {} total results for the query", count)
return count
except Exception as e:
@@ -392,11 +390,11 @@ Return ONLY the search query without any explanations.
self._simplify_query_cache = optimized_query
# Log original and optimized queries
- logger.info("Original query: '{}'", query)
- logger.info(f"Optimized for PubMed: '{optimized_query}'")
- logger.debug(
- f"Query optimization complete: '{query[:50]}...' -> '{optimized_query[:100]}...'"
+ logger.info(
+ f"Optimized query for PubMed "
+ f"({len(query)} -> {len(optimized_query)} chars)"
)
+ logger.debug("Query optimization complete")
return optimized_query
@@ -405,7 +403,7 @@ Return ONLY the search query without any explanations.
logger.exception(
f"Error optimizing query ({type(e).__name__}): {safe_msg}"
)
- logger.debug(f"Falling back to original query: '{query}'")
+ logger.debug("Falling back to the original query")
return query # Fall back to original query on error
def _simplify_query(self, query: str) -> str:
@@ -419,8 +417,8 @@ Return ONLY the search query without any explanations.
Returns:
Simplified query
"""
- logger.info(f"Simplifying query: {query}")
- logger.debug(f"Query simplification started for: '{query[:100]}...'")
+ logger.info(f"Simplifying PubMed query ({len(query)} chars)")
+ logger.debug("Query simplification started")
# Simple approach: remove field restrictions to broaden the search
import re
@@ -444,7 +442,7 @@ Return ONLY the search query without any explanations.
if simplified == query:
logger.debug("No simplification possible, returning original query")
- logger.info(f"Simplified query: {simplified}")
+ logger.info(f"Simplified PubMed query ({len(simplified)} chars)")
logger.debug(
f"Query simplified from {len(query)} to {len(simplified)} chars"
)
@@ -510,11 +508,17 @@ The default assumption should be that medical and scientific queries want RECENT
.lower()
)
- # Log the determination
- logger.info(f"Historical focus determination for query: '{query}'")
- logger.info(f"LLM determined historical focus: {answer}")
+ # Log the determination. Only the derived boolean goes to the
+ # log: the prompt above embeds the query, and an LLM that ignores
+ # the "answer ONLY yes/no" instruction can echo it back inside
+ # ``answer``.
+ is_historical = "yes" in answer
+ logger.info(
+ f"Historical focus determination for query ({len(query)} chars)"
+ )
+ logger.info(f"LLM determined historical focus: {is_historical}")
- return "yes" in answer
+ return is_historical
except Exception as e:
safe_msg = self._scrub_error(e)
@@ -651,7 +655,8 @@ The default assumption should be that medical and scientific queries want RECENT
logger.debug(f"Limiting results to last {self.days_limit} days")
logger.debug(
- f"PubMed search query: '{query}' with max_results={self.max_results}"
+ f"PubMed search query ({len(query)} chars) with "
+ f"max_results={self.max_results}"
)
self._last_wait_time = self.rate_tracker.apply_rate_limit(
@@ -673,7 +678,8 @@ The default assumption should be that medical and scientific queries want RECENT
total_count = data["esearchresult"].get("count", "unknown")
logger.info(
- f"PubMed search for '{query}' found {len(id_list)} results (total available: {total_count})"
+ f"PubMed search found {len(id_list)} results "
+ f"(total available: {total_count})"
)
if len(id_list) > 0:
logger.debug(f"First 5 PMIDs: {id_list[:5]}")
@@ -682,7 +688,7 @@ The default assumption should be that medical and scientific queries want RECENT
except Exception as e:
safe_msg = self._scrub_error(e)
logger.warning(
- f"Error searching PubMed for query '{query}' ({type(e).__name__}): {safe_msg}"
+ f"Error searching PubMed ({type(e).__name__}): {safe_msg}"
)
return []
@@ -1322,7 +1328,7 @@ The default assumption should be that medical and scientific queries want RECENT
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting PubMed previews for query: {query}")
+ logger.info("Getting PubMed previews")
# Optimize the query for PubMed if LLM is available
optimized_query = self._optimize_query_for_pubmed(query)
@@ -1337,7 +1343,7 @@ The default assumption should be that medical and scientific queries want RECENT
)
simplified_query = self._simplify_query(optimized_query)
if simplified_query != optimized_query:
- logger.info(f"Trying with simplified query: {simplified_query}")
+ logger.info("Trying with simplified query")
pmid_list, strategy = self._adaptive_search(simplified_query)
if pmid_list:
logger.info(
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_retriever.py b/src/local_deep_research/web_search_engines/engines/search_engine_retriever.py
index 6e56bc0773..6989dd1657 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_retriever.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_retriever.py
@@ -69,7 +69,7 @@ class RetrieverSearchEngine(BaseSearchEngine):
results.append(result)
logger.info(
- f"Retriever '{self.name}' returned {len(results)} results for query: {query}"
+ f"Retriever '{self.name}' returned {len(results)} results"
)
return results
@@ -141,7 +141,7 @@ class RetrieverSearchEngine(BaseSearchEngine):
previews.append(preview)
logger.info(
- f"Retriever '{self.name}' returned {len(previews)} previews for query: {query}"
+ f"Retriever '{self.name}' returned {len(previews)} previews"
)
return previews
@@ -199,7 +199,7 @@ class RetrieverSearchEngine(BaseSearchEngine):
results.append(result)
logger.info(
- f"Retriever '{self.name}' returned {len(results)} async results for query: {query}"
+ f"Retriever '{self.name}' returned {len(results)} async results"
)
return results
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
index 66df83ae99..72557323b0 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
@@ -353,7 +353,7 @@ class SearXNGSearchEngine(BaseSearchEngine):
)
return []
- logger.info(f"SearXNG running search for query: {query}")
+ logger.info("SearXNG running search")
try:
self._respect_rate_limit()
@@ -534,7 +534,7 @@ class SearXNGSearchEngine(BaseSearchEngine):
)
else:
logger.warning(
- f"SearXNG returned no valid results for query: {query}. "
+ "SearXNG returned no valid results. "
"This may indicate SearXNG backend engine issues or rate limiting."
)
return results
@@ -635,7 +635,7 @@ class SearXNGSearchEngine(BaseSearchEngine):
)
else:
logger.warning(
- f"SearXNG returned no valid results for query: {query}. "
+ "SearXNG returned no valid results. "
"This may indicate SearXNG backend engine issues or rate limiting."
)
return results
@@ -656,12 +656,12 @@ class SearXNGSearchEngine(BaseSearchEngine):
)
return []
- logger.info(f"Getting SearXNG previews for query: {query}")
+ logger.info("Getting SearXNG previews")
results = self._get_search_results(query)
if not results:
- logger.warning(f"No SearXNG results found for query: {query}")
+ logger.warning("No SearXNG results found")
return []
previews = []
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
index b4affdded5..28bba2fe45 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py
@@ -309,8 +309,10 @@ Return ONLY the optimized search query with no explanation.
)
return query
- logger.info(f"Original query: '{query}'")
- logger.info(f"Optimized for search: '{optimized_query}'")
+ logger.info(
+ f"Optimized query for Semantic Scholar "
+ f"({len(query)} -> {len(optimized_query)} chars)"
+ )
return optimized_query
except Exception as e:
@@ -372,13 +374,9 @@ Return ONLY the optimized search query with no explanation.
if "data" in response:
papers = response["data"]
- logger.info(
- f"Found {len(papers)} papers with direct search for query: '{query}'"
- )
+ logger.info(f"Found {len(papers)} papers with direct search")
return papers # type: ignore[no-any-return]
- logger.warning(
- f"No data in response for direct search query: '{query}'"
- )
+ logger.warning("No data in response for direct search")
return []
except Exception as e:
@@ -407,8 +405,7 @@ Return ONLY the optimized search query with no explanation.
if '"' in query:
unquoted_query = query.replace('"', "")
logger.info(
- "No results with quoted terms, trying without quotes: {}",
- unquoted_query,
+ "No results with quoted terms, retrying without quotes"
)
papers = self._direct_search(unquoted_query)
@@ -459,14 +456,13 @@ Format each query on a new line with no numbering or explanation. Keep each quer
for alt_query in alt_queries[
:3
]: # Limit to first 3 alternatives
- logger.info("Trying LLM-suggested query: {}", alt_query)
+ logger.info("Trying LLM-suggested alternative query")
alt_papers = self._direct_search(alt_query)
if alt_papers:
logger.info(
- "Found {} papers using LLM-suggested query: {}",
+ "Found {} papers using the LLM-suggested query",
len(alt_papers),
- alt_query,
)
strategy = "llm_alternative"
return alt_papers, strategy
@@ -484,7 +480,7 @@ Format each query on a new line with no numbering or explanation. Keep each quer
# Use up to 3 of the longest words
longer_words = sorted(longer_words, key=len, reverse=True)[:3]
key_terms_query = " ".join(longer_words)
- logger.info("Trying with key terms: {}", key_terms_query)
+ logger.info("Trying with key terms extracted from the query")
papers = self._direct_search(key_terms_query)
if papers:
@@ -495,7 +491,7 @@ Format each query on a new line with no numbering or explanation. Keep each quer
if words:
longest_word = max(words, key=len)
if len(longest_word) > 5: # Only use if it's reasonably long
- logger.info("Trying with single key term: {}", longest_word)
+ logger.info("Trying with the single longest key term")
papers = self._direct_search(longest_word)
if papers:
@@ -565,7 +561,7 @@ Format each query on a new line with no numbering or explanation. Keep each quer
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Semantic Scholar previews for query: {query}")
+ logger.info("Getting Semantic Scholar previews")
# Optimize the query if LLM is available
optimized_query = self._optimize_query(query)
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_stackexchange.py b/src/local_deep_research/web_search_engines/engines/search_engine_stackexchange.py
index 973eb1620b..03de21996a 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_stackexchange.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_stackexchange.py
@@ -195,9 +195,7 @@ class StackExchangeSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(
- f"Getting Stack Exchange previews for query: {query} on {self.site}"
- )
+ logger.info(f"Getting Stack Exchange previews on {self.site}")
# Apply rate limiting
self._last_wait_time = self.rate_tracker.apply_rate_limit(
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
index f22c2ec66a..f3731a9c91 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
@@ -80,7 +80,7 @@ class WaybackSearchEngine(BaseSearchEngine):
# Check if query is a domain without http prefix
domain_pattern = re.compile(r"^(?:[-\w.]|(?:%[\da-fA-F]{2}))+\.\w+$")
if domain_pattern.match(query):
- logger.info(f"Query appears to be a domain: {query}")
+ logger.info("Query appears to be a domain")
return [f"http://{query}"]
# For non-URL queries, use DuckDuckGo to find relevant URLs
@@ -116,14 +116,14 @@ class WaybackSearchEngine(BaseSearchEngine):
# Fallback: treat the query as a potential domain or path
if "/" in query and "." in query:
- logger.info(f"Treating query as a partial URL: {query}")
+ logger.info("Treating query as a partial URL")
return [f"http://{query}"]
if "." in query:
- logger.info(f"Treating query as a domain: {query}")
+ logger.info("Treating query as a domain")
return [f"http://{query}"]
# Return empty list if nothing worked
- logger.warning(f"Could not extract any URLs from query: {query}")
+ logger.warning("Could not extract any URLs from the query")
return []
def _format_timestamp(self, timestamp: str) -> str:
@@ -254,13 +254,13 @@ class WaybackSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Wayback Machine previews for query: {query}")
+ logger.info("Getting Wayback Machine previews")
# Extract URLs from query
urls = self._extract_urls_from_query(query)
if not urls:
- logger.warning(f"No URLs found in query: {query}")
+ logger.warning("No URLs found in the query")
return []
# Get snapshots for each URL
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wikinews.py b/src/local_deep_research/web_search_engines/engines/search_engine_wikinews.py
index 7a2b9a83fa..3dc645deea 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_wikinews.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wikinews.py
@@ -192,8 +192,10 @@ NOW RETURN ONLY THE JSON OBJECT.
)
return query
- logger.info(f"Original query: '{query}'")
- logger.info(f"Optimized for Wikinews: '{optimized_query}'")
+ logger.info(
+ f"Optimized query for Wikinews "
+ f"({len(query)} -> {len(optimized_query)} chars)"
+ )
return optimized_query
@@ -240,7 +242,7 @@ Respond with ONE WORD ONLY: CURRENT, HISTORICAL, or UNCLEAR"""
if "CURRENT" in answer:
# For current events, focus on recent content
logger.info(
- f"Query '{query}' classified as CURRENT - focusing on recent content"
+ "Query classified as CURRENT - focusing on recent content"
)
self.from_date = datetime.now(UTC) - timedelta(
days=DEFAULT_RECENT_BACKWARD_DAYS
@@ -248,19 +250,19 @@ Respond with ONE WORD ONLY: CURRENT, HISTORICAL, or UNCLEAR"""
elif "HISTORICAL" in answer:
# For historical queries, go back as far as possible
logger.info(
- f"Query '{query}' classified as HISTORICAL - extending search timeframe"
+ "Query classified as HISTORICAL - extending search timeframe"
)
self.from_date = datetime.min.replace(tzinfo=UTC)
else:
logger.info(
- f"Query '{query}' classified as UNCLEAR - keeping original date range"
+ "Query classified as UNCLEAR - keeping original date range"
)
except (AttributeError, TypeError, ValueError, RuntimeError) as e:
# Keep original date parameters on error
safe_msg = self._scrub_error(e)
logger.exception(
- f"Error adapting date range for query '{query}' ({type(e).__name__}): keeping original date range: {safe_msg}"
+ f"Error adapting date range ({type(e).__name__}): keeping original date range: {safe_msg}"
)
def _fetch_search_results(
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
index 652c656c46..763a8390b4 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
@@ -79,7 +79,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Wikipedia page previews for query: {query}")
+ logger.info("Getting Wikipedia page previews")
try:
# Apply rate limiting before search request
@@ -99,7 +99,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
)
if not search_results:
- logger.info(f"No Wikipedia results found for query: {query}")
+ logger.info("No Wikipedia results found")
return []
# Generate previews with summaries.
@@ -204,9 +204,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
# Same 429-style failure on the outer wikipedia.search() call —
# log once at warning level and return an empty list.
logger.warning(
- "Wikipedia rate-limited on search for query '{}'; "
- "returning no previews",
- query,
+ "Wikipedia rate-limited on search; returning no previews"
)
return []
except Exception as e:
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_zenodo.py b/src/local_deep_research/web_search_engines/engines/search_engine_zenodo.py
index 4dd948315b..a8afe5109e 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_zenodo.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_zenodo.py
@@ -121,7 +121,7 @@ class ZenodoSearchEngine(BaseSearchEngine):
Returns:
List of preview dictionaries
"""
- logger.info(f"Getting Zenodo previews for query: {query}")
+ logger.info("Getting Zenodo previews")
# Apply rate limiting
self._last_wait_time = self.rate_tracker.apply_rate_limit(
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index f1d0e9d3d2..d4ef743b73 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -739,7 +739,7 @@ class BaseSearchEngine(ABC):
previews = self._get_previews(query)
if not previews:
logger.info(
- f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
+ f"Search engine {self.__class__.__name__} returned no preview results"
)
results_count = 0
return []
diff --git a/tests/security/test_sensitive_logging_hook.py b/tests/security/test_sensitive_logging_hook.py
index e7f2cece32..d5f4d634de 100644
--- a/tests/security/test_sensitive_logging_hook.py
+++ b/tests/security/test_sensitive_logging_hook.py
@@ -10,6 +10,8 @@ import sys
from importlib import import_module
from pathlib import Path
+import pytest
+
HOOKS_DIR = Path(__file__).parent.parent.parent / ".pre-commit-hooks"
sys.path.insert(0, str(HOOKS_DIR))
@@ -1322,8 +1324,14 @@ class TestSourceTreeInvariant:
"""Run the full checker over every file in the three dirs."""
failures = []
for path in self._secure_dir_files():
- tree = ast.parse(path.read_text(encoding="utf-8"))
- checker = SensitiveLoggingChecker(str(path))
+ content = path.read_text(encoding="utf-8")
+ tree = ast.parse(content)
+ # Mirrors check_file()'s construction so this sweep honours the
+ # same allow-search-query-log suppressions pre-commit does,
+ # rather than failing on a marker pre-commit would accept.
+ checker = SensitiveLoggingChecker(
+ str(path), source_lines=content.split("\n")
+ )
checker.visit(tree)
failures.extend(checker.errors)
assert not failures, "\n".join(failures)
@@ -1619,3 +1627,381 @@ class TestSourceTreeInvariant:
f" actual: {SECURE_LOGGING_DIRS}\n"
f" expected: {expected}"
)
+
+
+# ---------------------------------------------------------------------------
+# #5646: search queries on the empty-result path under web_search_engines/.
+# ---------------------------------------------------------------------------
+
+
+def _check_query_code(code: str, filename: str = IN_DIR) -> list:
+ """Run the checker with source lines attached (suppression comments)."""
+ tree = ast.parse(code)
+ checker = SensitiveLoggingChecker(filename, source_lines=code.split("\n"))
+ checker.visit(tree)
+ return [e for e in checker.errors if "search query" in e]
+
+
+class TestDetectsSearchQueryLogging:
+ """BaseSearchEngine.run() omits the query; engines must not re-add it."""
+
+ def test_detects_query_in_get_previews(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info(f"Getting X previews for query: {query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_get_search_results(self):
+ code = (
+ "def _get_search_results(self, query):\n"
+ ' logger.info(f"X running search for query: {query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_rewritten_query_variant(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info(f"trying {simplified_query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_warning_outside_preview_path(self):
+ code = (
+ "def _direct_search(self, query):\n"
+ ' logger.warning(f"No data for query: {query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_percent_format(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info("running search for %s" % query)\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_bind_chain(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.bind(engine="x").info("previews for {}", query)\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_allows_query_length_and_derived_values(self):
+ """Documented gap: only len(query) is genuinely non-reproducing.
+
+ ``query.split()`` reproduces the query verbatim modulo whitespace, but
+ the rule cannot ban calls on the query without also banning
+ ``len(query)``. No call site in the package uses it.
+ """
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info(f"query length {len(query)}")\n'
+ ' logger.info("terms %s", query.split())\n'
+ )
+ assert _check_query_code(code) == []
+
+ def test_detects_query_in_helper_called_from_the_preview_path(self):
+ """Scope is the package, not the lexically enclosing function name.
+
+ ``_optimize_query`` is reached from ``_get_previews`` one frame down;
+ scoping the rule by enclosing function name exempted every such
+ helper by construction, which is the leak this rule exists to stop.
+ """
+ code = (
+ "def _optimize_query(self, query):\n"
+ ' logger.info(f"Original query: {query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_str_format_call(self):
+ """``.format(query)`` is a Call, not a JoinedStr or a BinOp."""
+ code = (
+ "def _optimize_query(self, query):\n"
+ ' logger.warning("no results for {}".format(query))\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_string_concatenation(self):
+ """``"..." + query`` is a BinOp whose op is Add, not Mod."""
+ code = (
+ "def _optimize_query(self, query):\n"
+ ' logger.warning("no results for " + query)\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_bound_as_a_bind_kwarg(self):
+ """loguru copies bind kwargs into record["extra"] — a live sink.
+
+ The message itself is clean here; only ``bind()`` carries the query,
+ so a check that looks at the outer ``.warning()`` alone misses it.
+ """
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.bind(query=query).warning("empty")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_truncated_query_slices(self):
+ """``query[:100]`` still reproduces the first 100 chars the user typed."""
+ code = (
+ "def _optimize_query(self, query):\n"
+ ' logger.debug(f"head {query[:100]}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_allows_query_length_and_counts_keyed_by_query(self):
+ """The shapes a Subscript rule must not catch.
+
+ ``len(query)`` is a Call, and ``query_counts[engine]`` subscripts a
+ name the exact-name matcher does not treat as a query.
+ """
+ code = (
+ "def _optimize_query(self, query):\n"
+ ' logger.debug(f"{len(query)} chars, {query_counts[engine]} hits")\n'
+ )
+ assert _check_query_code(code) == []
+
+ def test_ignores_files_outside_web_search_engines(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.warning(f"no results for {query}")\n'
+ )
+ assert _check_query_code(code, filename=OUT_DIR) == []
+
+ def test_ignores_non_query_names(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info(f"url {query_url} params {query_params}")\n'
+ )
+ assert _check_query_code(code) == []
+
+ def test_suppression_comment_with_reason_silences_the_rule(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ " logger.info( # allow-search-query-log: needed to match "
+ "the upstream rejection log\n"
+ ' f"rejected: {query}"\n'
+ " )\n"
+ )
+ assert _check_query_code(code) == []
+
+ def test_suppression_comment_without_reason_does_not_silence(self):
+ code = (
+ "def _get_previews(self, query):\n"
+ " logger.info( # allow-search-query-log:\n"
+ ' f"rejected: {query}"\n'
+ " )\n"
+ )
+ assert len(_check_query_code(code)) == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'logger.info(f"allow-search-query-log: {query}")',
+ 'logger.info("allow-search-query-log:" + query)',
+ 'logger.info(f"{query}", note="allow-search-query-log: reason")',
+ 'logger.info(f"{query} # allow-search-query-log: reason")',
+ 'logger.info(\n f"{query}",\n note="allow-search-query-log: reason"\n)',
+ 'logger.info(f"""{query}\nallow-search-query-log: reason""")',
+ # PEP 701 (3.12+): a comment may sit inside a replacement field of
+ # a multi-line f-string. tokenize reports it as a COMMENT token
+ # even though it is part of the string literal.
+ 'logger.info(\n f"""no results for {\n'
+ " query # allow-search-query-log: this text lives inside "
+ 'a string literal\n }"""\n)',
+ # Same shape, but with the in-string comment on the method-name
+ # line, so the f-string tracking is the only thing rejecting it.
+ 'logger.info(f"""{query # allow-search-query-log: this text '
+ 'lives inside a string literal\n}""")',
+ ],
+ )
+ def test_marker_inside_string_does_not_suppress_query(self, code):
+ assert len(_check_query_code(code)) == 1
+
+ def test_suppression_comment_on_previous_line_does_not_silence_call(self):
+ code = (
+ "# allow-search-query-log: unrelated previous line\n"
+ 'logger.info(f"{query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_suppression_comment_on_closing_line_does_not_silence_call(self):
+ """The marker counts only on the call's method-name line.
+
+ A marker anywhere in the call's span let a reason written about one
+ part of a multi-line call exempt the whole thing.
+ """
+ code = (
+ 'logger.info(\n f"{query}"\n'
+ ") # allow-search-query-log: required upstream correlation\n"
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_suppression_comment_on_bind_kwarg_does_not_silence_the_message(
+ self,
+ ):
+ """A reason given for a bind() kwarg is not a reason for the message.
+
+ The package has eleven ``logger.bind(policy_audit=True).warning(``
+ sites of exactly this shape, where the comment justifying the bound
+ field sits lines above the method name.
+ """
+ code = (
+ "def _get_previews(self, query, engine):\n"
+ " logger.bind(\n"
+ " engine=engine, # allow-search-query-log: policy audit "
+ "needs the engine\n"
+ ' ).warning(f"No results for {query}")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_suppression_comment_on_the_method_name_line_silences_call(self):
+ """A bind chain broken across lines is exempt from its own last line."""
+ code = (
+ "def _get_previews(self, query, engine):\n"
+ " logger.bind(\n"
+ " engine=engine,\n"
+ " ).warning( # allow-search-query-log: required upstream "
+ "correlation\n"
+ ' f"No results for {query}"\n'
+ " )\n"
+ )
+ assert _check_query_code(code) == []
+
+ @pytest.mark.parametrize(
+ "comment",
+ [
+ # The accidental one: the comment written while cleaning these up.
+ "# TODO: drop the allow-search-query-log: marker below",
+ # The marker must open the comment, not merely occur in it.
+ "# xxallow-search-query-log: y",
+ # A written reason, not a single punctuation character.
+ "# allow-search-query-log: .",
+ "# allow-search-query-log: short",
+ ],
+ )
+ def test_unqualified_marker_does_not_suppress_query(self, comment):
+ code = (
+ "def _get_previews(self, query):\n"
+ f' logger.info(f"empty {{query}}") {comment}\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_in_bind_chain_at_debug_level(self):
+ """``debug`` is in the shared level set; the bind branch omitted it.
+
+ ``logger.bind(query=query).debug(...)`` was the one shape neither the
+ direct-call rule (its receiver is a Call, not the ``logger`` Name) nor
+ the bind-chain rule (its level set had no ``debug``) inspected.
+ """
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.bind(query=query).debug("No preview results")\n'
+ )
+ assert len(_check_query_code(code)) == 1
+
+ def test_detects_query_passed_as_a_brace_style_keyword_argument(self):
+ """``logger.info("...{q}", q=query)`` is the style loguru wants.
+
+ ``check-loguru-formatting.py`` pushes writers toward brace
+ placeholders, which put the query in a keyword argument rather than
+ in the message expression, so the keyword scan is the only branch
+ that sees it.
+ """
+ code = (
+ "def _get_previews(self, query):\n"
+ ' logger.info("No results for {q}", q=query)\n'
+ ' logger.warning("Empty: {q}", q=query[:100])\n'
+ )
+ assert len(_check_query_code(code)) == 2
+
+ def test_reason_length_counts_non_space_characters_not_stripped_length(
+ self,
+ ):
+ """``QUERY_LOG_REASON_MIN_CHARS`` counts non-space characters.
+
+ The check is ``len("".join(reason.split()))``, not
+ ``len(reason.strip())`` — a reason padded with internal whitespace
+ can be long after ``.strip()`` while carrying few actual words.
+ """
+ reason_ok = "a b c d e f g h" # 8 non-space chars: accepted
+ assert len("".join(reason_ok.split())) == 8
+ code_ok = (
+ "def _get_previews(self, query):\n"
+ " logger.info( # allow-search-query-log: " + reason_ok + "\n"
+ ' f"rejected: {query}"\n'
+ " )\n"
+ )
+ assert _check_query_code(code_ok) == []
+
+ # Same reason, padded with extra internal spaces: .strip() alone
+ # leaves 21 characters (well over the 8-char minimum), but only 3 are
+ # non-space, so this must still be rejected.
+ reason_padded = " a b c "
+ assert len(reason_padded.strip()) == 21
+ assert len("".join(reason_padded.split())) == 3
+ code_padded = (
+ "def _get_previews(self, query):\n"
+ " logger.info( # allow-search-query-log:" + reason_padded + "\n"
+ ' f"rejected: {query}"\n'
+ " )\n"
+ )
+ assert len(_check_query_code(code_padded)) == 1
+
+ def test_tokenize_error_fails_closed_and_does_not_suppress(self):
+ """A tokenize failure must never leave a stale suppression standing.
+
+ The constructor tokenizes ``source_lines`` up front and, on
+ ``TokenError``/``IndentationError``, clears
+ ``query_log_suppression_lines`` (fail-closed) rather than keeping
+ whatever markers it had already collected. Feed the checker a
+ well-formed AST (so ``visit()`` can run) alongside source lines that
+ break ``tokenize`` (an unterminated string) to exercise that branch
+ directly — a marker that would otherwise clearly suppress the call
+ does not.
+ """
+ good_source = (
+ "def _get_previews(self, query):\n"
+ " logger.info( # allow-search-query-log: enough reason given here\n"
+ ' f"empty {query}"\n'
+ " )\n"
+ )
+ # Sanity: this exact source, tokenized cleanly, does suppress.
+ assert _check_query_code(good_source) == []
+
+ broken_source_lines = (good_source + ' x = "unterminated\n').split(
+ "\n"
+ )
+ tree = ast.parse(good_source)
+ checker = SensitiveLoggingChecker(
+ IN_DIR, source_lines=broken_source_lines
+ )
+ assert checker.query_log_suppression_lines == set()
+ checker.visit(tree)
+ assert any("search query" in e for e in checker.errors)
+
+
+ "separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c"]
+)
[email protected]("comment_on_query", [False, True])
+def test_query_exemption_uses_physical_source_lines(
+ tmp_path, separator, comment_on_query
+):
+ source = 'note = """' + separator + '"""\n'
+ comment = " # allow-search-query-log: required upstream correlation"
+ source += (
+ 'logger.info("safe")' + ("" if comment_on_query else comment) + "\n"
+ )
+ source += 'logger.info(f"{query}")' + (comment if comment_on_query else "")
+ path = tmp_path / "src/local_deep_research/web_search_engines/example.py"
+ path.parent.mkdir(parents=True)
+ path.write_text(source, encoding="utf-8")
+
+ errors = [
+ error
+ for error in hook_module.check_file(path)
+ if "search query" in error
+ ]
+ assert bool(errors) is not comment_on_query
diff --git a/tests/web_search_engines/engines/test_guardian_coverage.py b/tests/web_search_engines/engines/test_guardian_coverage.py
index 3a239b5a09..af02c908c8 100644
--- a/tests/web_search_engines/engines/test_guardian_coverage.py
+++ b/tests/web_search_engines/engines/test_guardian_coverage.py
@@ -577,8 +577,10 @@ class TestGetPreviewsLogic:
]
self._run_previews(engine, articles, query="my original")
assert hasattr(engine, "_search_metadata")
- assert engine._search_metadata["original_query"] == "my original"
- assert engine._search_metadata["optimized_query"] == "opt"
+ assert engine._search_metadata["original_query_length"] == len(
+ "my original"
+ )
+ assert engine._search_metadata["optimized_query_length"] == len("opt")
assert engine._search_metadata["strategy"] == "initial"
def test_full_articles_cache_populated(self):
diff --git a/tests/web_search_engines/test_adapter_failure_contracts.py b/tests/web_search_engines/test_adapter_failure_contracts.py
index 8692336d46..e6873637c8 100644
--- a/tests/web_search_engines/test_adapter_failure_contracts.py
+++ b/tests/web_search_engines/test_adapter_failure_contracts.py
@@ -1277,7 +1277,7 @@ class TestResearchQueryIsNotPersistedToLogs:
):
"""Negative control for the whole section: arXiv logs a constant
string instead of the query, and the detector says so. Without
- this the xfails below could be passing on a broken detector."""
+ this the assertions below could be passing on a broken detector."""
_run_arxiv_for_logs(inert_tracker)
assert captured_logs.records, "no records captured -- sink is dead"
assert captured_logs.messages_containing(PROBE_QUERY) == []
@@ -1290,17 +1290,10 @@ class TestResearchQueryIsNotPersistedToLogs:
pytest.param(_run_pubmed_for_logs, id="pubmed"),
],
)
- @pytest.mark.xfail(
- strict=True,
- reason=(
- "#5734: these adapters interpolate the raw query into INFO / "
- "WARNING log records, which database_sink persists. Remove "
- "this marker once the query is dropped or hashed."
- ),
- )
def test_adapters_must_not_log_the_query_at_database_sink_level(
self, inert_tracker, captured_logs, runner
):
+ """#5646 removes raw-query logging; these regressions now pass normally."""
runner(inert_tracker)
assert captured_logs.records, "no records captured -- sink is dead"
leaks = captured_logs.messages_containing(PROBE_QUERY)
diff --git a/tests/web_search_engines/test_search_engine_base_query_logging.py b/tests/web_search_engines/test_search_engine_base_query_logging.py
new file mode 100644
index 0000000000..be3135f438
--- /dev/null
+++ b/tests/web_search_engines/test_search_engine_base_query_logging.py
@@ -0,0 +1,296 @@
+"""The search query must not reach the log on the empty-result path.
+
+``BaseSearchEngine.run()`` logs "returned no preview results" without the
+query, but it only reaches that line *after* calling ``_get_previews()`` and
+it records metrics in its ``finally`` block — so the engine implementations
+and ``SearchTracker`` are part of the same leak surface and are covered here
+too. ``.pre-commit-hooks/check-sensitive-logging.py`` enforces the rule
+statically; these tests pin the runtime behaviour.
+"""
+
+from unittest.mock import Mock, call, patch
+
+import pytest
+
+from local_deep_research.web_search_engines.engines.search_engine_guardian import (
+ GuardianSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_mojeek import (
+ MojeekSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_pubchem import (
+ PubChemSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (
+ SemanticScholarSearchEngine,
+)
+from local_deep_research.web_search_engines.engines.search_engine_searxng import (
+ SearXNGSearchEngine,
+)
+from local_deep_research.web_search_engines.search_engine_base import (
+ BaseSearchEngine,
+)
+
+QUERY = "unique-query-must-not-be-logged"
+
+# The fallback ladders below log *parts* of the query rather than the whole
+# string, so ``QUERY not in ...`` would pass while a term the user typed was
+# written to the sink. Every word here is long enough to survive both of
+# Semantic Scholar's length filters (>6 chars for the key-terms rung, >5 for
+# the single-longest-term rung) and none of them occurs in any fixed message.
+PROBE_QUERY = "paleobotany triskelion zymurgy"
+PROBE_WORDS = PROBE_QUERY.split()
+
+
+def _assert_no_probe_word(mock_logger):
+ """No word of PROBE_QUERY may appear in any captured logger call."""
+ logged = repr(mock_logger.mock_calls)
+ leaked = [word for word in PROBE_WORDS if word in logged]
+ assert not leaked, f"query terms reached the log: {leaked}\n{logged}"
+
+
+class _NoPreviewSearchEngine(BaseSearchEngine):
+ def _get_previews(self, query):
+ return []
+
+ def _get_full_content(self, relevant_items):
+ return relevant_items
+
+
+def test_run_does_not_log_query_when_no_previews_are_returned():
+ engine = _NoPreviewSearchEngine(programmatic_mode=True)
+ engine.rate_tracker = Mock(enabled=False)
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_base.logger"
+ ) as mock_logger:
+ results = engine.run(QUERY)
+
+ assert results == []
+ # Positive assertion first: without it every remaining assertion would
+ # also pass if run() raised early, if the empty branch became
+ # unreachable, or if the logger.info call were deleted outright.
+ assert (
+ call.info(
+ "Search engine _NoPreviewSearchEngine returned no preview results"
+ )
+ in mock_logger.mock_calls
+ )
+ # repr(mock_calls) covers positional args, keyword args, and chained
+ # calls (e.g. logger.bind(...).info(...)) in one flat string.
+ assert QUERY not in repr(mock_logger.mock_calls)
+
+
+ "engine_cls, module, entry_message, empty_message",
+ [
+ (
+ SearXNGSearchEngine,
+ "search_engine_searxng",
+ "Getting SearXNG previews",
+ "No SearXNG results found",
+ ),
+ (
+ MojeekSearchEngine,
+ "search_engine_mojeek",
+ "Getting Mojeek previews",
+ "No Mojeek results found",
+ ),
+ ],
+)
+def test_engine_get_previews_omits_query_on_empty_results(
+ engine_cls, module, entry_message, empty_message
+):
+ """The engine's own entry log and empty-result warning must stay quiet.
+
+ Both fire before ``run()`` reaches its sanitised message, so a query left
+ in either of them makes the base-class fix worthless for that engine.
+ """
+ # Bypass __init__: it wants settings/DB access, and _get_previews only
+ # needs the availability flag and the results helper.
+ engine = engine_cls.__new__(engine_cls)
+ engine._is_available = True
+ engine._get_search_results = Mock(return_value=[])
+
+ with patch(
+ f"local_deep_research.web_search_engines.engines.{module}.logger"
+ ) as mock_logger:
+ previews = engine._get_previews(QUERY)
+
+ assert previews == []
+ engine._get_search_results.assert_called_once_with(QUERY)
+ assert call.info(entry_message) in mock_logger.mock_calls
+ assert call.warning(empty_message) in mock_logger.mock_calls
+ assert QUERY not in repr(mock_logger.mock_calls)
+
+
[email protected]("missing", ["username", "user_password"])
+def test_search_tracker_omits_query_when_context_is_incomplete(missing):
+ """run()'s finally block records metrics; its bail-outs logged the query.
+
+ The no-password branch additionally paired the query with the username.
+ """
+ from local_deep_research.metrics import search_tracker
+
+ context = {
+ "research_id": "1",
+ "username": "alice",
+ "user_password": "s3cret",
+ }
+ context.pop(missing)
+
+ with (
+ patch.object(
+ search_tracker, "get_search_context", return_value=context
+ ),
+ patch.object(search_tracker, "logger") as mock_logger,
+ ):
+ search_tracker.SearchTracker.record_search(
+ engine_name="searxng", query=QUERY, results_count=0
+ )
+
+ assert mock_logger.warning.call_count == 1
+ logged = repr(mock_logger.mock_calls)
+ assert "Cannot save search metrics" in logged
+ assert "searxng" in logged
+ assert QUERY not in logged
+ if missing == "user_password":
+ # The username is kept deliberately - it is what an operator needs to
+ # find the broken research context. What is removed is the pairing of
+ # that username with the text the user searched for.
+ assert "alice" in logged
+
+
+def test_semantic_scholar_fallback_ladder_omits_every_query_term():
+ """The empty-result ladder logged a word taken straight from the query.
+
+ ``_adaptive_search`` walks four rungs when the direct search comes back
+ empty; the last two build their retry string out of the user's own words
+ (``key_terms_query``, then ``longest_word``). Those are substrings of the
+ query, so an assertion on the whole query string cannot see them.
+
+ Revert that fails this test: restore
+ ``logger.info("Trying with single key term: {}", longest_word)`` at
+ ``search_engine_semantic_scholar.py:494``.
+ """
+ engine = SemanticScholarSearchEngine.__new__(SemanticScholarSearchEngine)
+ # No LLM: _optimize_query returns the raw user query unchanged, which is
+ # the configuration in which this path carries verbatim user text.
+ engine.llm = None
+ engine._direct_search = Mock(return_value=[])
+
+ with patch(
+ "local_deep_research.web_search_engines.engines."
+ "search_engine_semantic_scholar.logger"
+ ) as mock_logger:
+ papers, strategy = engine._adaptive_search(PROBE_QUERY)
+
+ assert papers == []
+ assert strategy == "standard"
+ # Positive assertions: without them this test would also pass if the
+ # ladder stopped running, or if both fallback log lines were deleted.
+ assert engine._direct_search.call_count == 3
+ assert (
+ call.info("Trying with key terms extracted from the query")
+ in mock_logger.mock_calls
+ )
+ assert (
+ call.info("Trying with the single longest key term")
+ in mock_logger.mock_calls
+ )
+ _assert_no_probe_word(mock_logger)
+
+
+def test_guardian_success_path_omits_the_query_from_search_metadata():
+ """Guardian overrides run(), so the base-class fix never applies to it.
+
+ ``_search_metadata`` carried ``original_query`` and ``optimized_query``,
+ and ``run()`` logs the whole dict at INFO whenever previews are non-empty
+ — i.e. on every Guardian search that returns something.
+
+ Revert that fails this test: restore ``"original_query": query`` and
+ ``"optimized_query": optimized_query`` in the ``_search_metadata`` dict
+ built by ``search_engine_guardian.py::_get_previews``.
+ """
+ optimized = "triskelion zymurgy"
+ article = {
+ "id": "a1",
+ "title": "t",
+ "link": "https://example.invalid/a1",
+ "snippet": "s",
+ "publication_date": "2026-01-01",
+ "section": "World",
+ "author": "a",
+ "full_content": "c",
+ }
+
+ engine = GuardianSearchEngine.__new__(GuardianSearchEngine)
+ engine.llm = None
+ engine.max_filtered_results = None
+ engine.from_date = "2026-01-01"
+ engine.to_date = "2026-01-02"
+ engine.section = None
+ engine.order_by = "relevance"
+ engine._original_date_params = {
+ "from_date": engine.from_date,
+ "to_date": engine.to_date,
+ }
+ engine._optimize_query_for_guardian = Mock(return_value=optimized)
+ engine._adapt_dates_for_query_type = Mock()
+ engine._adaptive_search = Mock(return_value=([article], "standard"))
+
+ with patch(
+ "local_deep_research.web_search_engines.engines."
+ "search_engine_guardian.logger"
+ ) as mock_logger:
+ results = engine.run(PROBE_QUERY)
+
+ assert results == [article]
+ # Positive assertion: the metadata line must still be reached, otherwise
+ # the absence assertions below are vacuous.
+ logged = repr(mock_logger.mock_calls)
+ assert "Search metadata:" in logged
+ assert "'strategy': 'standard'" in logged
+ _assert_no_probe_word(mock_logger)
+ # The optimized query is derived from the user's text and was logged
+ # under its own key; its terms must be gone too.
+ for word in optimized.split():
+ assert word not in logged
+
+
+def test_pubchem_fallback_processing_error_omits_the_query():
+ """The autocomplete-empty fallback re-uses the raw query as a "name".
+
+ When ``_search_compounds`` finds nothing, ``_get_previews`` tries a
+ direct lookup and, if that succeeds, sets ``compound_names = [query]``
+ (``search_engine_pubchem.py:265``) — the query itself becomes the only
+ "compound name" the per-compound loop processes. If that processing then
+ raises, the exception-logging call site logged that name verbatim.
+
+ Revert that fails this test: restore
+ ``f"Error processing PubChem compound: {name} ..."`` at
+ ``search_engine_pubchem.py:361``.
+ """
+ engine = PubChemSearchEngine.__new__(PubChemSearchEngine)
+ engine.engine_type = "PubChemSearchEngine"
+ engine.rate_tracker = Mock()
+ engine.max_results = 10
+ engine._search_compounds = Mock(return_value=[])
+ # First call: the direct-lookup probe that seeds compound_names=[query].
+ # Second call: the per-compound loop re-fetching that same "name", which
+ # fails and reaches the exception-logging call site under test.
+ engine._get_compound_by_name = Mock(
+ side_effect=[{"cid": 1}, ValueError("boom")]
+ )
+
+ with patch(
+ "local_deep_research.web_search_engines.engines."
+ "search_engine_pubchem.logger"
+ ) as mock_logger:
+ previews = engine._get_previews(PROBE_QUERY)
+
+ assert previews == []
+ # Positive assertion: the exception path must actually be reached,
+ # otherwise the absence check below is vacuous.
+ assert mock_logger.exception.call_count == 1
+ _assert_no_probe_word(mock_logger)