~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python3
"""
Pre-commit hook to detect potential logging of sensitive data.
Prevents passwords, tokens, and other sensitive information from being logged.
Note: this hook is part of the reason we do NOT enforce ``raise X from e``
universally (see ADR-0003). Exception chains preserved via ``from e`` can
leak PII through error-tracking services and downstream handlers. Code that
wraps user-facing exceptions should omit ``from e`` to break the chain.
"""
import ast
import bisect
import io
import re
import sys
import tokenize
from pathlib import Path
from typing import List, Optional
# Hooks in this directory are standalone scripts, not a package, so we
# add the directory to sys.path and import shared constants. Same idiom
# the _commit_analysis consumers use; see _hook_common.py for the rationale.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _hook_common import SECURE_LOGGING_DIRS # noqa: E402
SENSITIVE_VARS = {
"password",
"user_password",
"pwd",
"secret",
"api_key",
"private_key",
"auth",
"credential",
"csrf_token", # CSRF tokens should not be logged
"bearer_token",
"access_token",
"refresh_token",
"auth_token",
"session_id", # Can be used to retrieve passwords
"encryption_key",
"secret_key",
"auth_header",
}
SENSITIVE_DICT_KEYS = {
"shared_research_context", # Contains user_password for SQLCipher
"settings_snapshot", # Contains API keys and other sensitive data
"kwargs", # Often contains user_password and gets passed through many functions
"credentials", # Obviously contains authentication data
"auth_data", # Authentication related data
"auth_context", # Authentication context
"user_data", # May contain user passwords
"login_data", # Login form data including passwords
"form_data", # Form submissions may contain passwords
"post_data", # POST request data
"payload", # API request payloads
"body", # Request body data
"session", # Flask session data (when used as dict)
"flask_session", # Explicit Flask session
"g.user_password", # Flask g object with password
"connection_string", # Database connection strings with passwords
"conn_str",
"db_url",
"database_url",
"conn_params",
"db_config",
"environ", # os.environ contains sensitive vars
}
# These are false positives - specific variable names that are safe to log
FALSE_POSITIVE_VARS = {
# Token counting related - these are LLM tokens, not auth tokens
"prompt_tokens",
"completion_tokens",
"token_research_count",
"tokens_truncated",
"estimated_prompt_tokens",
"estimated_completion_tokens",
"max_tokens",
"new_max_tokens",
# Configuration paths and names (not the actual config contents)
"config_path",
"config_name",
"config_hash",
"context_window_size",
# Database paths (not credentials)
"auth_db_path",
# Safe/redacted versions
"safe_settings", # This is already redacted
}
# Specific file + variable combinations that are allowed
# Each entry must have an explanation of why it's safe
ALLOWED_LOGGING = [
{
"file_pattern": "settings/logger.py",
"variable": "safe_settings",
"reason": "This variable contains already redacted settings with sensitive values replaced by ***REDACTED***",
},
{
"file_pattern": "settings/logger.py",
"variable": "settings",
"reason": "This module is responsible for safe logging and applies redaction before logging when log_level is 'debug_unsafe'",
},
{
"file_pattern": "tests/",
"variable": "csrf_token",
"reason": "Test files need to log CSRF tokens for debugging authentication issues during test development",
},
{
"file_pattern": "web/services/socket_service.py",
"variable": "kwargs",
"reason": "These are logger wrapper methods where kwargs are logger configuration options (like exc_info, extra), not actual data being logged",
},
{
"file_pattern": "search_engine_openlibrary.py",
"variable": "author_key",
"reason": "OpenLibrary author identifier (e.g. /authors/OL123A), not a cryptographic key",
},
{
"file_pattern": "web/queue/processor_v2.py",
"variable": "user_session",
"reason": "Queue routing key (username:session_id format), not session contents",
},
]
# ---------------------------------------------------------------------------
# Search-query logging (#5646)
# ---------------------------------------------------------------------------
# The user's search query is free text they typed: it is the most identifying
# thing that flows through the search stack, and every log sink (stderr, the
# log database, the frontend log view) is shared across research sessions.
# ``BaseSearchEngine.run()`` no longer interpolates it into its empty-result
# message, but ``run()`` only reaches that line *after* calling
# ``_get_previews()`` — so an engine that logs the query on entry, or in its
# own empty-result warning, undoes the fix for that engine.
#
# Scope is the whole ``web_search_engines/`` package, at shared log levels.
# An earlier version of this rule keyed off the *lexically enclosing function
# name* (``_get_previews`` / ``_get_search_results``) plus the
# production-visible levels. That is defeated by adding a frame: the engines'
# preview implementations call ``_optimize_query_for_*``, ``_simplify_query``,
# ``_adaptive_search`` and friends, and every one of those helpers was exempt
# by construction while sitting squarely on the empty-search path. Scoping by
# call graph is not something an AST-per-file hook can do, so the scope is the
# package instead.
#
# Query text is never needed here: the engine name and a result count say what
# happened, and the query itself is still persisted per search by
# ``SearchTracker.record_search`` → ``SearchCall`` in the metrics database.
#
# What this rule does *not* catch, deliberately or otherwise:
#
# * derived values reached through a *call* — ``len(query)``,
# ``query.split()``. ``len(query)`` is genuinely non-reproducing;
# ``query.split()`` reproduces the query modulo whitespace. This matcher
# deliberately leaves generic calls unclassified; it does not attempt
# to determine which calls preserve query text. Truncated slices
# (``query[:100]``) *are* flagged: a Subscript over a query name is only
# ever a piece of the user's text, and the shapes that would be false
# positives — ``len(query)`` is a Call, ``query_counts[engine]``
# subscripts a non-query name — are unaffected.
# * the query reached through anything other than a bare ``query`` /
# ``*_query`` name — ``params["q"]``, ``self.query``, ``str(query)``, or a
# local called ``expanded`` / ``simplified`` / ``query_terms``. Exact-name
# matching is what keeps ``query_url`` / ``query_params`` /
# ``query_count`` out of the results.
# * ``logger.trace()`` and ``logger.success()``: the shared
# ``_is_logger_call`` level set this rule reuses does not list the two
# loguru-only levels, so neither is inspected by any rule in this file.
# * query logging outside ``web_search_engines/`` (~35 sites in
# ``advanced_search_system/``, ``api.py`` and the news pipeline).
SEARCH_ENGINE_DIRS = ("src/local_deep_research/web_search_engines/",)
# Per-line escape hatch. The marker must open a real Python comment on the
# physical line carrying the logging call's method name, and be followed by a
# written justification of at least QUERY_LOG_REASON_MIN_CHARS non-space
# characters, e.g.::
#
# logger.warning( # allow-search-query-log: operators match this against
# # the upstream engine's own rejection log
# f"Engine rejected query: {query}"
# )
QUERY_LOG_SUPPRESSION = "allow-search-query-log:"
# The marker must open the comment (``# allow-search-query-log: ...``) and be
# followed by a real sentence. A substring match anywhere in the comment let
# ``# TODO: drop the allow-search-query-log: marker below`` — the comment a
# developer writes while cleaning these up — silence the rule, and a
# one-character reason such as ``.`` satisfied a bare "non-empty" test.
QUERY_LOG_REASON_MIN_CHARS = 8
# PEP 701 (Python 3.12+) lets a comment live inside the replacement field of a
# multi-line f-string; tokenize reports it as a COMMENT even though it is part
# of a string literal, so the f-string nesting level has to be tracked to keep
# "marker text inside a string is not an exemption" true. Absent on older
# interpreters, where no such token is ever emitted.
FSTRING_START = getattr(tokenize, "FSTRING_START", -1)
FSTRING_END = getattr(tokenize, "FSTRING_END", -2)
# Levels every logger rule inspects. ``_is_logger_call`` and the
# ``logger.bind(...)`` chain check must use the same set: when they drifted,
# ``logger.bind(query=query).debug(...)`` was the one shape no rule saw.
LOGGER_LEVEL_METHODS = frozenset(
{"info", "debug", "warning", "error", "critical", "exception", "log"}
)
# The only sanctioned logger import inside SECURE_LOGGING_DIRS. Matched
# exactly (module + level), never by suffix, so lookalike modules such as
# ``notsecurity.secure_logging`` cannot spoof it.
WRAPPER_MODULE_RELATIVE = "security.secure_logging"
WRAPPER_MODULE_ABSOLUTE = "local_deep_research.security.secure_logging"
# Default hint; per-file depth is computed at __init__ time so the message
# reflects the actual file under src/local_deep_research/.
WRAPPER_IMPORT_HINT = (
"use 'from ...security.secure_logging import logger' "
"(relative depth per file)"
)
# Attribute names reserved inside SECURE_LOGGING_DIRS: routes from the
# SecureLogger wrapper (or any object) back to raw loguru handles.
# .opt / .catch — wrapper delegates these to raw loguru ungated
# .patch — patchers can mutate record["exception"] and attach
# an ungated traceback
# .logger — raw singleton re-exports (log_utils.logger,
# local_deep_research.logger) and future self.logger
# ._logger — SecureLogger's wrapped handle (__slots__)
# ._loguru_logger — raw singleton inside secure_logging
# .__getattr__ — logger.__getattr__("opt") delegates to raw loguru
BANNED_LOGGER_ATTRS = {
"opt",
"catch",
"patch",
"logger",
"_logger",
"_loguru_logger",
"__getattr__",
}
# Traceback-formatting attribute names: even when the ``traceback`` module
# import ban is dodged (dynamic import, re-export), these calls produce raw
# traceback text that must never reach production logs.
BANNED_TRACEBACK_ATTRS = {
"format_exc",
"format_exception",
"format_exception_only",
"format_tb",
"print_exc",
"print_exception",
"print_tb",
}
# Module names whose dynamic import (import_module / __import__) is banned in
# SECURE_LOGGING_DIRS — same modules the static import bans cover.
BANNED_DYNAMIC_IMPORTS = {"loguru", "traceback"}
# Wrapper-preserving chain methods safe for secure dirs: SecureLogger.bind()
# re-wraps without letting callers mutate the raw loguru record. Messages are
# still production-visible and must pass the exception-variable check.
WRAPPER_CHAIN_METHODS = {"bind"}
# ``.patch`` on these bare module names is a genuine HTTP-client PATCH call
# (``requests.patch(url, ...)``, ``httpx.patch(url, ...)``), not a logger —
# the concrete false positive #4976 named. This is deliberately the ONLY
# escape hatch from the fail-closed ``opt``/``catch``/``patch`` ban. It is
# a whole-file DENYLIST of in-file spellings plus a few structural
# conditions — not a model of execution order and not a proof that the
# name is the real module. A receiver is exempted only when:
#
# 1. its bare name is one of these, bound by a genuine top-level,
# unaliased ``import requests``/``import httpx``;
# 2. that import appears textually before every reference to the name
# and before every ``def``/``lambda``/``class`` whose body references
# it (a function defined earlier could run before the import);
# 3. nothing else in the file binds, deletes or shadows that name (any
# assignment/augmented/annotated target, ``del``, walrus, loop/
# comprehension/``with``/``except`` target, parameter, ``def``/
# ``class``, type parameter, ``match`` capture, other or star
# import);
# 4. the reference is not evaluated in any ``class`` scope (a class
# body, outside the bodies of the functions defined in it), and no
# ``class`` in the file passes keywords (``metaclass=``, or any
# keyword a metaclass ``__prepare__`` could turn into a namespace);
# 5. the file contains none of the spellings in
# ``_NAMESPACE_VOCABULARY`` (as a name, attribute, imported name or
# identifier token inside a string/bytes literal), no import of a
# ``_NAMESPACE_MODULES`` module, no ``getattr`` other than a direct
# call with a literal attribute name, no direct top-level ``def
# __getattr__`` (one nested in ``if``/``try``, or an assignment to
# the name, is not matched), no write/delete of a dunder attribute
# (``x.__class__ = ...``), and no attribute write to a
# ``.requests``/``.httpx``/``.patch`` attribute or rooted at an
# exempted name (``requests.x = ...``);
# 6. the bare ``requests``/``httpx`` name is never used as a value —
# every load of it is the receiver of an attribute access. Passing
# the module object anywhere (``f(requests)``, ``x = requests``,
# ``[requests]``, ``return requests``) lets code this rule cannot
# follow mutate it, e.g. ``functools.update_wrapper(requests,
# logger, assigned=("patch",))`` copies the logger's ``patch`` onto
# the module.
#
# Any one failure turns the exemption off and the ``.patch`` is banned as
# on every other receiver, so a mistake here can only drop a finding for
# a spelling these lists do not name — never add one main would not emit.
# The rule sees only this file: it cannot see code in other modules
# (another module writing this one's globals or ``sys.modules``), a
# shadowing ``requests.py``/``httpx.py`` on ``sys.path``, or a
# namespace-reaching API these lists do not name (for example an
# unlisted library that resolves a computed dotted name or deserialises
# an external payload). ``.opt``/``.catch`` have no such concrete,
# nameable false positive and stay fully receiver-blind — see
# ``visit_Attribute``.
SAFE_PATCH_RECEIVER_MODULES = {"requests", "httpx"}
# Spellings that can reach a module namespace or module object (or run
# code that can) without a binding statement. One occurrence anywhere
# in a file — as a name, an attribute, an imported name or an
# identifier token inside a string/bytes literal (a dotted-name
# resolver, format field or code string spells it there) — revokes
# the requests/httpx ``.patch`` exemption for the whole file. This can
# never add a finding main would not emit: main bans every ``.patch``.
_NAMESPACE_VOCABULARY = frozenset(
{
# builtins that return, write or execute into a namespace
"globals",
"locals",
"vars",
"setattr",
"delattr",
"exec",
"eval",
"compile",
"__import__",
"__builtins__",
"builtins",
"operator",
"inspect",
"importlib",
"ChainMap",
# a module table, import hooks, namespace-holding attributes and
# the raw writers behind setattr/item assignment
"modules",
"meta_path",
"path_hooks",
"path_importer_cache",
"__dict__",
"__globals__",
"__code__",
"f_globals",
"f_locals",
# frame and code-object routes to a namespace (a frame's
# builtins/globals, the caller chain, a traceback's or
# generator's/coroutine's frame)
"f_builtins",
"_getframe",
"f_back",
"f_code",
"tb_frame",
"gi_frame",
"cr_frame",
"ag_frame",
"gi_code",
"cr_code",
"ag_code",
# module re-execution and class/type machinery (a module's spec
# or loader, an object's class swap, a metaclass namespace)
"__spec__",
"__loader__",
"__class__",
"__bases__",
"__mro__",
"__subclasses__",
"__prepare__",
"__missing__",
"__setattr__",
"__setitem__",
"__delattr__",
"__getattribute__",
}
)
# Modules whose import alone revokes the exemption: each can reach a
# live namespace, frame or object graph (gc, ctypes), resolve a dotted
# name or format field to a live object (logging.config, pkgutil,
# pydoc, string.Formatter), run a code string (runpy ... doctest),
# patch a module attribute (unittest.mock, mock), or resolve globals by
# name while deserialising. Matched against import statements only
# (the dotted path or its first component) — without an import in this
# file (or ``__import__``/``importlib``, which are vocabulary above)
# the name is just a local.
_NAMESPACE_MODULES = frozenset(
{
"gc",
"logging.config",
"pkgutil",
"pydoc",
"string",
"mock",
"ctypes",
"runpy",
"code",
"codeop",
"pdb",
"bdb",
"cProfile",
"profile",
"timeit",
"trace",
"doctest",
"unittest",
"pickle",
"_pickle",
"copyreg",
"marshal",
"shelve",
"dill",
"cloudpickle",
"jsonpickle",
"yaml",
}
)
# Identifier tokens inside string/bytes literals (see
# _NAMESPACE_VOCABULARY).
_IDENTIFIER_TOKEN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_NAME_DOMAIN_BASELINES = {
"getattr": frozenset({"getattr"}),
"builtins": frozenset(),
}
# Attribute spelling of each builtin-function domain on the builtins
# module itself: ``builtins.getattr``.
_BUILTIN_FN_LEAF = {
"getattr": "getattr",
}
# Statement fields whose bindings are not guaranteed to execute — or
# to execute in textual order — relative to code that follows the
# construct. A binding under one of these fields can only ADD an alias,
# never clear one. This only governs ALIAS names (``ga = getattr``): a
# tracked builtin's own bare name is never retired by any binding (see
# ``_name_may``). A mistake here can only drop a ``getattr``-alias
# finding (main has none); the requests/httpx ``.patch`` exemption does
# not consult these timelines. Every field that may not run, or may run
# out of textual order, belongs here.
#
# * conditional branches, loop bodies/targets, match cases,
# short-circuit operands, comprehension filters/elements;
# * a ``with``/``async with`` *item* and *body*: an earlier item
# such as ``contextlib.suppress`` can swallow an exception raised
# while a later item is evaluated or entered, skipping that item's
# ``as`` target and the whole body; and a context manager can
# swallow an exception raised partway through the body;
# * an ``assert``'s *test*/*msg* (both are stripped under ``-O``);
# * an assignment's *target* (Assign/AnnAssign): a walrus nested in
# a subscript target runs AFTER the right-hand side, so ordering it
# at its textual position would let it clear an alias the RHS still
# reads (an AugAssign target is evaluated before its RHS, so it
# stays ordered);
# * a For/AsyncFor/comprehension *target*: rebound per iteration,
# never if the iterable is empty;
# * every annotation (AnnAssign/parameter/return) and PEP 695 type
# parameter or ``type`` alias value: purely conservative. A walrus
# is a SyntaxError wherever these are evaluated lazily (PEP 695
# bounds and ``type`` values always; annotations under
# ``from __future__ import annotations`` or on 3.14+), and where an
# annotation still runs eagerly it runs in order — treating it as
# conditional only keeps an alias alive, so it can add a finding,
# never drop one.
#
# Everything else (an If/While *test*, a For *iter*, a Try
# *finalbody*, a Match *subject*, and a comprehension's *first*
# generator *iter*) evaluates unconditionally and in order.
_CONDITIONAL_CHILD_FIELDS = {
ast.If: frozenset({"body", "orelse"}),
ast.While: frozenset({"body", "orelse"}),
ast.For: frozenset({"target", "body", "orelse"}),
ast.AsyncFor: frozenset({"target", "body", "orelse"}),
ast.Try: frozenset({"body", "handlers", "orelse"}),
ast.Match: frozenset({"cases"}),
ast.With: frozenset({"items", "body"}),
ast.AsyncWith: frozenset({"items", "body"}),
ast.Assert: frozenset({"test", "msg"}),
ast.Assign: frozenset({"targets"}),
ast.AnnAssign: frozenset({"target", "annotation"}),
ast.arg: frozenset({"annotation"}),
ast.comprehension: frozenset({"target", "ifs"}),
ast.ListComp: frozenset({"elt"}),
ast.SetComp: frozenset({"elt"}),
ast.GeneratorExp: frozenset({"elt"}),
ast.DictComp: frozenset({"key", "value"}),
ast.BoolOp: frozenset({"values"}),
ast.IfExp: frozenset({"body", "orelse"}),
}
# ``ast.TryStar`` (PEP 654 ``except*``) only exists on Python 3.11+.
# This hook runs as a pre-commit ``language: script`` — no isolated
# venv — so it imports under whatever system Python is on PATH; a bare
# ``ast.TryStar`` reference above would raise ``AttributeError`` at
# import time on 3.10. Guard it and only add the entry when present;
# no behaviour change on 3.11+, where it is identical to ``ast.Try``.
_TRY_STAR = getattr(ast, "TryStar", None)
if _TRY_STAR is not None:
_CONDITIONAL_CHILD_FIELDS[_TRY_STAR] = frozenset(
{"body", "handlers", "orelse"}
)
# Same 3.10 guard for PEP 695 ``type X = ...`` (3.12+).
_TYPE_ALIAS = getattr(ast, "TypeAlias", None)
if _TYPE_ALIAS is not None:
_CONDITIONAL_CHILD_FIELDS[_TYPE_ALIAS] = frozenset({"type_params", "value"})
_DELETED_BINDING = object()
class SensitiveLoggingChecker(ast.NodeVisitor):
"""AST visitor to detect sensitive data in logging statements."""
def __init__(self, filename: str, source_lines: Optional[List[str]] = None):
self.filename = filename
self.errors: List[str] = []
# Track exception variable names from enclosing except handlers
self._except_var_stack: List[Optional[str]] = []
# Scope-local names that refer to the sys module. Used to catch
# ``import sys as s; s.exc_info()`` without leaking local aliases into
# unrelated functions.
self._sys_name_scopes = [{"sys"}]
self._shadowed_name_scopes = [set()]
self._name_scope_kinds = ["module"]
# Single normalized dir check reused by every dir-scoped rule so
# Windows paths cannot diverge between checks.
self.in_secure_dir = any(
d in filename.replace("\\", "/") for d in SECURE_LOGGING_DIRS
)
self.in_search_engine_dir = any(
d in filename.replace("\\", "/") for d in SEARCH_ENGINE_DIRS
)
# Only actual Python comments can justify a query-log exemption.
# source_lines must use physical LF boundaries, matching ast.parse.
# Tokenize once so marker text inside strings cannot silence a call.
self.query_log_suppression_lines: set[int] = set()
if self.in_search_engine_dir and source_lines:
try:
tokens = tokenize.generate_tokens(
io.StringIO("\n".join(source_lines)).readline
)
fstring_depth = 0
for token in tokens:
if token.type == FSTRING_START:
fstring_depth += 1
continue
if token.type == FSTRING_END:
fstring_depth = max(0, fstring_depth - 1)
continue
if token.type != tokenize.COMMENT or fstring_depth:
continue
# The marker has to open the comment, not merely appear
# somewhere in it, and the reason has to be written out.
text = token.string.lstrip("#").strip()
if not text.startswith(QUERY_LOG_SUPPRESSION):
continue
reason = text[len(QUERY_LOG_SUPPRESSION) :]
if (
len("".join(reason.split()))
< QUERY_LOG_REASON_MIN_CHARS
):
continue
self.query_log_suppression_lines.add(token.start[0])
except (tokenize.TokenError, IndentationError):
# Incomplete source cannot justify an exemption.
self.query_log_suppression_lines.clear()
# Per-file wrapper-import hint. Depth is counted from the
# local_deep_research package root so the leading-dot count matches
# the file's actual position in the tree.
self.wrapper_import_hint = self._compute_wrapper_import_hint()
# Populated by _prescan_safe_patch_receivers() the first time
# visit() sees the module root — see SAFE_PATCH_RECEIVER_MODULES.
self._safe_patch_receiver_names: set = set()
# Populated by _prescan_getattr_binding_state() — see
# _NAME_DOMAIN_BASELINES. Flattened module-level final state,
# consumed by the source-census tests; enforcement uses the
# position/scope-aware timelines below.
self._getattr_names = {"getattr"}
# Name binding events indexed per (domain, name, scope) as sorted
# positions plus the running replay state after each event, so
# a lookup is a bisection instead of a replay of the whole
# timeline (see _record_name_event / _name_may).
self._scope_timelines: dict = {
domain: {} for domain in _NAME_DOMAIN_BASELINES
}
# Attribute-held aliases (``h.ga = getattr``), keyed by the
# unparsed receiver expression plus attribute name so one
# receiver's rebind cannot erase another receiver's alias, and
# indexed per scope like the name events (see
# _record_attr_event / _attr_may_be).
self._attribute_timelines: dict = {
domain: {} for domain in _NAME_DOMAIN_BASELINES
}
# Lexical (node id, kind) scope stack mirroring the prescan's
# scope paths at query time.
self._lex_scopes: List[tuple] = []
# Definition position of every lexical scope the prescan
# created, bounding which enclosing-scope bindings a nested
# query is guaranteed to see.
self._scope_def_pos: dict = {}
# First namespace-vocabulary hit that revoked the requests/httpx
# exemption, as (line, spelling), for the diagnostic.
self._patch_exemption_revoked_by: Optional[tuple] = None
# Per-name revocations (a reference before the qualifying
# import), as {name: (line, description)}.
self._patch_exemption_revoked_for: dict = {}
# ids of Name nodes evaluated in a class scope: never exempted
# (see _prescan_safe_patch_receivers).
self._class_body_name_ids: set = set()
self._prescanned = False
def visit(self, node):
if not self._prescanned and isinstance(node, ast.Module):
self._prescanned = True
if self.in_secure_dir:
self._prescan_getattr_binding_state(node)
self._prescan_safe_patch_receivers(node)
return super().visit(node)
def _prescan_safe_patch_receivers(self, tree: ast.Module) -> None:
"""Whole-file denylist check that a bare name is not a logger.
Only a name that (a) is genuinely bound by a top-level unaliased
``import requests``/``import httpx`` that textually precedes
every reference to it and every ``def``/``lambda``/``class``
whose body references it, (b) is never bound by any other form
anywhere in the file, in a file that (c) contains no spelling
flagged by ``_namespace_vocabulary_hit`` qualifies — and even
then never where it is evaluated in a ``class`` scope. See
SAFE_PATCH_RECEIVER_MODULES. Nothing here models execution
order: one occurrence anywhere, reachable or not, revokes.
"""
import_positions: dict = {}
qualifying_alias_ids = set()
for stmt in tree.body:
if isinstance(stmt, ast.Import):
for alias in stmt.names:
if (
alias.name in SAFE_PATCH_RECEIVER_MODULES
and alias.asname is None
):
import_positions.setdefault(
alias.name, (stmt.lineno, stmt.col_offset)
)
qualifying_alias_ids.add(id(alias))
if not import_positions:
return
# A name looked up in a class body resolves through the class
# namespace first, which a metaclass ``__prepare__`` (or a
# ``__missing__`` on the mapping it returns) controls — possibly
# one inherited from a base class defined in another module — so
# a reference evaluated in any class scope is never exempted.
# Only the bodies of functions defined there (``def``/``lambda``)
# are skipped: they look names up in the module globals, never
# in the class namespace. Their decorators, defaults and
# annotations run in the class scope and stay covered.
for node in ast.walk(tree):
if isinstance(node, ast.ClassDef):
stack = list(node.body)
while stack:
inner = stack.pop()
if isinstance(inner, ast.Name):
self._class_body_name_ids.add(id(inner))
if isinstance(
inner, (ast.FunctionDef, ast.AsyncFunctionDef)
):
stack.extend(inner.decorator_list)
stack.append(inner.args)
if inner.returns is not None:
stack.append(inner.returns)
stack.extend(getattr(inner, "type_params", ()))
elif isinstance(inner, ast.Lambda):
stack.append(inner.args)
else:
stack.extend(ast.iter_child_nodes(inner))
hit = self._namespace_vocabulary_hit(tree)
if hit is not None:
self._patch_exemption_revoked_by = hit
return
rebound_names = set()
for node in ast.walk(tree):
self._collect_bound_names(node, qualifying_alias_ids, rebound_names)
early = self._references_before_import(tree, import_positions)
self._patch_exemption_revoked_for.update(early)
self._safe_patch_receiver_names = (
set(import_positions) - rebound_names - set(early)
)
@staticmethod
def _references_before_import(
tree: ast.Module, import_positions: dict
) -> dict:
"""Names referenced textually before their qualifying import.
Maps each such name to ``(line, description)`` of the earliest
offending reference, or ``def``/``lambda``/``class`` that starts
before the import and references the name in its body.
"""
early: dict = {}
for node in ast.walk(tree):
if isinstance(node, ast.Name):
names = {node.id} & set(import_positions)
start = (node.lineno, node.col_offset)
what = f"a reference to {node.id}"
elif isinstance(
node,
(
ast.FunctionDef,
ast.AsyncFunctionDef,
ast.Lambda,
ast.ClassDef,
),
):
start = min(
[(node.lineno, node.col_offset)]
+ [
(dec.lineno, dec.col_offset)
for dec in getattr(node, "decorator_list", ())
]
)
if not any(start < pos for pos in import_positions.values()):
continue
names = {
inner.id
for inner in ast.walk(node)
if isinstance(inner, ast.Name)
and inner.id in import_positions
}
what = f"a {type(node).__name__} that uses it"
else:
continue
for name in names:
if start < import_positions[name]:
found = (start[0], f"{what} before import {name}")
if name not in early or found < early[name]:
early[name] = found
return early
@staticmethod
def _namespace_vocabulary_hit(tree: ast.Module) -> Optional[tuple]:
"""Earliest spelling that can rebind a module global out of band.
Returns ``(line, description)`` for the first-by-line name,
attribute, imported name or identifier token inside a string/
bytes literal that is in _NAMESPACE_VOCABULARY; import of a
_NAMESPACE_MODULES module; ``getattr`` spelling other than a
direct call with a literal attribute name; attribute write
that targets a ``requests``/``httpx``/``patch`` attribute, a
dunder attribute, or is rooted at a ``requests``/``httpx`` name;
a load of a bare ``requests``/``httpx`` name that is not the
receiver of an attribute access (the module object used as a
value); an ``import requests``/``import httpx`` under another
name, or a ``from ... import requests``/``httpx``, at any scope;
``class`` with keywords (``metaclass=`` or any other); or a
direct top-level ``def __getattr__`` (not one nested in
``if``/``try``, and not an assignment to the name). ``None``
when clean.
"""
literal_getattr_funcs = {
id(node.func)
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and len(node.args) >= 2
and not any(isinstance(arg, ast.Starred) for arg in node.args)
and isinstance(node.args[1], ast.Constant)
and isinstance(node.args[1].value, str)
}
attribute_receiver_ids = {
id(node.value)
for node in ast.walk(tree)
if isinstance(node, ast.Attribute)
}
hits = [
(stmt.lineno, "a module-level def __getattr__")
for stmt in tree.body
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef))
and stmt.name == "__getattr__"
]
for node in ast.walk(tree):
line = getattr(node, "lineno", 0)
if isinstance(node, ast.ClassDef):
if node.keywords:
hits.append((line, f"keywords on class {node.name}"))
continue
if isinstance(node, ast.Name):
spelling = node.id
if (
spelling in SAFE_PATCH_RECEIVER_MODULES
and isinstance(node.ctx, ast.Load)
and id(node) not in attribute_receiver_ids
):
hits.append((line, f"{spelling} used as a value"))
elif isinstance(node, ast.Attribute):
spelling = node.attr
if isinstance(node.ctx, (ast.Store, ast.Del)):
root = node.value
while isinstance(root, (ast.Attribute, ast.Subscript)):
root = root.value
if (
node.attr in SAFE_PATCH_RECEIVER_MODULES
or node.attr == "patch"
or (
len(node.attr) > 4
and node.attr.startswith("__")
and node.attr.endswith("__")
)
or (
isinstance(root, ast.Name)
and root.id in SAFE_PATCH_RECEIVER_MODULES
)
):
hits.append((line, f"a write to .{node.attr}"))
elif isinstance(node, ast.Constant) and isinstance(
node.value, (str, bytes)
):
text = (
node.value.decode("latin-1")
if isinstance(node.value, bytes)
else node.value
)
found = sorted(
set(_IDENTIFIER_TOKEN.findall(text)) & _NAMESPACE_VOCABULARY
)
if found:
hits.append((line, f"{found[0]!r} in a string literal"))
continue
elif isinstance(node, (ast.Import, ast.ImportFrom)):
for alias in node.names:
if isinstance(node, ast.ImportFrom):
base = node.module or ""
modules = [base, f"{base}.{alias.name}".strip(".")]
else:
modules = [alias.name]
words = {
part for module in modules for part in module.split(".")
}
words.update({alias.name, alias.asname})
if (words & _NAMESPACE_VOCABULARY) or any(
module in _NAMESPACE_MODULES
or module.split(".")[0] in _NAMESPACE_MODULES
for module in modules
):
hits.append((line, f"an import of {modules[-1]}"))
# A second binding of requests/httpx under another
# name (``import requests as rq``, at any scope) can
# be handed to code this rule cannot follow without
# the bare name ever being used as a value. Revoke
# the exemption rather than track every alias.
if (
isinstance(node, ast.Import)
and alias.name.split(".")[0]
in SAFE_PATCH_RECEIVER_MODULES
and alias.asname not in (None, alias.name)
):
hits.append(
(line, f"{alias.name} imported as {alias.asname}")
)
elif (
isinstance(node, ast.ImportFrom)
and alias.name in SAFE_PATCH_RECEIVER_MODULES
):
# ``from x import requests [as y]`` binds some
# object as (or instead of) the module.
hits.append(
(line, f"{alias.name} imported from {base or '.'}")
)
continue
else:
continue
if spelling in _NAMESPACE_VOCABULARY:
hits.append((line, spelling))
elif spelling == "getattr" and id(node) not in (
literal_getattr_funcs
):
hits.append((line, "a computed getattr"))
return min(hits) if hits else None
def _prescan_getattr_binding_state(self, tree: ast.Module) -> None:
"""Build position/scope-aware state for tracked builtin names.
The earlier monotonic alias set marked a name forever, so a
definite rebind or parameter shadow still produced a diagnostic.
This records every binding on a timeline instead. A binding is
``may`` when its value is a direct or module-qualified spelling
of ``getattr`` (a ``getattr`` alias, ``builtins.getattr``), a
statically knowable alias, or a conditional containing one.
Other definitely executed bindings clear an ALIAS; they never
clear the bare ``getattr`` itself (see ``_name_may``).
``global``/``nonlocal`` declarations remap each write to the
scope Python actually mutates, and attribute-held aliases are
keyed by receiver so one receiver's rebind cannot erase
another's. These timelines only ever ADD ``getattr(...)``
diagnostics; the requests/httpx ``.patch`` exemption does not
consult them.
"""
pending = []
pending_attrs = []
scope_decls: dict = {}
scope_locals: dict = {}
def local_bindings(scope_node):
names = (
set(self._func_param_names(scope_node.args))
if hasattr(scope_node, "args")
else set()
)
stack = list(ast.iter_child_nodes(scope_node))
while stack:
child = stack.pop()
if isinstance(
child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
):
names.add(child.name)
continue
if isinstance(child, ast.Lambda):
continue
if isinstance(child, ast.Name) and isinstance(
child.ctx, (ast.Store, ast.Del)
):
names.add(child.id)
elif isinstance(child, (ast.Import, ast.ImportFrom)):
names.update(
alias.asname or alias.name.split(".")[0]
for alias in child.names
)
elif isinstance(child, ast.ExceptHandler) and child.name:
names.add(child.name)
elif (
isinstance(child, (ast.MatchAs, ast.MatchStar))
and child.name
):
names.add(child.name)
elif isinstance(child, ast.MatchMapping) and child.rest:
names.add(child.rest)
stack.extend(ast.iter_child_nodes(child))
global_names, nonlocal_names = scope_declarations(scope_node)
return names - global_names - nonlocal_names
def scope_declarations(scope_node):
"""global/nonlocal names declared in this exact scope."""
globals_seen = set()
nonlocals_seen = set()
stack = list(ast.iter_child_nodes(scope_node))
while stack:
child = stack.pop()
if isinstance(
child,
(
ast.FunctionDef,
ast.AsyncFunctionDef,
ast.Lambda,
ast.ClassDef,
),
):
continue
if isinstance(child, ast.Global):
globals_seen.update(child.names)
elif isinstance(child, ast.Nonlocal):
nonlocals_seen.update(child.names)
stack.extend(ast.iter_child_nodes(child))
return globals_seen, nonlocals_seen
def binding_scope(path, name):
"""Lexical scope a binding of ``name`` at ``path`` lands in."""
if not path:
return path
globals_seen, nonlocals_seen = scope_decls.get(path, ((), ()))
if name in globals_seen:
return ()
if name in nonlocals_seen:
for depth in range(len(path) - 1, 0, -1):
candidate = path[:depth]
if path[depth - 1][
1
] == "function" and name in scope_locals.get(candidate, ()):
return candidate
return path
def record(name, value, pos, path, definite=True, value_path=None):
target_path = binding_scope(path, name)
# A function body's outer write may never run. In particular,
# its clear cannot retire a builtin at module definition time.
definite = definite and target_path == path
for domain in _NAME_DOMAIN_BASELINES:
pending.append(
(
pos,
domain,
name,
value,
target_path,
definite,
path if value_path is None else value_path,
)
)
def record_target(target, value, path, definite=True, pos=None):
# Assignment evaluates the RHS before any target is bound.
if pos is None:
pos = (target.end_lineno, target.end_col_offset)
if isinstance(target, ast.Name):
record(
target.id,
value,
pos,
path,
definite,
)
elif isinstance(target, (ast.Tuple, ast.List)):
if (
isinstance(value, (ast.Tuple, ast.List))
and len(value.elts) == len(target.elts)
and not any(
isinstance(element, ast.Starred)
for element in value.elts
)
):
for element, element_value in zip(target.elts, value.elts):
record_target(
element, element_value, path, definite, pos
)
else:
for element in target.elts:
record_target(element, value, path, definite, pos)
elif isinstance(target, ast.Starred):
record_target(target.value, value, path, definite, pos)
elif isinstance(target, ast.Attribute):
pending_attrs.append(
(
pos,
(ast.unparse(target.value), target.attr),
value,
path,
definite,
)
)
def record_params(args, path):
positional = list(args.posonlyargs) + list(args.args)
first_default = len(positional) - len(args.defaults)
for index, arg in enumerate(positional):
default = (
args.defaults[index - first_default]
if index >= first_default
else None
)
record(
arg.arg,
default if default is not None else False,
(args.end_lineno, args.end_col_offset)
if hasattr(args, "end_lineno")
else (arg.lineno, arg.col_offset),
path,
value_path=path[:-1],
)
for arg, default in zip(args.kwonlyargs, args.kw_defaults):
record(
arg.arg,
default if default is not None else False,
(args.end_lineno, args.end_col_offset)
if hasattr(args, "end_lineno")
else (arg.lineno, arg.col_offset),
path,
value_path=path[:-1],
)
for arg in (args.vararg, args.kwarg):
if arg is not None:
record(
arg.arg,
False,
(arg.lineno, arg.col_offset),
path,
)
def walk(node, path, definite=True):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
record(
node.name,
False,
(node.end_lineno, node.end_col_offset),
path,
definite,
)
inner = path + ((id(node), "function"),)
self._scope_def_pos[inner] = (
node.lineno,
node.col_offset,
)
scope_decls[inner] = scope_declarations(node)
scope_locals[inner] = local_bindings(node)
record_params(node.args, inner)
walk(node.args, path, definite)
for statement in node.body:
walk(statement, inner)
for decorator in node.decorator_list:
walk(decorator, path, definite)
# Lazily evaluated (see _CONDITIONAL_CHILD_FIELDS).
if node.returns is not None:
walk(node.returns, path, False)
for type_param in getattr(node, "type_params", ()) or ():
walk(type_param, path, False)
return
if isinstance(node, ast.Lambda):
inner = path + ((id(node), "function"),)
self._scope_def_pos[inner] = (
node.lineno,
node.col_offset,
)
scope_decls[inner] = scope_declarations(node)
scope_locals[inner] = local_bindings(node)
record_params(node.args, inner)
walk(node.args, path, definite)
walk(node.body, inner)
return
if isinstance(node, ast.ClassDef):
record(
node.name,
False,
(node.end_lineno, node.end_col_offset),
path,
definite,
)
inner = path + ((id(node), "class"),)
self._scope_def_pos[inner] = (
node.lineno,
node.col_offset,
)
scope_decls[inner] = scope_declarations(node)
scope_locals[inner] = local_bindings(node)
for decorator in node.decorator_list:
walk(decorator, path, definite)
for base in node.bases:
walk(base, path, definite)
for keyword in node.keywords:
walk(keyword, path, definite)
for statement in node.body:
walk(statement, inner)
return
if isinstance(node, ast.Assign):
for target in node.targets:
record_target(
target,
node.value,
path,
definite,
(node.end_lineno, node.end_col_offset),
)
elif isinstance(node, ast.AnnAssign) and node.value is not None:
record_target(
node.target,
node.value,
path,
definite,
(node.end_lineno, node.end_col_offset),
)
elif isinstance(node, ast.AugAssign):
record_target(
node.target,
node.target,
path,
definite,
(node.end_lineno, node.end_col_offset),
)
elif isinstance(node, ast.NamedExpr):
record_target(
node.target,
node.value,
path,
definite,
(node.end_lineno, node.end_col_offset),
)
elif isinstance(node, (ast.For, ast.AsyncFor)):
record_target(node.target, False, path, False)
elif isinstance(node, ast.comprehension):
record_target(node.target, False, path, False)
elif isinstance(node, (ast.With, ast.AsyncWith)):
# Never definite: an earlier item (``suppress``) can
# swallow the exception that skips this ``as`` target.
for item in node.items:
if item.optional_vars is not None:
record_target(item.optional_vars, False, path, False)
elif isinstance(node, ast.Delete):
names = set()
for target in node.targets:
self._collect_target_names(target, names)
for name in names:
record(
name,
_DELETED_BINDING,
(node.end_lineno, node.end_col_offset),
path,
definite,
)
elif isinstance(node, (ast.MatchAs, ast.MatchStar)):
if node.name is not None:
record(
node.name,
False,
(node.lineno, node.col_offset),
path,
False,
)
elif isinstance(node, ast.MatchMapping):
if node.rest is not None:
record(
node.rest,
False,
(node.lineno, node.col_offset),
path,
False,
)
elif isinstance(node, ast.ExceptHandler):
if node.name is not None:
record(
node.name,
False,
(node.lineno, node.col_offset),
path,
False,
)
# Python deletes the handler name when the handler
# ends; in a class body the lookup then falls back
# to the enclosing (module) binding.
record(
node.name,
_DELETED_BINDING,
(node.end_lineno, node.end_col_offset),
path,
False,
)
elif isinstance(node, (ast.Import, ast.ImportFrom)):
pos = (node.end_lineno, node.end_col_offset)
for alias in node.names:
if alias.name == "*":
continue
is_import = isinstance(node, ast.Import)
bound = alias.asname or (
alias.name.split(".")[0] if is_import else alias.name
)
from_builtins = (
not is_import
and node.module == "builtins"
and node.level == 0
)
target_path = binding_scope(path, bound)
for domain in _NAME_DOMAIN_BASELINES:
may = (
is_import and alias.name == "builtins"
if domain == "builtins"
else from_builtins
and alias.name == _BUILTIN_FN_LEAF.get(domain)
)
pending.append(
(
pos,
domain,
bound,
may,
target_path,
definite and target_path == path,
path,
)
)
conditional_fields = _CONDITIONAL_CHILD_FIELDS.get(type(node))
for field, value in ast.iter_fields(node):
if isinstance(value, ast.AST):
children = [value]
elif (
isinstance(value, list)
and value
and all(isinstance(item, ast.AST) for item in value)
):
children = value
else:
continue
child_definite = definite and not (
conditional_fields is not None
and field in conditional_fields
)
for child in children:
walk(child, path, child_definite)
walk(tree, ())
# Name and attribute events interleave in position order so an
# attribute alias established earlier (``h.ga = getattr``) is
# visible to a name bound from it later (``lookup = h.ga``) and
# vice versa.
events = [
(pos, 0, (domain, key), value, path, definite, path)
for pos, key, value, path, definite in pending_attrs
for domain in _NAME_DOMAIN_BASELINES
]
events.extend(
(pos, 1, (domain, name), value, path, definite, value_path)
for pos, domain, name, value, path, definite, value_path in pending
)
events.sort(key=lambda event: event[0])
for pos, kind, key, value, path, definite, value_path in events:
domain = key[0]
may = (
value
if isinstance(value, bool) or value is _DELETED_BINDING
else self._expr_may_be(
value,
domain,
(value.lineno, value.col_offset),
value_path,
)
)
if kind == 0:
self._record_attr_event(
domain, key[1], pos, may, definite, path
)
else:
self._record_name_event(
domain, key[1], pos, may, definite, path
)
self._getattr_names = {"getattr"} | {
name
for name in self._scope_timelines["getattr"]
if self._name_may(name, "getattr", (10**9, 0), ())
}
def _expr_may_be(self, expr, domain: str, pos, path) -> bool:
"""Whether an expression may evaluate to the tracked domain."""
if isinstance(expr, ast.Name):
return self._name_may(
expr.id,
domain,
(expr.lineno, expr.col_offset),
path,
)
if isinstance(expr, ast.NamedExpr):
return self._expr_may_be(expr.value, domain, pos, path)
if isinstance(expr, ast.IfExp):
return self._expr_may_be(
expr.body, domain, pos, path
) or self._expr_may_be(expr.orelse, domain, pos, path)
if isinstance(expr, ast.BoolOp):
return any(
self._expr_may_be(value, domain, pos, path)
for value in expr.values
)
if isinstance(expr, ast.Starred):
return self._expr_may_be(expr.value, domain, pos, path)
if isinstance(expr, (ast.Tuple, ast.List)):
return any(
self._expr_may_be(element, domain, pos, path)
for element in expr.elts
)
leaf = _BUILTIN_FN_LEAF.get(domain)
if isinstance(expr, ast.Attribute):
if (
leaf is not None
and expr.attr == leaf
and (
# ``__import__("builtins").getattr``: a computed
# receiver cannot be proven not to be builtins.
not isinstance(expr.value, ast.Name)
or self._name_may(
expr.value.id,
"builtins",
(expr.value.lineno, expr.value.col_offset),
path,
)
)
):
return True
return self._attr_may_be(expr, domain, pos, path)
# Selection out of a container that may hold the builtin
# (``[getattr][0]``, ``{"k": getattr}["k"]``, ``x[0]`` where
# ``x = (getattr,)``), or a lookup keyed by the builtin's own
# name (``vars(builtins)["getattr"]``,
# ``builtins.__dict__["getattr"]``).
if isinstance(expr, ast.Subscript):
if (
leaf is not None
and isinstance(expr.slice, ast.Constant)
and expr.slice.value == leaf
):
return True
return self._expr_may_be(expr.value, domain, pos, path)
if isinstance(expr, ast.Dict):
return any(
self._expr_may_be(value, domain, pos, path)
for value in expr.values
)
if isinstance(expr, ast.Set):
return any(
self._expr_may_be(element, domain, pos, path)
for element in expr.elts
)
if isinstance(expr, ast.Call):
# ``(lambda: getattr)()`` returns its body.
if isinstance(expr.func, ast.Lambda):
return self._expr_may_be(expr.func.body, domain, pos, path)
# ``getattr(builtins, "getattr")``: a literal lookup of the
# builtin's own name through a getattr spelling.
if (
leaf is not None
and len(expr.args) >= 2
and isinstance(expr.args[1], ast.Constant)
and expr.args[1].value == leaf
and self._expr_may_be(expr.func, "getattr", pos, path)
):
return True
return False
def _name_may(self, name: str, domain: str, pos, path) -> bool:
"""Resolve one name's nearest visible binding at ``pos``.
The bare ``getattr`` (the baseline member of
_NAME_DOMAIN_BASELINES) ALWAYS may be that builtin, whatever
the file binds to it, exactly as on main. Deciding that a
rebind definitely retired the builtin requires modelling every
way a binding can fail to run or be undone (a suppressed
``with`` item, an ``except ... as`` handler's implicit ``del``,
a walrus in an assignment target that runs after the RHS, an
opaque RHS that still yields the builtin); getting any one of
them wrong silently removed a diagnostic main emits. Timelines
therefore only ever ADD aliases (``ga = getattr``); they never
subtract the builtin itself.
"""
if name in _NAME_DOMAIN_BASELINES[domain]:
return True
scopes = self._scope_timelines[domain].get(name)
if not scopes:
return False
cutoff = self._scope_def_pos.get(path)
for depth in range(len(path), -1, -1):
scope = path[:depth]
entry = scopes.get(scope)
if entry is None or not self._scope_visible(scope, path):
continue
positions, states, last_alias_pos = entry
if depth == len(path):
# A class-body name that MAY have been deleted (a
# conditional ``del``, an ``except ... as`` handler's
# implicit delete) may resolve to the enclosing binding.
index = bisect.bisect_right(positions, pos)
if index:
state, fallthrough = states[index - 1]
else:
state = None if scope and scope[-1][1] == "class" else False
fallthrough = False
if state is True:
return True
if state is None or fallthrough:
continue
return state
# Enclosing scope: the querying scope's body runs no earlier
# than its own definition, so only bindings established
# there are guaranteed visible. Later writes count only
# when they (re)introduce an alias — a later definite clear
# cannot be relied on at an unknown call time.
if cutoff is not None:
index = bisect.bisect_right(positions, cutoff)
if index and states[index - 1][0] is True:
return True
return last_alias_pos is not None and (
cutoff is None or last_alias_pos > cutoff
)
return False
def _record_name_event(self, domain, name, pos, may, definite, path):
"""Append one binding event to the per-scope replay index.
Events arrive in position order, so each scope's positions stay
sorted and ``states[i]`` is the replay state after event ``i``:
an alias binding sets it, a definite clear or delete resets it
(a class body resets to "unbound here", falling through to the
enclosing scope), and a conditional delete in a class body
marks a possible fall-through.
"""
entry = self._scope_timelines[domain].setdefault(name, {}).get(path)
is_class = bool(path and path[-1][1] == "class")
if entry is None:
entry = [[], [], None]
self._scope_timelines[domain][name][path] = entry
state, fallthrough = (None if is_class else False), False
else:
state, fallthrough = entry[1][-1]
if may is _DELETED_BINDING:
if definite:
state = None if is_class else False
fallthrough = False
elif is_class:
fallthrough = True
elif may:
state = True
entry[2] = pos
elif definite:
state = False
fallthrough = False
entry[0].append(pos)
entry[1].append((state, fallthrough))
@staticmethod
def _scope_visible(event_path, query_path) -> bool:
"""Whether a lexical binding scope is visible at a call site."""
if len(event_path) > len(query_path):
return False
if query_path[: len(event_path)] != event_path:
return False
if event_path and event_path[-1][1] == "class":
return len(event_path) == len(query_path)
return True
def _class_body_attr_may(self, attr: str, domain: str = "getattr") -> bool:
"""Whether a class body binds ``attr`` to a getattr alias.
``class H: ga = getattr`` puts the alias on every instance,
and an instance's class cannot be recovered statically, so any
``<obj>.ga`` may reach it. A definite same-class rebind retires
it; a rebind in a different class does not.
"""
return any(
scope and scope[-1][1] == "class" and states[-1][0] is True
for scope, (_, states, _) in self._scope_timelines[domain]
.get(attr, {})
.items()
)
def _attr_may_be_getattr(self, attr_node: ast.Attribute, pos, path) -> bool:
return self._attr_may_be(attr_node, "getattr", pos, path)
def _attr_may_be(
self, attr_node: ast.Attribute, domain: str, pos, path
) -> bool:
"""Resolve an attribute-held alias (``h.ga = getattr``).
The timeline is keyed by the unparsed receiver expression
plus attribute name, so another receiver's definite rebind
cannot erase a live alias. Same-scope writes replay in
position order; enclosing-scope writes are bounded by the
querying scope's definition (a later definite clear cannot be
relied on); and a write in any other function/class scope may
execute in any order relative to the call, so a ``may`` there
always sticks.
"""
attr = attr_node.attr
if self._class_body_attr_may(attr, domain):
return True
index = self._attribute_timelines[domain].get(
(ast.unparse(attr_node.value), attr)
)
if index is None:
return False
scopes, alias_scopes = index
entry = scopes.get(path)
if entry is not None:
at = bisect.bisect_right(entry[0], pos)
if at and entry[1][at - 1] is not None and entry[1][at - 1][1]:
return True
cutoff = self._scope_def_pos.get(path)
latest = None
for depth in range(len(path)):
entry = scopes.get(path[:depth])
if entry is None:
continue
positions, decisive, last_alias_pos = entry
if cutoff is not None:
at = bisect.bisect_right(positions, cutoff)
if at and decisive[at - 1] is not None:
if latest is None or decisive[at - 1][0] > latest[0]:
latest = decisive[at - 1]
if last_alias_pos is not None and (
cutoff is None or last_alias_pos > cutoff
):
return True
if latest is not None and latest[1]:
return True
# Any other scope: execution order relative to the call is
# unknowable, so only a live alias matters.
visible = sum(
path[:depth] in alias_scopes for depth in range(len(path) + 1)
)
return len(alias_scopes) > visible
def _record_attr_event(self, domain, key, pos, may, definite, path):
"""Append one attribute-alias event to the per-scope index.
Per scope: sorted positions, the last decisive event
``(position, is_alias)`` after each event (an alias write or a
definite clear; a conditional clear decides nothing), and the
position of the last alias write. ``alias_scopes`` holds every
scope that ever writes an alias.
"""
scopes, alias_scopes = self._attribute_timelines[domain].setdefault(
key, ({}, set())
)
entry = scopes.setdefault(path, [[], [], None])
decisive = entry[1][-1] if entry[1] else None
if may:
decisive = (pos, True)
entry[2] = pos
alias_scopes.add(path)
elif definite:
decisive = (pos, False)
entry[0].append(pos)
entry[1].append(decisive)
@staticmethod
def _collect_bound_names(
node: ast.AST, qualifying_alias_ids: set, bound: set
) -> None:
"""Record every name ``node`` binds, deletes or shadows.
``qualifying_alias_ids`` are the top-level aliases that ESTABLISH
the exemption, so they alone do not invalidate it. A star import
binds unknowable names and records every exempted name.
"""
if isinstance(node, ast.Name):
if isinstance(node.ctx, (ast.Store, ast.Del)):
bound.add(node.id)
elif isinstance(node, ast.arg):
bound.add(node.arg)
elif isinstance(
node,
(
ast.FunctionDef,
ast.AsyncFunctionDef,
ast.ClassDef,
ast.ExceptHandler,
ast.MatchAs,
ast.MatchStar,
),
) or type(node).__name__ in ("TypeVar", "ParamSpec", "TypeVarTuple"):
if node.name is not None:
bound.add(node.name)
elif isinstance(node, ast.MatchMapping):
if node.rest is not None:
bound.add(node.rest)
elif isinstance(node, ast.Import):
for alias in node.names:
if id(alias) not in qualifying_alias_ids:
bound.add(alias.asname or alias.name.split(".")[0])
elif isinstance(node, ast.ImportFrom):
for alias in node.names:
if alias.name == "*":
bound.update(SAFE_PATCH_RECEIVER_MODULES)
else:
bound.add(alias.asname or alias.name)
def _collect_target_names(self, target: ast.AST, names: set) -> None:
if isinstance(target, ast.Name):
names.add(target.id)
elif isinstance(target, (ast.Tuple, ast.List)):
for elt in target.elts:
self._collect_target_names(elt, names)
elif isinstance(target, ast.Starred):
self._collect_target_names(target.value, names)
def _is_safe_patch_receiver(self, expr: ast.AST) -> bool:
"""True only for the narrow, denylist-gated ``.patch`` case.
See SAFE_PATCH_RECEIVER_MODULES: a bare ``requests``/``httpx``
name, not evaluated in any ``class`` scope, that is imported at top
level before any reference to it, bound by nothing else in the
file, in a file with none of the denylisted spellings or
structures. This is a denylist of in-file spellings plus those
structural conditions, not a proof: it cannot see other modules
(which can write this module's globals or ``sys.modules``), a
shadowing ``requests.py``/``httpx.py`` on ``sys.path``, or a
namespace-reaching API the lists do not name. It is
intentionally NOT a general receiver-shape allowlist — anything
else (attributes, subscripts, calls, parameters, dict/list
elements, ternaries, aliases) stays banned by default.
"""
return (
isinstance(expr, ast.Name)
and expr.id in self._safe_patch_receiver_names
and id(expr) not in self._class_body_name_ids
)
def _patch_exemption_note(self, expr: ast.AST) -> str:
"""Explain a revoked requests/httpx exemption in the diagnostic."""
if (
not isinstance(expr, ast.Name)
or expr.id not in SAFE_PATCH_RECEIVER_MODULES
):
return ""
revoked = self._patch_exemption_revoked_by
if revoked is None:
revoked = self._patch_exemption_revoked_for.get(expr.id)
if revoked is not None:
return (
f" (the {expr.id}.patch exemption is revoked for this file "
f"by {revoked[1]} on line {revoked[0]})"
)
if id(expr) in self._class_body_name_ids:
return (
f" (the {expr.id}.patch exemption never applies inside a "
f"class body)"
)
return ""
def _compute_wrapper_import_hint(self) -> str:
"""Compute the per-file import hint with the correct dot count.
Depth = number of package levels from (and including)
``local_deep_research`` down to the file's parent directory. Files
outside ``src/local_deep_research/...`` fall back to the generic
constant hint.
"""
normalized = self.filename.replace("\\", "/")
marker = "/local_deep_research/"
idx = normalized.find(marker)
if idx == -1:
return WRAPPER_IMPORT_HINT
tail = normalized[idx + len(marker) :]
# tail is e.g. "llm/providers/implementations/x.py"; count slashes
# in the directory portion (= package depth below local_deep_research).
if "/" in tail:
dir_part = tail.rsplit("/", 1)[0]
else:
dir_part = "" # file sits directly under local_deep_research/
if not dir_part:
depth = 1 # local_deep_research package itself
else:
depth = dir_part.count("/") + 2
dots = "." * depth
return (
f"use 'from {dots}security.secure_logging import logger' "
f"(relative depth per file)"
)
def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None:
"""Track exception variable names inside except blocks."""
if self.in_secure_dir and node.name == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"'except ... as logger' shadows the secure logger — "
f"rename the exception variable"
)
previous_shadowed_names = set(self._shadowed_name_scopes[-1])
if self.in_secure_dir and node.name:
self._mark_name_shadow(node.name)
previous_except_stack_len = len(self._except_var_stack)
self._except_var_stack.append(node.name)
self.generic_visit(node)
# A match statement may add aliases that remain bound for the rest of
# this handler. Discard the handler name and all such aliases together.
del self._except_var_stack[previous_except_stack_len:]
self._shadowed_name_scopes[-1] = previous_shadowed_names
@staticmethod
def _pattern_bound_names(pattern: ast.AST) -> List[str]:
"""Collect every name bound anywhere in a match pattern."""
names: List[str] = []
if pattern is None:
return names
if isinstance(pattern, ast.MatchAs):
if pattern.name:
names.append(pattern.name)
names.extend(
SensitiveLoggingChecker._pattern_bound_names(pattern.pattern)
)
elif isinstance(pattern, ast.MatchSequence):
for child in pattern.patterns:
names.extend(
SensitiveLoggingChecker._pattern_bound_names(child)
)
elif isinstance(pattern, ast.MatchStar):
if pattern.name:
names.append(pattern.name)
elif isinstance(pattern, ast.MatchMapping):
for child in pattern.patterns:
names.extend(
SensitiveLoggingChecker._pattern_bound_names(child)
)
if pattern.rest:
names.append(pattern.rest)
elif isinstance(pattern, ast.MatchClass):
for child in pattern.patterns:
names.extend(
SensitiveLoggingChecker._pattern_bound_names(child)
)
for child in pattern.kwd_patterns:
names.extend(
SensitiveLoggingChecker._pattern_bound_names(child)
)
elif isinstance(pattern, ast.MatchOr):
for child in pattern.patterns:
names.extend(
SensitiveLoggingChecker._pattern_bound_names(child)
)
return names
@staticmethod
def _subject_is_exception(
expr: ast.AST, active: set, exception_expr_ids=None
) -> bool:
"""Whether an expression evaluates to an active exception itself."""
if exception_expr_ids is not None and id(expr) in exception_expr_ids:
return True
if isinstance(expr, ast.Name):
return expr.id in active
if isinstance(expr, ast.NamedExpr):
if isinstance(expr.target, ast.Name) and expr.target.id in active:
return True
return SensitiveLoggingChecker._subject_is_exception(
expr.value, active, exception_expr_ids
)
return False
def _apply_definite_subject_assignments(
self, expr: ast.AST, exception_expr_ids: set
) -> None:
"""Apply walrus transfers guaranteed while evaluating a subject.
Tuple/list/set elements and dict keys/values evaluate in order before
matching begins. Boolean and conditional expressions are deliberately
not descended because their branches may be skipped.
"""
if isinstance(expr, ast.NamedExpr):
self._apply_definite_subject_assignments(
expr.value, exception_expr_ids
)
if isinstance(expr.target, ast.Name):
active = {
name for name in self._except_var_stack if name is not None
}
value_is_exception = self._subject_is_exception(
expr.value, active, exception_expr_ids
)
self._except_var_stack[:] = [
name
for name in self._except_var_stack
if name != expr.target.id
]
if value_is_exception:
exception_expr_ids.add(id(expr))
self._except_var_stack.append(expr.target.id)
elif isinstance(expr, ast.Name):
if expr.id in self._except_var_stack:
exception_expr_ids.add(id(expr))
elif isinstance(expr, (ast.Tuple, ast.List, ast.Set)):
for element in expr.elts:
if isinstance(element, ast.Starred):
element = element.value
self._apply_definite_subject_assignments(
element, exception_expr_ids
)
elif isinstance(expr, ast.Dict):
for key, value in zip(expr.keys, expr.values):
if key is not None:
self._apply_definite_subject_assignments(
key, exception_expr_ids
)
self._apply_definite_subject_assignments(
value, exception_expr_ids
)
@classmethod
def _pattern_exception_aliases(
cls,
subject: ast.AST,
pattern: ast.AST,
active: set,
exception_expr_ids=None,
) -> List[str]:
"""Map provable exception identities from a subject into a pattern."""
aliases: List[str] = []
if pattern is None:
return aliases
subject_is_exception = cls._subject_is_exception(
subject, active, exception_expr_ids
)
if isinstance(subject, ast.NamedExpr):
if subject_is_exception and isinstance(subject.target, ast.Name):
aliases.append(subject.target.id)
subject = subject.value
if isinstance(pattern, ast.MatchAs):
if subject_is_exception and pattern.name:
aliases.append(pattern.name)
aliases.extend(
cls._pattern_exception_aliases(
subject,
pattern.pattern,
active,
exception_expr_ids,
)
)
elif isinstance(pattern, ast.MatchOr):
for alternative in pattern.patterns:
aliases.extend(
cls._pattern_exception_aliases(
subject,
alternative,
active,
exception_expr_ids,
)
)
elif isinstance(pattern, ast.MatchSequence) and isinstance(
subject, (ast.Tuple, ast.List)
):
pattern_star = next(
(
index
for index, child in enumerate(pattern.patterns)
if isinstance(child, ast.MatchStar)
),
None,
)
subject_star = next(
(
index
for index, child in enumerate(subject.elts)
if isinstance(child, ast.Starred)
),
None,
)
if pattern_star is None and subject_star is None:
if len(subject.elts) != len(pattern.patterns):
return aliases
for element, child_pattern in zip(
subject.elts, pattern.patterns
):
aliases.extend(
cls._pattern_exception_aliases(
element,
child_pattern,
active,
exception_expr_ids,
)
)
elif pattern_star is not None:
# A MatchStar capture receives a new list, but fixed prefix
# and suffix positions still have provable correspondence.
prefix_count = pattern_star
for index in range(prefix_count):
if index >= len(subject.elts) or (
subject_star is not None and index >= subject_star
):
break
aliases.extend(
cls._pattern_exception_aliases(
subject.elts[index],
pattern.patterns[index],
active,
exception_expr_ids,
)
)
suffix_count = len(pattern.patterns) - pattern_star - 1
for offset in range(1, suffix_count + 1):
subject_index = len(subject.elts) - offset
if subject_index < 0 or (
subject_star is not None
and subject_index <= subject_star
):
break
aliases.extend(
cls._pattern_exception_aliases(
subject.elts[subject_index],
pattern.patterns[-offset],
active,
exception_expr_ids,
)
)
else:
# The subject contains a starred expansion but the pattern is
# fixed-length. Its explicit prefix and suffix still line up
# whenever the case matches.
for index in range(subject_star):
if index >= len(pattern.patterns):
break
aliases.extend(
cls._pattern_exception_aliases(
subject.elts[index],
pattern.patterns[index],
active,
exception_expr_ids,
)
)
suffix_count = len(subject.elts) - subject_star - 1
for offset in range(1, suffix_count + 1):
if offset > len(pattern.patterns):
break
aliases.extend(
cls._pattern_exception_aliases(
subject.elts[-offset],
pattern.patterns[-offset],
active,
exception_expr_ids,
)
)
elif isinstance(pattern, ast.MatchMapping) and isinstance(
subject, ast.Dict
):
subject_values = {
key.value: value
for key, value in zip(subject.keys, subject.values)
if isinstance(key, ast.Constant)
}
for key, child_pattern in zip(pattern.keys, pattern.patterns):
if (
isinstance(key, ast.Constant)
and key.value in subject_values
):
aliases.extend(
cls._pattern_exception_aliases(
subject_values[key.value],
child_pattern,
active,
exception_expr_ids,
)
)
return aliases
def visit_Match(self, node: ast.Match) -> None:
"""Track match bindings separately from proven exception aliases."""
if not self.in_secure_dir:
self.generic_visit(node)
return
previous_subject_exception_vars = list(self._except_var_stack)
self.visit(node.subject)
self._except_var_stack[:] = previous_subject_exception_vars
subject_exception_expr_ids: set = set()
self._apply_definite_subject_assignments(
node.subject, subject_exception_expr_ids
)
aliases_after_match: List[str] = []
for case in node.cases:
active = {
name for name in self._except_var_stack if name is not None
}
all_bound = list(
dict.fromkeys(self._pattern_bound_names(case.pattern))
)
aliases = list(
dict.fromkeys(
self._pattern_exception_aliases(
node.subject,
case.pattern,
active,
subject_exception_expr_ids,
)
)
)
previous_except_vars = list(self._except_var_stack)
previous_sys_names = set(self._sys_name_scopes[-1])
previous_shadowed_names = set(self._shadowed_name_scopes[-1])
# Pattern bindings definitely shadow prior meanings in the guard
# and body of this case. Add back only proven exception aliases.
bound_set = set(all_bound)
self._except_var_stack[:] = [
name for name in self._except_var_stack if name not in bound_set
]
for name in all_bound:
self._flag_logger_binding(
ast.Name(id=name, ctx=ast.Store()), case.pattern.lineno
)
self._mark_name_shadow(name)
self._except_var_stack.extend(aliases)
self.visit(case.pattern)
guard_state: List[Optional[str]] = []
if case.guard is not None:
self.visit(case.guard)
guard_state = list(self._except_var_stack)
for statement in case.body:
self.visit(statement)
body_state = list(self._except_var_stack)
self._except_var_stack[:] = previous_except_vars
self._sys_name_scopes[-1] = previous_sys_names
self._shadowed_name_scopes[-1] = previous_shadowed_names
if case.guard is not None:
aliases_after_match.extend(guard_state)
aliases_after_match.extend(body_state)
# A guard can fail after binding names, allowing evaluation to
# continue with the next case. Body effects cannot reach later
# cases because a selected body ends the match.
if case.guard is not None:
for name in dict.fromkeys(guard_state):
if name not in self._except_var_stack:
self._except_var_stack.append(name)
# Successful pattern bindings and nested matches remain visible after
# the match. Do not erase prior sys interpretations here: a case may
# not match, so both meanings must be considered possible.
for name in dict.fromkeys(aliases_after_match):
if name not in self._except_var_stack:
self._except_var_stack.append(name)
def _push_name_scope(self, shadowed_names=(), kind: str = "block") -> None:
self._sys_name_scopes.append(set())
self._shadowed_name_scopes.append(set(shadowed_names))
self._name_scope_kinds.append(kind)
def _pop_name_scope(self) -> None:
self._sys_name_scopes.pop()
self._shadowed_name_scopes.pop()
self._name_scope_kinds.pop()
def _name_refers_to_sys(self, name: str) -> bool:
nearest_function_scope = max(
(
idx
for idx, kind in enumerate(self._name_scope_kinds)
if kind == "function"
),
default=-1,
)
for idx in range(len(self._sys_name_scopes) - 1, -1, -1):
if (
self._name_scope_kinds[idx] == "class"
and nearest_function_scope > idx
):
continue
sys_names = self._sys_name_scopes[idx]
shadowed_names = self._shadowed_name_scopes[idx]
if name in shadowed_names:
return False
if name in sys_names:
return True
return False
def _mark_name_shadow(self, name: str) -> None:
self._sys_name_scopes[-1].discard(name)
self._shadowed_name_scopes[-1].add(name)
def _mark_target_shadow(self, target: ast.AST) -> None:
if isinstance(target, ast.Name):
self._mark_name_shadow(target.id)
elif isinstance(target, (ast.Tuple, ast.List)):
for elt in target.elts:
self._mark_target_shadow(elt)
elif isinstance(target, ast.Starred):
self._mark_target_shadow(target.value)
@staticmethod
def _is_literal_dynamic_import(node: ast.AST, module: str) -> bool:
if not isinstance(node, ast.Call) or not node.args:
return False
func = node.func
is_dynamic_import = (
(isinstance(func, ast.Attribute) and func.attr == "import_module")
or (isinstance(func, ast.Name) and func.id == "import_module")
or (isinstance(func, ast.Name) and func.id == "__import__")
)
return (
is_dynamic_import
and isinstance(node.args[0], ast.Constant)
and node.args[0].value == module
)
def _expr_is_sys_module(self, node: ast.AST) -> bool:
return (
isinstance(node, ast.Name) and self._name_refers_to_sys(node.id)
) or self._is_literal_dynamic_import(node, "sys")
# ------------------------------------------------------------------
# Wrapper-import enforcement (SECURE_LOGGING_DIRS only, #4183 step 2)
# ------------------------------------------------------------------
def visit_Import(self, node: ast.Import) -> None:
"""Ban raw-logger import routes in secure-logging dirs."""
if self.in_secure_dir:
for alias in node.names:
root = alias.name.split(".")[0]
leaf = alias.name.split(".")[-1]
bound = alias.asname or root
if alias.name == "sys" or (
root == "sys" and alias.asname is None
):
# `import sys`, `import sys as s`, and submodule forms
# like `import sys.monitoring` (which bind the root
# name `sys`) all make the bound name refer to sys.
self._sys_name_scopes[-1].add(bound)
self._shadowed_name_scopes[-1].discard(bound)
else:
self._mark_name_shadow(bound)
if root == "loguru":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"raw loguru import is banned in secure-logging dirs — "
f"{self.wrapper_import_hint}"
)
elif leaf == "secure_logging":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing the secure_logging module exposes raw "
f"handles (secure_logging._loguru_logger) — "
f"{self.wrapper_import_hint}"
)
elif leaf == "log_utils":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing utilities.log_utils exposes the raw "
f"loguru logger (log_utils.logger) — "
f"{self.wrapper_import_hint}"
)
elif root == "local_deep_research" and alias.asname is None:
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"absolute 'import local_deep_research...' binds the "
f"root package, whose .logger attribute is the raw "
f"loguru logger — use relative imports; "
f"{self.wrapper_import_hint}"
)
elif root == "traceback":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"'import traceback' is banned in secure-logging dirs — "
f"tracebacks must never be interpolated into log "
f"messages; use logger.exception() (diagnose-gated) "
f"with a scrubbed safe_msg"
)
if bound == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"import binds the local name 'logger' — only the "
f"secure_logging wrapper may bind 'logger' here; "
f"{self.wrapper_import_hint}"
)
self.generic_visit(node)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
"""Allow only the exact secure_logging wrapper import of 'logger'."""
if self.in_secure_dir:
self._check_import_from_secure_dir(node)
self.generic_visit(node)
def _check_import_from_secure_dir(self, node: ast.ImportFrom) -> None:
module = node.module or ""
is_wrapper_module = (
node.level > 0 and module == WRAPPER_MODULE_RELATIVE
) or (node.level == 0 and module == WRAPPER_MODULE_ABSOLUTE)
for alias in node.names:
if alias.name != "*":
self._mark_name_shadow(alias.asname or alias.name)
if module.split(".")[0] == "loguru":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"raw loguru import is banned in secure-logging dirs — "
f"{self.wrapper_import_hint}"
)
return
if node.level == 0 and module == "traceback":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"'from traceback import ...' is banned in secure-logging "
f"dirs — tracebacks must never be interpolated into log "
f"messages; use logger.exception() (diagnose-gated) with a "
f"scrubbed safe_msg"
)
return
if node.level == 0 and module == "sys":
for alias in node.names:
if alias.name in ("*", "exc_info"):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"'from sys import exc_info' is banned in "
f"secure-logging dirs — tracebacks must never be "
f"interpolated into log messages; use "
f"logger.exception() (diagnose-gated) with a "
f"scrubbed safe_msg"
)
if is_wrapper_module:
for alias in node.names:
if alias.name != "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing '{alias.name}' from secure_logging is "
f"banned — only the unaliased 'logger' proxy may be "
f"imported in secure-logging dirs"
)
elif alias.asname not in (None, "logger"):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"aliasing the secure logger to "
f"'{alias.asname}' is banned — the local name must "
f"remain 'logger'"
)
return
# Module-import routes to raw handles
if (node.level > 0 and module == "utilities.log_utils") or (
node.level == 0
and module == "local_deep_research.utilities.log_utils"
):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing from utilities.log_utils is banned in "
f"secure-logging dirs — it re-exports the raw loguru "
f"logger; {self.wrapper_import_hint}"
)
return
for alias in node.names:
if alias.name == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing 'logger' from a module other than "
f"security.secure_logging is banned in secure-logging "
f"dirs — {self.wrapper_import_hint}"
)
elif alias.name == "secure_logging" and (
(node.level > 0 and module == "security")
or (
node.level == 0 and module == "local_deep_research.security"
)
):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing the secure_logging module exposes raw "
f"handles (secure_logging._loguru_logger) — "
f"{self.wrapper_import_hint}"
)
elif alias.name == "log_utils" and (
(node.level > 0 and module == "utilities")
or (
node.level == 0
and module == "local_deep_research.utilities"
)
):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"importing utilities.log_utils exposes the raw loguru "
f"logger (log_utils.logger) — {self.wrapper_import_hint}"
)
elif alias.asname == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"import binds the local name 'logger' — only the "
f"secure_logging wrapper may bind 'logger' here; "
f"{self.wrapper_import_hint}"
)
# ------------------------------------------------------------------
# Logger rebinding / shadowing bans (SECURE_LOGGING_DIRS only)
# ------------------------------------------------------------------
def _flag_logger_binding(self, target: ast.AST, lineno: int) -> None:
"""Flag any assignment target that binds the name 'logger'."""
if isinstance(target, ast.Name) and target.id == "logger":
self.errors.append(
f"{self.filename}:{lineno}: "
f"rebinding the name 'logger' is banned in secure-logging "
f"dirs — only the secure_logging wrapper import may bind "
f"'logger'"
)
elif isinstance(target, (ast.Tuple, ast.List)):
for elt in target.elts:
self._flag_logger_binding(elt, lineno)
elif isinstance(target, ast.Starred):
self._flag_logger_binding(target.value, lineno)
def _check_logger_derivation(self, value: ast.AST, lineno: int) -> None:
"""Flag assignments that derive a new local name from a logger.
Covers ``log = logger``, ``log = logger.bind(...)``,
``x = log_utils.logger``, ``log = pkg.logger.bind(...)``.
"""
expr = value
while (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Attribute)
and expr.func.attr in WRAPPER_CHAIN_METHODS
):
expr = expr.func.value
if (isinstance(expr, ast.Name) and expr.id == "logger") or (
isinstance(expr, ast.Attribute) and expr.attr == "logger"
):
if isinstance(expr, ast.Attribute):
# ``x = pkg.logger`` is already reported by visit_Attribute's
# .logger ban; avoid a second diagnostic for the same line.
return
self.errors.append(
f"{self.filename}:{lineno}: "
f"assigning a logger to another name is banned in "
f"secure-logging dirs — aliased loggers evade the "
f"exception-variable checks; call 'logger' directly"
)
def visit_Assign(self, node: ast.Assign) -> None:
if self.in_secure_dir:
for target in node.targets:
self._flag_logger_binding(target, node.lineno)
self._check_logger_derivation(node.value, node.lineno)
for target in node.targets:
self._mark_target_shadow(target)
self.generic_visit(node)
def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
if self.in_secure_dir:
self._flag_logger_binding(node.target, node.lineno)
if node.value is not None:
self._check_logger_derivation(node.value, node.lineno)
self._mark_target_shadow(node.target)
self.generic_visit(node)
def visit_AugAssign(self, node: ast.AugAssign) -> None:
if self.in_secure_dir:
self._flag_logger_binding(node.target, node.lineno)
self._mark_target_shadow(node.target)
self.generic_visit(node)
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
if self.in_secure_dir:
self._flag_logger_binding(node.target, node.lineno)
self._check_logger_derivation(node.value, node.lineno)
self._mark_target_shadow(node.target)
if isinstance(node.target, ast.Name):
active = {
name for name in self._except_var_stack if name is not None
}
if self._subject_is_exception(node.value, active):
if node.target.id not in self._except_var_stack:
self._except_var_stack.append(node.target.id)
self.generic_visit(node)
def visit_For(self, node: ast.For) -> None:
if self.in_secure_dir:
self._flag_logger_binding(node.target, node.lineno)
self._mark_target_shadow(node.target)
self.generic_visit(node)
def visit_AsyncFor(self, node: ast.AsyncFor) -> None:
if self.in_secure_dir:
self._flag_logger_binding(node.target, node.lineno)
self._mark_target_shadow(node.target)
self.generic_visit(node)
def visit_With(self, node: ast.With) -> None:
if self.in_secure_dir:
for item in node.items:
if item.optional_vars is not None:
self._flag_logger_binding(item.optional_vars, node.lineno)
self._mark_target_shadow(item.optional_vars)
self.generic_visit(node)
def visit_AsyncWith(self, node: ast.AsyncWith) -> None:
if self.in_secure_dir:
for item in node.items:
if item.optional_vars is not None:
self._flag_logger_binding(item.optional_vars, node.lineno)
self._mark_target_shadow(item.optional_vars)
self.generic_visit(node)
@staticmethod
def _func_param_names(args: ast.arguments) -> List[str]:
all_args = (
list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)
)
if args.vararg:
all_args.append(args.vararg)
if args.kwarg:
all_args.append(args.kwarg)
return [arg.arg for arg in all_args]
def _check_func_params(self, node) -> None:
args = node.args
for arg_name in self._func_param_names(args):
if arg_name == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"parameter named 'logger' shadows the secure logger in "
f"secure-logging dirs — rename the parameter"
)
def _visit_function(self, node) -> None:
if self.in_secure_dir:
self._check_func_params(node)
# Defaults, annotations and decorators are evaluated outside the
# function body, before its parameters exist.
self.visit(node.args)
for decorator in getattr(node, "decorator_list", ()):
self.visit(decorator)
returns = getattr(node, "returns", None)
if returns is not None:
self.visit(returns)
for type_param in getattr(node, "type_params", ()):
self.visit(type_param)
self._push_name_scope(
self._func_param_names(node.args), kind="function"
)
self._lex_scopes.append((id(node), "function"))
if isinstance(node, ast.Lambda):
self.visit(node.body)
else:
for statement in node.body:
self.visit(statement)
self._lex_scopes.pop()
self._pop_name_scope()
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
self._visit_function(node)
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
self._visit_function(node)
def visit_Lambda(self, node: ast.Lambda) -> None:
self._visit_function(node)
def visit_ClassDef(self, node: ast.ClassDef) -> None:
for expr in [
*node.decorator_list,
*node.bases,
*node.keywords,
*getattr(node, "type_params", ()),
]:
self.visit(expr)
self._push_name_scope(kind="class")
self._lex_scopes.append((id(node), "class"))
for statement in node.body:
self.visit(statement)
self._lex_scopes.pop()
self._pop_name_scope()
# ------------------------------------------------------------------
# Bypass-route detection (SECURE_LOGGING_DIRS only)
# ------------------------------------------------------------------
def visit_Attribute(self, node: ast.Attribute) -> None:
if self.in_secure_dir:
# Fail closed: opt/catch/patch are banned on ANY receiver unless
# provably not a logger. A receiver is "unknown" (a parameter, an
# attribute not named logger, a dict/list element, a ternary, an
# alias chain, ...) far more easily than it is provably safe, so
# the default must be to block, not to allow. The single narrow,
# justified exception is ``requests.patch``/``httpx.patch`` — see
# SAFE_PATCH_RECEIVER_MODULES. ``opt``/``catch`` have no such
# concrete, nameable false positive and stay fully receiver-blind.
if node.attr == "patch" and self._is_safe_patch_receiver(
node.value
):
pass
elif node.attr == "patch":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f".patch is banned in secure-logging dirs — patchers "
f'mutate record["exception"] and reattach '
f"tracebacks ungated; use logger.bind() for metadata"
f"{self._patch_exemption_note(node.value)}"
)
elif node.attr in ("opt", "catch"):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f".{node.attr} is banned in secure-logging dirs — the "
f"secure_logging wrapper delegates it to raw loguru, "
f"reattaching tracebacks ungated; use "
f"logger.exception() with a scrubbed safe_msg"
)
elif node.attr == "logger":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f".logger attribute access is banned in secure-logging "
f"dirs — it reaches raw loguru re-exports "
f"(log_utils.logger, local_deep_research.logger); "
f"{self.wrapper_import_hint}"
)
elif node.attr in ("_logger", "_loguru_logger", "__getattr__"):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f".{node.attr} accesses a raw loguru handle — banned in "
f"secure-logging dirs"
)
elif node.attr in BANNED_TRACEBACK_ATTRS:
self.errors.append(
f"{self.filename}:{node.lineno}: "
f".{node.attr} produces raw traceback text — banned in "
f"secure-logging dirs; use logger.exception() "
f"(diagnose-gated) with a scrubbed safe_msg"
)
elif node.attr == "exc_info" and self._expr_is_sys_module(
node.value
):
receiver = (
node.value.id if isinstance(node.value, ast.Name) else "sys"
)
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"{receiver}.exc_info is banned in secure-logging dirs — "
f"tracebacks must never be interpolated into log "
f"messages; use logger.exception() (diagnose-gated) "
f"with a scrubbed safe_msg"
)
self.generic_visit(node)
def _check_secure_dir_call(self, node: ast.Call) -> None:
"""Literal getattr/import_module dodges + chained message checks."""
func = node.func
pos = (node.lineno, node.col_offset)
scope_path = tuple(self._lex_scopes)
if isinstance(func, ast.Name):
is_getattr_call = self._name_may(
func.id, "getattr", pos, scope_path
)
elif isinstance(func, ast.Attribute):
is_getattr_call = self._attr_may_be_getattr(
func, pos, scope_path
) or (
func.attr == "getattr"
and isinstance(func.value, ast.Name)
and self._name_may(func.value.id, "builtins", pos, scope_path)
)
else:
is_getattr_call = False
if (
is_getattr_call
and len(node.args) >= 2
and isinstance(node.args[1], ast.Constant)
and isinstance(node.args[1].value, str)
):
attr = node.args[1].value
# Fail closed, mirroring visit_Attribute: getattr(x, "patch") is
# banned on every receiver in practice — passing a bare
# requests/httpx name to getattr is a value use, which revokes
# the exemption (SAFE_PATCH_RECEIVER_MODULES, condition 6).
# getattr(x, "opt"/"catch") stay receiver-blind — no concrete
# false positive was ever named for them.
if attr in BANNED_LOGGER_ATTRS and not (
attr == "patch" and self._is_safe_patch_receiver(node.args[0])
):
if attr == "patch":
reason = (
'patchers can mutate record["exception"] and '
"reattach tracebacks ungated"
) + self._patch_exemption_note(node.args[0])
else:
reason = "it reaches raw loguru handles"
self.errors.append(
f"{self.filename}:{node.lineno}: "
f'getattr(..., "{attr}") is banned in secure-logging '
f"dirs — {reason}"
)
elif attr in BANNED_TRACEBACK_ATTRS:
self.errors.append(
f"{self.filename}:{node.lineno}: "
f'getattr(..., "{attr}") produces raw traceback text — '
f"banned in secure-logging dirs"
)
elif attr == "exc_info" and self._expr_is_sys_module(node.args[0]):
receiver = (
node.args[0].id
if isinstance(node.args[0], ast.Name)
else "sys"
)
self.errors.append(
f"{self.filename}:{node.lineno}: "
f'getattr({receiver}, "exc_info") is banned in secure-logging '
f"dirs — tracebacks must never be interpolated into "
f"log messages"
)
# importlib.import_module("loguru"/"traceback"), __import__(...)
dynamic_import_module = (
node.args[0].value
if node.args and isinstance(node.args[0], ast.Constant)
else None
)
if dynamic_import_module in BANNED_DYNAMIC_IMPORTS and (
self._is_literal_dynamic_import(node, dynamic_import_module)
):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f'dynamic import of "{dynamic_import_module}" is banned in '
f"secure-logging dirs — {self.wrapper_import_hint}"
)
# logger.bind(...) chains are wrapper-preserving, but the message is
# production-visible — run the exception-variable check.
if (
isinstance(func, ast.Attribute)
and func.attr in LOGGER_LEVEL_METHODS
and self._is_wrapper_chain(func.value)
):
self._check_exception_var_in_log(node)
self._check_search_query_in_log(node)
def _is_wrapper_chain(self, expr: ast.AST) -> bool:
"""True for ``bind()`` chains rooted at a logger receiver.
Returns False for non-``Call`` receivers (e.g. bare ``logger``) so
that direct ``logger.error(...)`` calls are not double-checked —
``_is_logger_call`` already routes them through the message checks.
"""
if not isinstance(expr, ast.Call):
return False
seen_bind = False
while (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Attribute)
and expr.func.attr in WRAPPER_CHAIN_METHODS
):
seen_bind = True
expr = expr.func.value
if not seen_bind:
return False
if isinstance(expr, ast.Name):
return expr.id == "logger"
if isinstance(expr, ast.Attribute):
return expr.attr == "logger"
return False
def visit_Call(self, node: ast.Call) -> None:
"""Check function calls for logging sensitive data."""
# Check if it's a logger call
if self._is_logger_call(node):
self._check_sensitive_logging(node)
self._check_exc_info_on_warning(node)
self._check_exception_var_in_log(node)
self._check_search_query_in_log(node)
if self.in_secure_dir:
self._check_secure_dir_call(node)
self.generic_visit(node)
def _is_logger_call(self, node: ast.Call) -> bool:
"""Check if the call is to a logger method."""
if isinstance(node.func, ast.Attribute):
# Check for logger.info, logger.debug, etc.
attr_name = node.func.attr
if attr_name in LOGGER_LEVEL_METHODS:
if isinstance(node.func.value, ast.Name):
return node.func.value.id == "logger"
if isinstance(node.func.value, ast.Attribute):
# Handle self.logger, cls.logger, etc.
return node.func.value.attr == "logger"
return False
def _check_sensitive_logging(self, node: ast.Call) -> None:
"""Check if sensitive data is being logged."""
for arg in node.args:
self._check_expression_for_sensitive_data(arg, node.lineno)
for keyword in node.keywords:
self._check_expression_for_sensitive_data(
keyword.value, node.lineno
)
def _get_log_level(self, node: ast.Call) -> Optional[str]:
"""Return the logging level name (e.g. 'warning', 'debug') or None."""
if isinstance(node.func, ast.Attribute):
return node.func.attr
return None
def _check_exc_info_on_warning(self, node: ast.Call) -> None:
"""Flag exc_info=True on warning/error/critical level logs.
exc_info dumps full tracebacks which are visible in production.
Use logger.exception() instead: its traceback still reaches the
configured sinks (see _hook_common.py) — it is not dev-only — or
use logger.debug(..., exc_info=True), which is off in production.
"""
level = self._get_log_level(node)
if level not in {"warning", "error", "critical"}:
return
for kw in node.keywords:
if (
kw.arg == "exc_info"
and isinstance(kw.value, ast.Constant)
and kw.value.value is True
):
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"exc_info=True on logger.{level}() exposes tracebacks in production — "
f"use logger.exception() or logger.debug(..., exc_info=True) instead"
)
@staticmethod
def _is_search_query_name(expr: ast.AST) -> bool:
"""True for the raw search query variable or a rewritten variant.
Matched by exact name, not substring: ``query``, ``optimized_query``,
``simplified_query`` and friends carry the user's text, while
``query_url``, ``query_params`` or ``query_count`` do not.
"""
return isinstance(expr, ast.Name) and (
expr.id == "query" or expr.id.endswith("_query")
)
@classmethod
def _reproduced_query_name(cls, expr: ast.AST) -> Optional[str]:
"""Name of the query this expression reproduces the text of, if any.
A bare ``query`` reproduces all of it; ``query[:100]`` reproduces the
first hundred characters, which is still the text the user typed.
Genuinely non-reproducing derivations stay out: ``len(query)`` and
``query.split()`` are Calls, not Subscripts, and ``query_counts[...]``
subscripts a name that is not a query name to begin with.
"""
if cls._is_search_query_name(expr):
return expr.id # type: ignore[attr-defined]
if isinstance(expr, ast.Subscript) and cls._is_search_query_name(
expr.value
):
return expr.value.id # type: ignore[attr-defined]
return None
def _interpolated_query_names(self, expr: ast.AST) -> List[str]:
"""Query names whose *value* this argument puts into the message.
Interpolating the name itself counts, and so does interpolating a
slice of it. ``len(query)`` and ``query.split()`` are not flagged --
the first does not reproduce the text at all, the second is a Call
the matcher does not look inside.
"""
found = []
name = self._reproduced_query_name(expr)
if name:
found.append(name)
for sub in ast.walk(expr):
if isinstance(sub, ast.JoinedStr):
for value in sub.values:
if isinstance(value, ast.FormattedValue):
name = self._reproduced_query_name(value.value)
if name:
found.append(name)
elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Mod):
# "... %s" % query / "... %s %s" % (query, other)
operands = (
sub.right.elts
if isinstance(sub.right, ast.Tuple)
else [sub.right]
)
found.extend(
name
for name in map(self._reproduced_query_name, operands)
if name
)
elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Add):
# "... " + query (string concatenation)
found.extend(
name
for name in map(
self._reproduced_query_name, (sub.left, sub.right)
)
if name
)
elif (
isinstance(sub, ast.Call)
and isinstance(sub.func, ast.Attribute)
and sub.func.attr == "format"
):
# "... {}".format(query) / .format(q=query)
found.extend(
name
for name in map(self._reproduced_query_name, sub.args)
if name
)
found.extend(
name
for name in map(
self._reproduced_query_name,
[k.value for k in sub.keywords],
)
if name
)
return found
def _bound_query_names(self, node: ast.Call) -> List[str]:
"""Query names attached via a ``logger.bind(...)`` chain.
``logger.bind(query=query).warning("empty")`` never puts the query in
the message, but loguru copies bind kwargs into ``record["extra"]``,
which structured sinks serialise — so it is a live sink and the check
has to look past the outer call it is invoked on.
"""
found: List[str] = []
expr = node.func
expr = expr.value if isinstance(expr, ast.Attribute) else None
while (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Attribute)
and expr.func.attr in WRAPPER_CHAIN_METHODS
):
for arg in expr.args:
found.extend(self._interpolated_query_names(arg))
for keyword in expr.keywords:
found.extend(self._interpolated_query_names(keyword.value))
expr = expr.func.value
return found
def _query_log_suppressed(self, node: ast.Call) -> bool:
"""True if the call carries a justified suppression comment.
The marker counts only on the physical line the call's method name
sits on. Accepting it anywhere in the call's span let a comment
written about an unrelated argument exempt the message: a reason
given for a ``logger.bind(engine=engine, # ...)`` keyword silenced
the ``.warning(f"... {query}")`` several lines below it, and the
package has eleven ``logger.bind(policy_audit=True).warning(`` sites
of exactly that shape.
"""
func = node.func
lineno = node.lineno
if isinstance(func, ast.Attribute):
lineno = getattr(func, "end_lineno", None) or node.lineno
return lineno in self.query_log_suppression_lines
def _check_search_query_in_log(self, node: ast.Call) -> None:
"""Flag search queries logged on the empty-search path (#5646).
See the SEARCH_ENGINE_DIRS block above for why the scope is the whole
``web_search_engines/`` package at the shared log levels, rather than the
preview-path functions at the production-visible levels only.
"""
if not self.in_search_engine_dir:
return
level = self._get_log_level(node)
names = []
for arg in node.args:
names.extend(self._interpolated_query_names(arg))
for keyword in node.keywords:
names.extend(self._interpolated_query_names(keyword.value))
names.extend(self._bound_query_names(node))
if not names:
return
if self._query_log_suppressed(node):
return
for name in sorted(set(names)):
self.errors.append(
f"{self.filename}:{node.lineno}: search query '{name}' "
f"interpolated into logger.{level}() under "
f"web_search_engines/ "
f"- BaseSearchEngine.run() no longer logs the query, so this "
f"puts it straight back into the log. Log the engine name and "
f"a result count instead; the query itself is still persisted "
f"per search by SearchTracker.record_search. If the query "
f"really is required here, open a comment on the "
f"logger.{level}() line with "
f"'# {QUERY_LOG_SUPPRESSION} <reason>' and a reason of at "
f"least {QUERY_LOG_REASON_MIN_CHARS} non-space characters"
)
def _check_exception_var_in_log(self, node: ast.Call) -> None:
"""Flag logging calls that interpolate the exception variable.
Patterns like logger.warning(f"...{e}") or logger.warning("...%s", e)
leak error details. Prefer logger.exception() — its output still
reaches the configured sinks, it is not dev-only (see
_hook_common.py); see below for why plain logger.exception() calls
are exempted here outside the secure-logging dirs.
"""
level = self._get_log_level(node)
# logger.debug() is fine — not visible in production
if level == "debug":
return
# Outside the secure_logging dirs, plain logger.exception() always
# attaches the active exception to the record (loguru's default,
# not dev-only — see _hook_common.py); the traceback already
# carries the exception's text on every configured sink, so
# interpolating the same variable into the message adds nothing
# and this check skips it below. Inside the secure_logging dirs,
# the wrapper attaches that exception only in diagnose mode
# (security/secure_logging.py) — by default nothing is attached,
# so the message is the one channel that can leak it there.
if level == "exception" and not self.in_secure_dir:
return
# Collect current except variable names
except_vars = {v for v in self._except_var_stack if v is not None}
if not except_vars:
return
# Check positional args (f-strings and %-style args)
for arg in node.args:
if self._expr_references_vars(arg, except_vars):
if level == "exception":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"Exception variable in logger.exception() — this message is "
f"production-visible at ERROR (secure_logging wrapper) — "
f"log a scrubbed message (scrub_error(); engines: self._scrub_error()) instead"
)
else:
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"Exception variable in logger.{level}() leaks error details — "
f"use logger.exception() or remove the variable"
)
return
# Check keyword args: loguru-style logger.exception("msg: {err}", err=e) bypasses
# the positional check above since the exception variable is in a keyword value.
for kw in node.keywords:
if self._expr_references_vars(kw.value, except_vars):
if level == "exception":
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"Exception variable in logger.exception() keyword arg — this message is "
f"production-visible at ERROR (secure_logging wrapper) — "
f"log a scrubbed message (scrub_error(); engines: self._scrub_error()) instead"
)
else:
self.errors.append(
f"{self.filename}:{node.lineno}: "
f"Exception variable in logger.{level}() keyword arg leaks error details — "
f"use logger.exception() or remove the variable"
)
return
def _expr_references_vars(self, expr: ast.AST, var_names: set) -> bool:
"""Check if an expression references any of the given variable names."""
if isinstance(expr, ast.Name):
return expr.id in var_names
if isinstance(expr, ast.JoinedStr):
# f-string: check each formatted value
return any(
isinstance(v, ast.FormattedValue)
and self._expr_references_vars(v.value, var_names)
for v in expr.values
)
if isinstance(expr, ast.Call):
# str(e), repr(e), "...".format(x=e), str(e).upper(), etc. —
# the exception can hide in the receiver (func) and in keywords,
# not just positional args.
return (
self._expr_references_vars(expr.func, var_names)
or any(
self._expr_references_vars(a, var_names) for a in expr.args
)
or any(
self._expr_references_vars(kw.value, var_names)
for kw in expr.keywords
)
)
if isinstance(expr, ast.Attribute):
# e.args, e.strerror, e.response.text, e.__cause__ carry the
# same error detail as str(e). Allowed: the exception *class
# name* — type(e).__name__ / e.__class__.__name__ — which is
# the sanctioned pattern for scrubbed log lines.
if expr.attr == "__name__" and self._is_exc_class_ref(
expr.value, var_names
):
return False
return self._expr_references_vars(expr.value, var_names)
if isinstance(expr, ast.Subscript):
# e.args[0], e["detail"]
return self._expr_references_vars(
expr.value, var_names
) or self._expr_references_vars(expr.slice, var_names)
if isinstance(expr, ast.BinOp):
# "message: %s" % e
return self._expr_references_vars(
expr.right, var_names
) or self._expr_references_vars(expr.left, var_names)
if isinstance(expr, ast.Tuple):
# ("message %s %s", e, other)
return any(
self._expr_references_vars(el, var_names) for el in expr.elts
)
return False
def _is_exc_class_ref(self, expr: ast.AST, var_names: set) -> bool:
"""True for type(e) or e.__class__ over an exception variable."""
if (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Name)
and expr.func.id == "type"
and len(expr.args) == 1
and not expr.keywords
):
return self._expr_references_vars(expr.args[0], var_names)
if isinstance(expr, ast.Attribute) and expr.attr == "__class__":
return self._expr_references_vars(expr.value, var_names)
return False
def _is_allowed_logging(self, variable_name: str) -> bool:
"""Check if this specific logging is allowed."""
for allowed in ALLOWED_LOGGING:
if (
allowed["file_pattern"] in self.filename
and allowed["variable"] == variable_name
):
return True
return False
def _check_expression_for_sensitive_data(
self, expr: ast.AST, lineno: int
) -> None:
"""Recursively check an expression for sensitive data."""
if isinstance(expr, ast.Name):
# Skip if it's a known false positive
if expr.id in FALSE_POSITIVE_VARS:
return
# Skip if it's specifically allowed for this file
if self._is_allowed_logging(expr.id):
return
# Direct variable logging
if any(
sensitive in expr.id.lower() for sensitive in SENSITIVE_VARS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive variable '{expr.id}'"
)
# Check for sensitive dictionaries being logged directly
if any(
sensitive in expr.id.lower()
for sensitive in SENSITIVE_DICT_KEYS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive dictionary '{expr.id}'"
)
elif isinstance(expr, ast.Attribute):
# Check for Flask request object attributes
if isinstance(expr.value, ast.Name) and expr.value.id == "request":
if expr.attr in {
"form",
"json",
"data",
"headers",
"authorization",
"cookies",
}:
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive Flask request.{expr.attr} which may contain passwords or tokens"
)
return
# Check for Flask g object attributes
if isinstance(expr.value, ast.Name) and expr.value.id == "g":
if (
"password" in expr.attr.lower()
or "token" in expr.attr.lower()
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive Flask g.{expr.attr}"
)
return
# Check for os.environ
if (
isinstance(expr.value, ast.Name)
and expr.value.id == "os"
and expr.attr == "environ"
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of os.environ which contains environment variables"
)
return
# Check for dict.keys(), dict.values(), dict.items()
if expr.attr in {"keys", "values", "items"}:
if isinstance(expr.value, ast.Name):
if any(
sensitive in expr.value.id.lower()
for sensitive in SENSITIVE_DICT_KEYS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive dictionary keys/values from '{expr.value.id}.{expr.attr}()'"
)
# Check for sensitive attributes like obj.password, obj.api_key, etc.
if any(
sensitive in expr.attr.lower() for sensitive in SENSITIVE_VARS
):
# Skip false positives
if expr.attr in FALSE_POSITIVE_VARS:
return
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive attribute '{expr.attr}'"
)
elif isinstance(expr, ast.Call):
# Check for list(), str(), repr() of sensitive dicts
if isinstance(expr.func, ast.Name):
if expr.func.id in {"list", "str", "repr"}:
for arg in expr.args:
# Check for list(dict.keys()), list(dict.values()), etc.
if isinstance(arg, ast.Call) and isinstance(
arg.func, ast.Attribute
):
if arg.func.attr in {"keys", "values", "items"}:
if isinstance(arg.func.value, ast.Name):
if any(
sensitive in arg.func.value.id.lower()
for sensitive in SENSITIVE_DICT_KEYS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive dictionary keys/values via '{expr.func.id}({arg.func.value.id}.{arg.func.attr}())'"
)
# Also check direct conversion of sensitive dicts
elif isinstance(arg, ast.Name):
if any(
sensitive in arg.id.lower()
for sensitive in SENSITIVE_DICT_KEYS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of sensitive dictionary via '{expr.func.id}({arg.id})'"
)
elif isinstance(expr, ast.JoinedStr):
# Check f-strings
for value in expr.values:
if isinstance(value, ast.FormattedValue):
self._check_expression_for_sensitive_data(
value.value, lineno
)
elif isinstance(expr, ast.Dict):
# Check dictionary literals for sensitive keys
for key in expr.keys:
if isinstance(key, ast.Constant) and isinstance(key.value, str):
if any(
sensitive in key.value.lower()
for sensitive in SENSITIVE_VARS
):
self.errors.append(
f"{self.filename}:{lineno}: Potential logging of dictionary with sensitive key '{key.value}'"
)
elif isinstance(expr, ast.BinOp):
# Check string formatting
self._check_expression_for_sensitive_data(expr.left, lineno)
self._check_expression_for_sensitive_data(expr.right, lineno)
def check_file(filepath: Path) -> List[str]:
"""Check a single Python file for sensitive logging."""
try:
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
tree = ast.parse(content, filename=str(filepath))
checker = SensitiveLoggingChecker(
str(filepath), source_lines=content.split("\n")
)
checker.visit(tree)
return checker.errors
except SyntaxError as e:
return [f"{filepath}:{e.lineno}: Syntax error - {e.msg}"]
except Exception as e:
return [f"{filepath}: Error checking file - {e}"]
def main():
"""Main entry point for the pre-commit hook."""
files = sys.argv[1:]
all_errors = []
for filename in files:
filepath = Path(filename)
if filepath.suffix == ".py":
errors = check_file(filepath)
all_errors.extend(errors)
if all_errors:
print("Sensitive data logging detected:")
for error in all_errors:
print(f" {error}")
print("\nPlease ensure sensitive data is not logged directly.")
print("Consider:")
print(
" - In llm/providers/, embeddings/providers/, web_search_engines/:"
)
print(
" use scrub_error() from security.log_sanitizer (engines: "
"self._scrub_error()) and log the safe_msg"
)
print(
" - Elsewhere: use logger.exception() (its traceback still "
"reaches the configured sinks — not dev-only) or remove the "
"variable"
)
print(
" - Using sanitized versions of sensitive data structures before logging"
)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())