~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Codebase-wide sweep for the seven defect patterns already proven live.
Each pattern below was found by hand in a specific file during review of the
Flask -> FastAPI port. This module turns each one into an AST scanner and runs
it over *every* module under ``src/local_deep_research``, so a pattern that was
found once is found everywhere it occurs.
Every scanner carries four guards, because a scanner that quietly matches
nothing is worse than no scanner at all:
``test_p<N>_positive_control``
A synthetic source containing the defect. The scanner MUST flag it.
``test_p<N>_negative_control``
A synthetic source containing the *repaired* shape. The scanner must NOT
flag it. Where the repair is subtle, the negative control is the real
production file with the real fix applied (see the hardlink-copy
verification recorded in ``MUTATION_CONTROL_NOTES``).
``test_p<N>_examines_a_floor_of_sites``
The scanner must have actually looked at a minimum number of candidate
sites. A scanner whose selector silently stops matching (a renamed
decorator, a moved helper) fails here rather than reporting "all clear".
``test_p<N>_known_live_instance_is_refound``
The site that motivated the pattern must appear in the results. If it does
not, the scanner is wrong -- not the codebase. Once repaired, that site
instead has a regression assertion requiring it to remain absent; the
synthetic positive control continues to prove the scanner detects the bug.
``test_p<N>_inventory_is_pinned`` then locks the full result set, so a new
occurrence of an already-proven defect fails the build instead of shipping.
Findings are pinned by (path, tag) rather than (path, line) so that unrelated
edits above a defect do not turn this file into churn.
Nothing here imports the application. It is pure ``ast`` over source text: no
database, no event loop, no settings bootstrap.
"""
# allow: no-sut-import - a guardian sweep. It reads every module under
# src/local_deep_research as SOURCE TEXT and asserts properties of that
# text; importing the package would boot settings and the database for no
# benefit and would make the sweep unable to see modules that fail to
# import.
from __future__ import annotations
import ast
import functools
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterator
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src" / "local_deep_research"
MUTATION_CONTROL_NOTES = """
Scanners 1, 4 and 7 were additionally verified against a hardlink copy of
src/ (`cp -al`) in which the known-defective statement was replaced by its
repaired form (unlink + rewrite, so the original was never touched). All
three findings disappeared and no unrelated finding changed, which
establishes that the scanners read the source rather than a hard-coded
answer. The synthetic negative controls below encode the same repairs.
"""
# ---------------------------------------------------------------------------
# Shared AST infrastructure
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class Site:
"""One scanner finding.
``tag`` is the stable identity used for pinning: a symbol name or column
name, never a line number.
"""
path: str
lineno: int
tag: str
detail: str
@property
def key(self) -> tuple[str, str]:
return (self.path, self.tag)
def __str__(self) -> str:
return f"{self.path}:{self.lineno} [{self.tag}] {self.detail}"
Module = tuple[str, ast.Module]
def _parse_tree(root: Path) -> tuple[Module, ...]:
out: list[Module] = []
for path in sorted(root.rglob("*.py")):
try:
text = path.read_text(encoding="utf-8")
except OSError: # pragma: no cover - unreadable file
continue
try:
tree = ast.parse(text, filename=str(path))
except SyntaxError: # pragma: no cover - not our problem here
continue
out.append((path.relative_to(REPO_ROOT).as_posix(), tree))
return tuple(out)
@functools.lru_cache(maxsize=4)
def modules(root: str = str(SRC_ROOT)) -> tuple[Module, ...]:
return _parse_tree(Path(root))
def synthetic(name: str, source: str) -> tuple[Module, ...]:
"""Parse an inline source string as a one-module corpus."""
return ((f"<synthetic:{name}>", ast.parse(source)),)
def attr_chain(node: ast.AST | None) -> str | None:
"""Render a Name/Attribute chain as dotted text; None for anything else."""
parts: list[str] = []
cur = node
while isinstance(cur, ast.Attribute):
parts.append(cur.attr)
cur = cur.value
if not isinstance(cur, ast.Name):
return None
parts.append(cur.id)
return ".".join(reversed(parts))
def call_name(node: ast.Call) -> str | None:
return attr_chain(node.func)
def names_in(node: ast.AST) -> set[str]:
return {n.id for n in ast.walk(node) if isinstance(n, ast.Name)}
def walk_own(fn: ast.AST) -> Iterator[ast.AST]:
"""Walk *fn*'s body without descending into nested function bodies."""
stack = list(ast.iter_child_nodes(fn))
while stack:
node = stack.pop()
yield node
if isinstance(
node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)
):
continue
stack.extend(ast.iter_child_nodes(node))
def functions(tree: ast.AST) -> Iterator[ast.FunctionDef]:
for node in ast.walk(tree):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
yield node
def params_of(fn: ast.AST) -> set[str]:
args = fn.args
out = {a.arg for a in [*args.posonlyargs, *args.args, *args.kwonlyargs]}
if args.vararg:
out.add(args.vararg.arg)
if args.kwarg:
out.add(args.kwarg.arg)
return out
def param_annotations(fn: ast.AST) -> dict[str, set[str]]:
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
return {}
args = fn.args
out: dict[str, set[str]] = {}
for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]:
if arg.annotation is None:
continue
out[arg.arg] = names_in(arg.annotation)
return out
def local_assignments(fn: ast.AST) -> dict[str, ast.AST]:
"""name -> first assigned expression in *fn* (one level, no CFG)."""
out: dict[str, ast.AST] = {}
for node in ast.walk(fn):
if isinstance(node, ast.Assign):
for tgt in node.targets:
if isinstance(tgt, ast.Name):
out.setdefault(tgt.id, node.value)
elif isinstance(node, ast.AnnAssign) and node.value is not None:
if isinstance(node.target, ast.Name):
out.setdefault(node.target.id, node.value)
return out
def resolve(expr: ast.AST, locs: dict[str, ast.AST], depth: int = 3) -> ast.AST:
"""Follow a Name back to its local assignment, a few hops."""
cur = expr
for _ in range(depth):
if isinstance(cur, ast.Name) and cur.id in locs:
nxt = locs[cur.id]
if nxt is cur:
break
cur = nxt
else:
break
return cur
def ordered(sites) -> list[Site]:
return sorted(set(sites), key=lambda s: (s.path, s.lineno, s.tag))
def keys(sites) -> set[tuple[str, str]]:
return {s.key for s in sites}
def report(label: str, sites) -> str:
body = "\n".join(f" {s}" for s in sites) or " (none)"
return f"{label} -- {len(sites)} finding(s):\n{body}"
COLUMN_CALLS = {"Column", "mapped_column", "sa.Column"}
def _column_assignments(cls: ast.ClassDef):
"""Yield (attr_name, Column(...) call) for a model class body."""
for stmt in cls.body:
target = value = None
if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1:
target, value = stmt.targets[0], stmt.value
elif isinstance(stmt, ast.AnnAssign):
target, value = stmt.target, stmt.value
if not isinstance(target, ast.Name) or not isinstance(value, ast.Call):
continue
if call_name(value) not in COLUMN_CALLS:
continue
yield target.id, value, stmt
def _first_type_name(col_call: ast.Call) -> str | None:
head = col_call.args[0] if col_call.args else None
if head is None:
return None
return attr_chain(head) or (
call_name(head) if isinstance(head, ast.Call) else None
)
# ===========================================================================
# PATTERN 1 -- an enum written by VALUE into a name-keyed column
#
# ``Column(Enum(PyEnum))`` persists the member NAME. ``Column(Enum(PyEnum,
# values_callable=...))`` persists the member VALUE. Writing the wrong one of
# the pair produces a row nothing can read back.
# ===========================================================================
ENUM_BASES = {
"Enum",
"enum.Enum",
"IntEnum",
"enum.IntEnum",
"StrEnum",
"enum.StrEnum",
}
# A str/int mixin member IS its value, so binding one to a String/Integer
# column stores the value rather than the "Cls.MEMBER" repr.
MIXIN_BASES = {
"str",
"int",
"IntEnum",
"enum.IntEnum",
"StrEnum",
"enum.StrEnum",
}
SA_ENUM_CALLS = {"Enum", "sa.Enum", "SQLEnum", "sqlalchemy.Enum"}
TEXT_TYPES = {
"String",
"Text",
"Unicode",
"UnicodeText",
"VARCHAR",
"sa.String",
"sa.Text",
}
WRITE_HELPERS = {"filter_by", "update", "values"}
STR_ANNOTATIONS = {"str", "String"}
@dataclass
class EnumDef:
names: set[str] = field(default_factory=set)
values: set[str] = field(default_factory=set)
mixin: bool = False
@dataclass
class EnumColumn:
model: str
attr: str
enum_cls: str
value_keyed: bool
path: str
lineno: int
def _bases(cls: ast.ClassDef) -> list[str]:
return [attr_chain(b) or "" for b in cls.bases]
def collect_enums(mods) -> dict[str, EnumDef]:
out: dict[str, EnumDef] = {}
for _path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
if not any(b in ENUM_BASES for b in _bases(cls)):
continue
info = EnumDef()
for stmt in cls.body:
if not isinstance(stmt, ast.Assign) or len(stmt.targets) != 1:
continue
tgt = stmt.targets[0]
val = stmt.value
if not isinstance(tgt, ast.Name):
continue
if not isinstance(val, ast.Constant):
continue
if not isinstance(val.value, (str, int)):
continue
info.names.add(tgt.id)
info.values.add(str(val.value))
if info.names:
info.mixin = any(b in MIXIN_BASES for b in _bases(cls))
out[cls.name] = info
return out
def collect_enum_columns(mods, enums) -> dict[tuple[str, str], EnumColumn]:
out: dict[tuple[str, str], EnumColumn] = {}
for path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
for attr, col_call, stmt in _column_assignments(cls):
enum_call = None
for sub in ast.walk(col_call):
if isinstance(sub, ast.Call) and (
call_name(sub) in SA_ENUM_CALLS
):
enum_call = sub
break
if enum_call is None or not enum_call.args:
continue
enum_cls = attr_chain(enum_call.args[0])
if enum_cls is None or enum_cls not in enums:
continue
out[(cls.name, attr)] = EnumColumn(
model=cls.name,
attr=attr,
enum_cls=enum_cls,
value_keyed=any(
kw.arg == "values_callable" for kw in enum_call.keywords
),
path=path,
lineno=stmt.lineno,
)
return out
def collect_text_columns(mods) -> set[tuple[str, str]]:
out: set[tuple[str, str]] = set()
for _path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
for attr, col_call, _stmt in _column_assignments(cls):
if _first_type_name(col_call) in TEXT_TYPES:
out.add((cls.name, attr))
return out
def _coerces_through(expr: ast.AST, enum_cls: str) -> bool:
"""``EnumCls(x)`` / ``EnumCls[x]`` -- the repaired shape, not the defect."""
if isinstance(expr, ast.Call) and attr_chain(expr.func) == enum_cls:
return True
if isinstance(expr, ast.Subscript) and attr_chain(expr.value) == enum_cls:
return True
return False
def _uncoerced_nodes(expr: ast.AST, enum_cls: str) -> Iterator[ast.AST]:
stack = [expr]
while stack:
node = stack.pop()
if _coerces_through(node, enum_cls):
continue
yield node
stack.extend(ast.iter_child_nodes(node))
def _classify_enum_write(
expr: ast.AST, col: EnumColumn, ed: EnumDef
) -> str | None:
chain = attr_chain(expr)
if chain is not None:
head = chain.split(".")
if head[0] == col.enum_cls and len(head) >= 2 and head[1] in ed.names:
accessor = head[2] if len(head) > 2 else None
if accessor == "value" and not col.value_keyed:
return (
f"writes {chain} (VALUE) into name-keyed "
f"{col.model}.{col.attr}"
)
if accessor == "name" and col.value_keyed:
return (
f"writes {chain} (NAME) into value-keyed "
f"{col.model}.{col.attr}"
)
return None
if _coerces_through(expr, col.enum_cls):
return None
lits = {
n.value
for n in _uncoerced_nodes(expr, col.enum_cls)
if isinstance(n, ast.Constant) and isinstance(n.value, str)
}
if not lits:
return None
only_values = lits & (ed.values - ed.names)
only_names = lits & (ed.names - ed.values)
if not col.value_keyed and only_values:
return (
f"writes enum VALUE string {sorted(only_values)!r} into "
f"name-keyed {col.model}.{col.attr}; Enum({col.enum_cls}) "
f"persists the NAME, so the row can never be read back"
)
if col.value_keyed and only_names:
return (
f"writes enum NAME string {sorted(only_names)!r} into "
f"value-keyed {col.model}.{col.attr}"
)
return None
def _model_evidence(obj: ast.AST, locs: dict[str, ast.AST]) -> set[str]:
"""Class names that could name the runtime type behind *obj*.
An attribute name on its own is never evidence: half the models in this
schema have a ``status`` column.
"""
if not isinstance(obj, ast.Name):
return set()
src = locs.get(obj.id)
if src is None:
return set()
out = names_in(src)
out |= {a for a in (attr_chain(x) or "" for x in ast.walk(src)) if a}
return out
def scan_pattern_1(mods):
enums = collect_enums(mods)
columns = collect_enum_columns(mods, enums)
text_cols = collect_text_columns(mods)
by_attr: dict[str, list[EnumColumn]] = {}
for col in columns.values():
by_attr.setdefault(col.attr, []).append(col)
sites: list[Site] = []
for path, tree in mods:
scopes: list[ast.AST] = [tree, *functions(tree)]
for scope in scopes:
locs = (
{}
if isinstance(scope, ast.Module)
else local_assignments(scope)
)
anns = param_annotations(scope)
for node in walk_own(scope):
if isinstance(node, ast.Call):
fname = call_name(node) or ""
tail = (
node.func.attr
if isinstance(node.func, ast.Attribute)
else fname
)
# Inverse arm: a bare enum member into a String column
# stores "Cls.MEMBER" unless the enum has a str mixin.
for kw in node.keywords:
if kw.arg is None or (tail, kw.arg) not in text_cols:
continue
chain = attr_chain(resolve(kw.value, locs))
if chain is None:
continue
parts = chain.split(".")
if (
len(parts) == 2
and parts[0] in enums
and parts[1] in enums[parts[0]].names
and not enums[parts[0]].mixin
):
sites.append(
Site(
path,
node.lineno,
f"{tail}.{kw.arg}",
f"{fname}({kw.arg}={chain}) binds a bare "
f"enum member to String column "
f"{tail}.{kw.arg}; stores the repr",
)
)
for kw in node.keywords:
if kw.arg is None or kw.arg not in by_attr:
continue
for col in by_attr[kw.arg]:
if tail != col.model:
if tail not in WRITE_HELPERS:
continue
base = (
node.func.value
if isinstance(node.func, ast.Attribute)
else None
)
if base is None:
continue
if col.model not in _model_evidence(base, locs):
continue
msg = _classify_enum_write(
resolve(kw.value, locs),
col,
enums[col.enum_cls],
)
if msg:
sites.append(
Site(
path,
node.lineno,
f"{col.model}.{col.attr}",
f"{fname}({kw.arg}=...): {msg}",
)
)
if isinstance(node, ast.Assign):
for tgt in node.targets:
if not isinstance(tgt, ast.Attribute):
continue
if tgt.attr not in by_attr:
continue
evidence = _model_evidence(tgt.value, locs)
if isinstance(tgt.value, ast.Name):
evidence |= anns.get(tgt.value.id, set())
for col in by_attr[tgt.attr]:
if col.model not in evidence:
continue
msg = _classify_enum_write(
resolve(node.value, locs),
col,
enums[col.enum_cls],
)
if msg:
sites.append(
Site(
path,
node.lineno,
f"{col.model}.{col.attr}",
f"{attr_chain(tgt)} = ...: {msg}",
)
)
stats = {
"enums": len(enums),
"enum_columns": len(columns),
"name_keyed_columns": sum(
1 for c in columns.values() if not c.value_keyed
),
"text_columns": len(text_cols),
}
return ordered(sites), stats
def scan_pattern_1_filters(mods):
"""Name-keyed enum columns filtered through plainly-typed values.
A ``str``-typed parameter can only ever carry the enum's VALUE, and a
name-keyed column stores its NAME -- so the filter silently matches
nothing. Callers that feed value strings into such a parameter are
reported too, via a small fixpoint over one-hop forwarding.
"""
enums = collect_enums(mods)
columns = collect_enum_columns(mods, enums)
sinks: list[Site] = []
tainted: dict[tuple[str, str], EnumColumn] = {}
examined = 0
for path, tree in mods:
for fn in functions(tree):
anns = param_annotations(fn)
for node in walk_own(fn):
target = operand = None
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "in_"
and node.args
):
target, operand = attr_chain(node.func.value), node.args[0]
elif isinstance(node, ast.Compare) and len(node.ops) == 1:
if isinstance(node.ops[0], (ast.Eq, ast.NotEq)):
target = attr_chain(node.left)
operand = node.comparators[0]
if not target or "." not in target or operand is None:
continue
model, _, attr = target.rpartition(".")
col = columns.get((model, attr))
if col is None or col.value_keyed:
continue
examined += 1
if not isinstance(operand, ast.Name):
continue
if not (anns.get(operand.id, set()) & STR_ANNOTATIONS):
continue
sinks.append(
Site(
path,
node.lineno,
f"{col.model}.{col.attr}<-{fn.name}:{operand.id}",
f"{target} is a name-keyed Enum({col.enum_cls}) but "
f"is filtered by the str-typed parameter "
f"'{operand.id}' of {fn.name}(); value strings never "
f"match the stored NAMEs",
)
)
tainted[(fn.name, operand.id)] = col
callers: list[Site] = []
for _ in range(4):
grew = False
for path, tree in mods:
for scope in [tree, *functions(tree)]:
sname = getattr(scope, "name", "<module>")
scope_params = (
set() if isinstance(scope, ast.Module) else params_of(scope)
)
for node in walk_own(scope):
if not isinstance(node, ast.Call):
continue
callee = call_name(node)
if callee is None:
continue
tail = callee.split(".")[-1]
for kw in node.keywords:
if kw.arg is None:
continue
col = tainted.get((tail, kw.arg))
if col is None:
continue
if (
isinstance(kw.value, ast.Name)
and kw.value.id in scope_params
):
key = (sname, kw.value.id)
if key not in tainted:
tainted[key] = col
grew = True
continue
lits = {
n.value
for n in ast.walk(kw.value)
if isinstance(n, ast.Constant)
and isinstance(n.value, str)
}
ed = enums[col.enum_cls]
bad = lits & (ed.values - ed.names)
if bad:
callers.append(
Site(
path,
node.lineno,
f"{callee}:{kw.arg}",
f"{callee}({kw.arg}={sorted(bad)!r}) "
f"sends enum VALUE strings to the "
f"name-keyed {col.model}.{col.attr} "
f"filter",
)
)
if not grew:
break
return ordered(sinks), ordered(callers), {"comparisons": examined}
P1_POSITIVE = """
import enum
from sqlalchemy import Column, Enum
class Flavour(enum.Enum):
VANILLA = "vanilla"
COCOA = "cocoa"
class Scoop(Base):
__tablename__ = "scoops"
flavour = Column(Enum(Flavour))
def make(data):
chosen = data.get("flavour", "cocoa")
return Scoop(flavour=chosen)
"""
P1_NEGATIVE = """
import enum
from sqlalchemy import Column, Enum
class Flavour(enum.Enum):
VANILLA = "vanilla"
COCOA = "cocoa"
class Scoop(Base):
__tablename__ = "scoops"
flavour = Column(Enum(Flavour))
# A value-keyed sibling: writing the VALUE here is correct.
backup = Column(
Enum(Flavour, values_callable=lambda obj: [e.value for e in obj])
)
def make(data):
chosen = Flavour(data.get("flavour", "cocoa"))
return Scoop(flavour=chosen, backup="cocoa")
def make_member():
return Scoop(flavour=Flavour.COCOA, backup=Flavour.COCOA)
"""
P1_KNOWN = (
"src/local_deep_research/news/core/card_storage.py",
"NewsCard.card_type",
)
P1_EXPECTED = {P1_KNOWN}
P1_FILTER_SINKS_EXPECTED = {
(
"src/local_deep_research/news/core/card_storage.py",
"NewsCard.card_type<-get_recent:card_types",
),
}
P1_FILTER_CALLERS_EXPECTED = {
(
"src/local_deep_research/news/core/storage_manager.py",
"CardFactory.get_recent_cards:card_types",
),
}
def test_p1_positive_control():
sites, _ = scan_pattern_1(synthetic("p1_pos", P1_POSITIVE))
assert keys(sites) == {("<synthetic:p1_pos>", "Scoop.flavour")}, report(
"pattern 1 positive control", sites
)
def test_p1_negative_control():
sites, _ = scan_pattern_1(synthetic("p1_neg", P1_NEGATIVE))
assert sites == [], report("pattern 1 negative control", sites)
def test_p1_examines_a_floor_of_sites():
_, stats = scan_pattern_1(modules())
_, _, fstats = scan_pattern_1_filters(modules())
assert stats["enums"] >= 40, stats
assert stats["enum_columns"] >= 15, stats
assert stats["name_keyed_columns"] >= 8, stats
assert stats["text_columns"] >= 200, stats
assert fstats["comparisons"] >= 3, fstats
def test_p1_known_live_instance_is_refound():
sites, _ = scan_pattern_1(modules())
assert P1_KNOWN in keys(sites), report(
"pattern 1: card_storage.py must still be flagged", sites
)
def test_p1_inventory_is_pinned():
sites, _ = scan_pattern_1(modules())
assert keys(sites) == P1_EXPECTED, report("pattern 1", sites)
def test_p1_filter_inventory_is_pinned():
sinks, callers, _ = scan_pattern_1_filters(modules())
assert keys(sinks) == P1_FILTER_SINKS_EXPECTED, report(
"pattern 1 read-side sinks", sinks
)
assert keys(callers) == P1_FILTER_CALLERS_EXPECTED, report(
"pattern 1 read-side callers", callers
)
# ===========================================================================
# PATTERN 2 -- a counter that allocates a UNIQUE key, decremented elsewhere
#
# ``UPDATE ... SET n = n + 1 RETURNING n`` is a safe sequence generator only
# while nothing ever moves the counter backwards. One decrement re-issues an
# already-used value, and the UNIQUE index then rejects every subsequent
# insert for that parent row -- permanently.
# ===========================================================================
INT_TYPES = {"Integer", "BigInteger", "SmallInteger", "sa.Integer"}
UNIQUE_CONSTRAINT_CALLS = {"UniqueConstraint", "sa.UniqueConstraint"}
@dataclass(frozen=True)
class UniqueIntColumn:
model: str
attr: str
constraint: tuple[str, ...]
path: str
lineno: int
def collect_unique_int_columns(mods) -> dict[tuple[str, str], UniqueIntColumn]:
out: dict[tuple[str, str], UniqueIntColumn] = {}
for path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
ints = {
attr
for attr, col, _stmt in _column_assignments(cls)
if _first_type_name(col) in INT_TYPES
}
if not ints:
continue
for node in ast.walk(cls):
if not isinstance(node, ast.Call):
continue
if call_name(node) not in UNIQUE_CONSTRAINT_CALLS:
continue
cols = tuple(
a.value
for a in node.args
if isinstance(a, ast.Constant) and isinstance(a.value, str)
)
for col in cols:
if col in ints:
out[(cls.name, col)] = UniqueIntColumn(
cls.name, col, cols, path, node.lineno
)
for attr, col_call, stmt in _column_assignments(cls):
if attr not in ints:
continue
if any(
kw.arg == "unique"
and isinstance(kw.value, ast.Constant)
and kw.value.value is True
for kw in col_call.keywords
):
out[(cls.name, attr)] = UniqueIntColumn(
cls.name, attr, (attr,), path, stmt.lineno
)
return out
def collect_counter_links(mods, unique_cols):
"""counter column -> UNIQUE columns its value is written into."""
links: dict[str, set[tuple[str, str]]] = {}
origins: dict[str, list[Site]] = {}
by_attr: dict[str, list[UniqueIntColumn]] = {}
for uc in unique_cols.values():
by_attr.setdefault(uc.attr, []).append(uc)
for path, tree in mods:
for fn in functions(tree):
locs = local_assignments(fn)
derived: dict[str, set[str]] = {}
for name, expr in locs.items():
if not isinstance(expr, ast.Call):
continue
cols = set()
for arg in [*expr.args, *[k.value for k in expr.keywords]]:
chain = attr_chain(arg)
if chain and chain.count(".") == 1:
cols.add(chain)
if cols:
derived[name] = cols
if not derived:
continue
for node in walk_own(fn):
if not isinstance(node, ast.Call):
continue
tail = (call_name(node) or "").split(".")[-1]
for kw in node.keywords:
if kw.arg is None or kw.arg not in by_attr:
continue
if not isinstance(kw.value, ast.Name):
continue
cols = derived.get(kw.value.id)
if not cols:
continue
for uc in by_attr[kw.arg]:
if tail != uc.model:
continue
for counter in cols:
links.setdefault(counter, set()).add(
(uc.model, uc.attr)
)
origins.setdefault(counter, []).append(
Site(
path,
node.lineno,
counter,
f"{counter} allocates "
f"{uc.model}.{uc.attr} "
f"(UNIQUE{list(uc.constraint)})",
)
)
return links, origins
def collect_column_decrements(mods) -> dict[str, list[Site]]:
out: dict[str, list[Site]] = {}
for path, tree in mods:
for node in ast.walk(tree):
if not isinstance(node, ast.BinOp) or not isinstance(
node.op, ast.Sub
):
continue
chain = attr_chain(node.left)
if chain is None or chain.count(".") != 1:
continue
out.setdefault(chain, []).append(
Site(path, node.lineno, chain, f"decrement of {chain}")
)
return out
def scan_pattern_2(mods):
unique_cols = collect_unique_int_columns(mods)
links, origins = collect_counter_links(mods, unique_cols)
decs = collect_column_decrements(mods)
sites: list[Site] = []
for counter, targets in sorted(links.items()):
for hit in decs.get(counter, []):
names = ", ".join(f"{m}.{a}" for m, a in sorted(targets))
where = "; ".join(f"{o.path}:{o.lineno}" for o in origins[counter])
sites.append(
Site(
hit.path,
hit.lineno,
counter,
f"{counter} is moved BACKWARDS here, yet it also "
f"allocates the UNIQUE key {names} (allocated at "
f"{where}); the next allocation re-issues a used value "
f"and every later insert for that row fails",
)
)
stats = {
"unique_int_columns": len(unique_cols),
"counter_links": len(links),
"decremented_columns": len(decs),
}
return ordered(sites), links, stats
P2_POSITIVE = """
from sqlalchemy import Column, Integer, String, UniqueConstraint
class Ledger(Base):
__tablename__ = "ledger"
id = Column(String, primary_key=True)
entry_count = Column(Integer, default=0)
class Entry(Base):
__tablename__ = "entries"
ledger_id = Column(String)
seq = Column(Integer, nullable=False)
__table_args__ = (UniqueConstraint("ledger_id", "seq"),)
def add_entry(db, ledger_id):
seq = bump(db, Ledger.entry_count, Ledger.id == ledger_id)
db.add(Entry(ledger_id=ledger_id, seq=seq))
def undo_entry(db, ledger_id):
db.execute(update(Ledger).values(entry_count=Ledger.entry_count - 1))
"""
P2_NEGATIVE = """
from sqlalchemy import Column, Integer, String, UniqueConstraint
class Ledger(Base):
__tablename__ = "ledger"
id = Column(String, primary_key=True)
entry_count = Column(Integer, default=0)
visible_count = Column(Integer, default=0)
class Entry(Base):
__tablename__ = "entries"
ledger_id = Column(String)
seq = Column(Integer, nullable=False)
__table_args__ = (UniqueConstraint("ledger_id", "seq"),)
def add_entry(db, ledger_id):
seq = bump(db, Ledger.entry_count, Ledger.id == ledger_id)
db.add(Entry(ledger_id=ledger_id, seq=seq))
def undo_entry(db, ledger_id):
# The monotonic allocator is left alone; a separate display counter
# absorbs the decrement.
db.execute(update(Ledger).values(visible_count=Ledger.visible_count - 1))
"""
P2_REPAIRED_COUNTER = "ChatSession.message_count"
P2_EXPECTED: set[tuple[str, str]] = set()
# Same construction, never decremented anywhere -- an in-repo true negative.
P2_SAFE_COUNTER = "ResearchHistory.step_count"
def test_p2_positive_control():
sites, _links, _ = scan_pattern_2(synthetic("p2_pos", P2_POSITIVE))
assert keys(sites) == {("<synthetic:p2_pos>", "Ledger.entry_count")}, (
report("pattern 2 positive control", sites)
)
def test_p2_negative_control():
sites, links, _ = scan_pattern_2(synthetic("p2_neg", P2_NEGATIVE))
# The link must still be discovered -- otherwise the control passes for
# the wrong reason (scanner blind, rather than code safe).
assert "Ledger.entry_count" in links
assert sites == [], report("pattern 2 negative control", sites)
def test_p2_examines_a_floor_of_sites():
_, links, stats = scan_pattern_2(modules())
assert stats["unique_int_columns"] >= 8, stats
assert stats["decremented_columns"] >= 5, stats
# Chat now allocates from the highest retained sequence, leaving the
# monotonic progress counter as the remaining counter-based allocator.
assert len(links) >= 1, links
def test_p2_chat_display_count_no_longer_allocates_a_sequence():
sites, links, _ = scan_pattern_2(modules())
assert P2_REPAIRED_COUNTER not in links, links
assert P2_REPAIRED_COUNTER not in {s.tag for s in sites}, report(
"pattern 2: chat sequence allocation must remain repaired",
sites,
)
def test_p2_safe_counter_is_not_flagged():
"""``step_count`` allocates a UNIQUE key the same way and is never
decremented; if it appears, the decrement detector is over-matching."""
sites, links, _ = scan_pattern_2(modules())
assert P2_SAFE_COUNTER in links, links
assert P2_SAFE_COUNTER not in {s.tag for s in sites}, report(
"pattern 2 in-repo true negative", sites
)
def test_p2_inventory_is_pinned():
sites, _links, _ = scan_pattern_2(modules())
assert keys(sites) == P2_EXPECTED, report("pattern 2", sites)
# ===========================================================================
# PATTERN 3 -- a guard whose predicate cannot see a referencing table
#
# A predicate that answers "is anything else pointing at this row?" is only
# as good as the list of tables it looks in. Every foreign key into the
# guarded table that the predicate does not query is a blind spot.
# ===========================================================================
GUARD_NAME_RE = re.compile(
r"exclusive|orphan|unused|unreferenc|safe_to|can_(delete|remove|rewrite|"
r"purge)|is_only|_sole|no_other|detach",
re.I,
)
GUARD_DOC_RE = re.compile(
r"no OTHER|belongs to .*alone|not referenced|nothing else (points|refers)|"
r"only such|safe to (delete|remove|rewrite)",
re.I,
)
EXISTENCE_METHODS = {"first", "one_or_none", "scalar", "count", "exists"}
@dataclass
class ModelInfo:
cls: str
table: str | None = None
fks: dict[str, str] = field(default_factory=dict)
def collect_models(mods) -> dict[str, ModelInfo]:
out: dict[str, ModelInfo] = {}
for _path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
info = ModelInfo(cls=cls.name)
for stmt in cls.body:
target = value = None
if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1:
target, value = stmt.targets[0], stmt.value
elif isinstance(stmt, ast.AnnAssign):
target, value = stmt.target, stmt.value
if not isinstance(target, ast.Name):
continue
if target.id == "__tablename__" and isinstance(
value, ast.Constant
):
info.table = value.value
continue
if not isinstance(value, ast.Call):
continue
for sub in ast.walk(value):
if (
isinstance(sub, ast.Call)
and call_name(sub) in {"ForeignKey", "sa.ForeignKey"}
and sub.args
and isinstance(sub.args[0], ast.Constant)
and isinstance(sub.args[0].value, str)
):
info.fks[target.id] = sub.args[0].value.split(".")[0]
if info.table:
out[cls.name] = info
return out
def _looks_like_predicate(fn) -> bool:
if fn.returns is not None and attr_chain(fn.returns) == "bool":
return True
for node in walk_own(fn):
if isinstance(node, ast.Return) and isinstance(
node.value, ast.Constant
):
if isinstance(node.value.value, bool):
return True
return False
def _asks_existence(fn) -> bool:
for node in walk_own(fn):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
if node.func.attr in EXISTENCE_METHODS:
return True
return False
def scan_pattern_3(mods):
models = collect_models(mods)
referrers: dict[str, dict[str, str]] = {}
for name, info in models.items():
for attr, target in info.fks.items():
referrers.setdefault(target, {})[name] = attr
sites: list[Site] = []
candidates = 0
for path, tree in mods:
for fn in functions(tree):
doc = ast.get_docstring(fn) or ""
if not (GUARD_NAME_RE.search(fn.name) or GUARD_DOC_RE.search(doc)):
continue
if not _looks_like_predicate(fn) or not _asks_existence(fn):
continue
candidates += 1
queried: set[str] = set()
attrs_used: set[str] = set()
for node in walk_own(fn):
if isinstance(node, ast.Attribute):
attrs_used.add(node.attr)
chain = attr_chain(node)
if chain and chain.count(".") == 1:
queried.add(chain.split(".")[0])
if isinstance(node, ast.Call) and (
(call_name(node) or "").split(".")[-1]
in {"query", "select", "exists"}
):
for arg in node.args:
head = attr_chain(arg)
if head:
queried.add(head.split(".")[0])
guarded: set[str] = set()
for model in queried:
info = models.get(model)
if info is None:
continue
for attr, table in info.fks.items():
if attr in attrs_used:
guarded.add(table)
for table in sorted(guarded):
others = referrers.get(table, {})
blind = {
cls: attr
for cls, attr in others.items()
if cls not in queried and models[cls].table != table
}
if not blind:
continue
seen = sorted(queried & set(others))
sites.append(
Site(
path,
fn.lineno,
f"{fn.name}:{table}",
f"{fn.name}() decides exclusivity over '{table}' by "
f"inspecting {seen}, but these tables also hold a "
f"foreign key into it and are never consulted: "
+ ", ".join(
f"{c}.{a}" for c, a in sorted(blind.items())
),
)
)
stats = {"models": len(models), "guard_candidates": candidates}
return ordered(sites), stats
P3_POSITIVE = '''
from sqlalchemy import Column, ForeignKey, String
class Doc(Base):
__tablename__ = "docs"
id = Column(String, primary_key=True)
class Shelf(Base):
__tablename__ = "shelves"
doc_id = Column(String, ForeignKey("docs.id"))
class Bookmark(Base):
__tablename__ = "bookmarks"
doc_id = Column(String, ForeignKey("docs.id"))
def is_unreferenced(session, doc) -> bool:
"""True when nothing else points at this document."""
other = session.query(Shelf).filter(Shelf.doc_id == doc.id).first()
return other is None
'''
P3_NEGATIVE = '''
from sqlalchemy import Column, ForeignKey, String
class Doc(Base):
__tablename__ = "docs"
id = Column(String, primary_key=True)
class Shelf(Base):
__tablename__ = "shelves"
doc_id = Column(String, ForeignKey("docs.id"))
class Bookmark(Base):
__tablename__ = "bookmarks"
doc_id = Column(String, ForeignKey("docs.id"))
def is_unreferenced(session, doc) -> bool:
"""True when nothing else points at this document."""
shelf = session.query(Shelf).filter(Shelf.doc_id == doc.id).first()
mark = session.query(Bookmark).filter(Bookmark.doc_id == doc.id).first()
return shelf is None and mark is None
'''
P3_KNOWN = (
"src/local_deep_research/research_library/zotero/sync_service.py",
"_exclusive_to_mapping:documents",
)
P3_EXPECTED = {
P3_KNOWN,
(
"src/local_deep_research/research_library/zotero/sync_service.py",
"_exclusive_to_mapping:collections",
),
}
P3_MUST_NAME = "NoteReference.target_document_id"
def test_p3_positive_control():
sites, _ = scan_pattern_3(synthetic("p3_pos", P3_POSITIVE))
assert keys(sites) == {("<synthetic:p3_pos>", "is_unreferenced:docs")}, (
report("pattern 3 positive control", sites)
)
assert "Bookmark.doc_id" in sites[0].detail
def test_p3_negative_control():
sites, _ = scan_pattern_3(synthetic("p3_neg", P3_NEGATIVE))
assert sites == [], report("pattern 3 negative control", sites)
def test_p3_examines_a_floor_of_sites():
_, stats = scan_pattern_3(modules())
assert stats["models"] >= 60, stats
assert stats["guard_candidates"] >= 1, stats
def test_p3_known_live_instance_is_refound():
sites, _ = scan_pattern_3(modules())
found = {s.key: s for s in sites}
assert P3_KNOWN in found, report("pattern 3", sites)
assert P3_MUST_NAME in found[P3_KNOWN].detail, found[P3_KNOWN].detail
def test_p3_inventory_is_pinned():
sites, _ = scan_pattern_3(modules())
assert keys(sites) == P3_EXPECTED, report("pattern 3", sites)
# ===========================================================================
# PATTERN 4 -- ``.first()`` used where a filter should have been applied
#
# ``.first()`` over a predicate that pins no unique key returns an arbitrary
# matching row. Checking the caller's real selector *afterwards* means the one
# row that would have satisfied the request is often not the one examined.
# A filter that does pin a UNIQUE or PRIMARY KEY column is exempt: there the
# post-check is a state guard, not a mis-ordered row selector.
# ===========================================================================
TERMINALS = {"first", "one_or_none", "scalar", "one"}
FILTER_METHODS = {"filter", "filter_by", "where"}
def _fluent_calls(node: ast.AST) -> Iterator[ast.Call]:
cur = node
while isinstance(cur, ast.Call) and isinstance(cur.func, ast.Attribute):
yield cur
cur = cur.func.value
def collect_unique_keys(mods):
singles: dict[str, set[str]] = {}
multis: dict[str, list[tuple[str, ...]]] = {}
for _path, tree in mods:
for cls in ast.walk(tree):
if not isinstance(cls, ast.ClassDef):
continue
single = {
attr
for attr, col, _stmt in _column_assignments(cls)
if any(
kw.arg in {"primary_key", "unique"}
and isinstance(kw.value, ast.Constant)
and kw.value.value is True
for kw in col.keywords
)
}
ucs = [
tuple(
a.value
for a in node.args
if isinstance(a, ast.Constant) and isinstance(a.value, str)
)
for node in ast.walk(cls)
if isinstance(node, ast.Call)
and call_name(node) in UNIQUE_CONSTRAINT_CALLS
]
if single:
singles[cls.name] = single
ucs = [u for u in ucs if u]
if ucs:
multis[cls.name] = ucs
return singles, multis
def _queried_model(calls) -> str | None:
for call in calls:
if call.func.attr in {"query", "select"} and call.args:
chain = attr_chain(call.args[0])
if chain:
return chain.split(".")[0]
return None
def _pinned_columns(calls) -> set[str]:
out: set[str] = set()
for call in calls:
if call.func.attr not in FILTER_METHODS:
continue
for kw in call.keywords:
if kw.arg:
out.add(kw.arg)
for arg in call.args:
for sub in ast.walk(arg):
if isinstance(sub, ast.Compare) and isinstance(
sub.ops[0], ast.Eq
):
chain = attr_chain(sub.left)
if chain and "." in chain:
out.add(chain.rpartition(".")[2])
return out
def _filter_names(calls) -> set[str]:
out: set[str] = set()
for call in calls:
if call.func.attr not in FILTER_METHODS:
continue
for arg in call.args:
out |= names_in(arg)
for kw in call.keywords:
if kw.arg:
out.add(kw.arg)
out |= names_in(kw.value)
return out
def _reads_from(node: ast.AST, var: str) -> bool:
for sub in ast.walk(node):
base = None
if isinstance(sub, (ast.Attribute, ast.Subscript)):
base = sub.value
elif isinstance(sub, ast.Call) and isinstance(sub.func, ast.Attribute):
base = sub.func.value
if isinstance(base, ast.Name) and base.id == var:
return True
chain = attr_chain(base) if base is not None else None
if chain and chain.split(".")[0] == var:
return True
return False
def scan_pattern_4(mods):
singles, multis = collect_unique_keys(mods)
sites: list[Site] = []
examined = 0
unique_pinned = 0
for path, tree in mods:
for fn in functions(tree):
fn_params = params_of(fn) - {"self", "cls"}
if not fn_params:
continue
for node in walk_own(fn):
if not isinstance(node, ast.Assign):
continue
if len(node.targets) != 1:
continue
target = node.targets[0]
value = node.value
if not isinstance(target, ast.Name):
continue
if not isinstance(value, ast.Call):
continue
if not isinstance(value.func, ast.Attribute):
continue
if value.func.attr not in TERMINALS:
continue
calls = list(_fluent_calls(value))
if not any(
c.func.attr in FILTER_METHODS | {"query", "select"}
for c in calls
):
continue
examined += 1
model = _queried_model(calls)
cols = _pinned_columns(calls)
if model is not None:
if cols & singles.get(model, set()):
unique_pinned += 1
continue
if any(set(uc) <= cols for uc in multis.get(model, [])):
unique_pinned += 1
continue
free = fn_params - _filter_names(calls)
if not free:
continue
var = target.id
aliases = {var}
for sub in walk_own(fn):
if (
isinstance(sub, ast.Assign)
and len(sub.targets) == 1
and isinstance(sub.targets[0], ast.Name)
and sub.lineno > node.lineno
and any(_reads_from(sub.value, a) for a in aliases)
):
aliases.add(sub.targets[0].id)
for cmp_node in walk_own(fn):
if not isinstance(cmp_node, ast.Compare):
continue
if cmp_node.lineno <= node.lineno:
continue
if not isinstance(
cmp_node.ops[0], (ast.Eq, ast.NotEq, ast.In, ast.NotIn)
):
continue
left = cmp_node.left
right = cmp_node.comparators[0]
if not any(
_reads_from(side, alias)
for side in (left, right)
for alias in aliases
):
continue
used = (names_in(left) | names_in(right)) & free
if not used:
continue
sites.append(
Site(
path,
node.lineno,
fn.name,
f"{fn.name}(): `{var} = ....{value.func.attr}()` "
f"pins only the non-unique {sorted(cols)} on "
f"{model or '<unknown>'}, while {sorted(used)} -- "
f"the caller's actual selector -- is only tested "
f"afterwards (line {cmp_node.lineno}); the row "
f"returned need not be the row that matches it",
)
)
break
stats = {
"terminal_queries": examined,
"unique_pinned_skipped": unique_pinned,
}
return ordered(sites), stats
P4_POSITIVE = """
from sqlalchemy import Column, Integer, String
class Job(Base):
__tablename__ = "jobs"
id = Column(Integer, primary_key=True)
kind = Column(String)
state = Column(String)
def cancel(session, tenant_id):
job = (
session.query(Job)
.filter(Job.kind == "index", Job.state == "running")
.first()
)
if job and job.tenant == tenant_id:
job.state = "cancelled"
"""
P4_NEGATIVE = """
from sqlalchemy import Column, Integer, String
class Job(Base):
__tablename__ = "jobs"
id = Column(Integer, primary_key=True)
kind = Column(String)
state = Column(String)
token = Column(String, unique=True)
def cancel(session, tenant_id):
job = (
session.query(Job)
.filter(
Job.kind == "index",
Job.state == "running",
Job.tenant == tenant_id,
)
.first()
)
if job:
job.state = "cancelled"
def by_token(session, token, expected_state):
job = session.query(Job).filter(Job.token == token).first()
if job and job.state == expected_state:
return job
return None
"""
P4_REPAIRED = ("src/local_deep_research/web/routers/rag.py", "cancel_indexing")
P4_EXPECTED: set[tuple[str, str]] = set()
def test_p4_positive_control():
sites, _ = scan_pattern_4(synthetic("p4_pos", P4_POSITIVE))
assert keys(sites) == {("<synthetic:p4_pos>", "cancel")}, report(
"pattern 4 positive control", sites
)
def test_p4_negative_control():
sites, _ = scan_pattern_4(synthetic("p4_neg", P4_NEGATIVE))
assert sites == [], report("pattern 4 negative control", sites)
def test_p4_examines_a_floor_of_sites():
_, stats = scan_pattern_4(modules())
assert stats["terminal_queries"] >= 250, stats
assert stats["unique_pinned_skipped"] >= 150, stats
def test_p4_cancel_indexing_no_longer_selects_an_arbitrary_task():
sites, _ = scan_pattern_4(modules())
assert P4_REPAIRED not in keys(sites), report(
"pattern 4: rag.py::cancel_indexing must remain repaired", sites
)
def test_p4_inventory_is_pinned():
sites, _ = scan_pattern_4(modules())
assert keys(sites) == P4_EXPECTED, report("pattern 4", sites)
# ===========================================================================
# PATTERN 5 -- a cap/limit read from a request field with no upper bound
#
# ``max(0, int(x))`` bounds a value from below only. A value that reaches a
# scheduler interval, a SQL OFFSET or an iteration count needs a ceiling as
# well, or the caller picks how much work the server does.
# ===========================================================================
TAINT_ACCESSOR_RE = re.compile(
r"^(request\.(query_params|args|path_params|form|headers|cookies|json)"
r"\.get"
r"|(data|body|payload|json_data|json_body|params|args|form_data|req_data)"
r"\.get)$"
)
COERCERS = {"int", "float", "max", "_parse_int_param", "abs", "round"}
SINK_METHODS = {"limit", "offset", "fetchmany", "seek"}
SINK_FUNCTIONS = {"range", "timedelta", "sleep", "islice"}
SINK_KWARG_RE = re.compile(
r"(^|_)(limit|offset|minutes|seconds|hours|days|interval|delay|timeout|"
r"per_page|page_size|max_results|num_results|iterations|depth|top_k|"
r"chunk_size|batch_size|refresh_minutes|n_results)$",
re.I,
)
CLAMP_FUNCTIONS = {"min"}
CLAMP_NAME_RE = re.compile(r"clamp|bound|cap|coerce_range|_clip", re.I)
UPPER_BOUND_KWARGS = {
"max_val",
"max",
"maximum",
"upper",
"cap",
"max_value",
"limit_max",
"hard_max",
}
def _taint_source(expr: ast.AST) -> str | None:
for sub in ast.walk(expr):
if not isinstance(sub, ast.Call):
continue
name = call_name(sub)
if name and TAINT_ACCESSOR_RE.match(name):
key = (
repr(sub.args[0].value)
if sub.args and isinstance(sub.args[0], ast.Constant)
else ""
)
return f"{name}({key})"
return None
def _looks_numeric(expr: ast.AST) -> bool:
for sub in ast.walk(expr):
if isinstance(sub, ast.Call):
if (call_name(sub) or "").split(".")[-1] in COERCERS:
return True
if isinstance(sub, ast.Constant) and isinstance(sub.value, int):
if not isinstance(sub.value, bool):
return True
return False
def _imposes_ceiling(expr: ast.AST) -> bool:
for sub in ast.walk(expr):
if not isinstance(sub, ast.Call):
continue
name = (call_name(sub) or "").split(".")[-1]
if name in CLAMP_FUNCTIONS or CLAMP_NAME_RE.search(name):
return True
if any(kw.arg in UPPER_BOUND_KWARGS for kw in sub.keywords):
return True
return False
def _ceilinged_names(fn) -> set[str]:
out: set[str] = set()
for node in walk_own(fn):
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
if isinstance(target, ast.Name) and _imposes_ceiling(node.value):
out.add(target.id)
if isinstance(node, ast.Call):
if (call_name(node) or "").split(".")[-1] in CLAMP_FUNCTIONS:
for arg in node.args:
out |= names_in(arg)
if isinstance(node, ast.Compare) and isinstance(node.left, ast.Name):
if isinstance(node.ops[0], (ast.Gt, ast.GtE)):
out.add(node.left.id)
return out
def _lower_bounded_only(expr: ast.AST) -> bool:
for sub in ast.walk(expr):
if isinstance(sub, ast.Call):
if (call_name(sub) or "").split(".")[-1] == "max":
return True
return False
def scan_pattern_5(mods):
sites: list[Site] = []
examined = 0
for path, tree in mods:
for fn in functions(tree):
tainted: dict[str, tuple[ast.AST, str, int]] = {}
for node in walk_own(fn):
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
continue
target = node.targets[0]
if not isinstance(target, ast.Name):
continue
source = _taint_source(node.value)
if source is None:
continue
if not _looks_numeric(node.value) and not SINK_KWARG_RE.search(
target.id
):
continue
tainted[target.id] = (node.value, source, node.lineno)
if not tainted:
continue
examined += len(tainted)
ceilinged = _ceilinged_names(fn)
for name, (expr, source, lineno) in sorted(tainted.items()):
if name in ceilinged:
continue
hits: set[str] = set()
for node in walk_own(fn):
if not isinstance(node, ast.Call):
continue
full = call_name(node) or ""
tail = (
node.func.attr
if isinstance(node.func, ast.Attribute)
else full
)
label = full or f"....{tail}"
arg_names = set()
for arg in node.args:
arg_names |= names_in(arg)
if (
tail in SINK_METHODS or tail in SINK_FUNCTIONS
) and name in arg_names:
hits.add(f"{label}(...) line {node.lineno}")
for kw in node.keywords:
if kw.arg is None or not SINK_KWARG_RE.search(kw.arg):
continue
if (
isinstance(kw.value, ast.Name)
and kw.value.id == name
):
hits.add(f"{label}({kw.arg}=) line {node.lineno}")
if not hits:
continue
shape = (
"bounded below by max() but never above"
if _lower_bounded_only(expr)
else "no bound in either direction"
)
sites.append(
Site(
path,
lineno,
f"{fn.name}:{name}",
f"{fn.name}(): '{name}' comes from {source}; {shape}; "
f"it reaches {sorted(hits)}",
)
)
return ordered(sites), {"tainted_numeric_locals": examined}
P5_POSITIVE = """
def list_things(request, service):
offset = max(0, int(request.query_params.get("offset", 0)))
return service.fetch(offset=offset)
def schedule(request, data):
refresh_minutes = data.get("refresh_minutes")
return make_job(interval=refresh_minutes)
"""
P5_NEGATIVE = """
MAX_OFFSET = 100_000
def list_things(request, service):
offset = max(0, min(int(request.query_params.get("offset", 0)),
MAX_OFFSET))
return service.fetch(offset=offset)
def schedule(request, data):
refresh_minutes = data.get("refresh_minutes")
if refresh_minutes > 1440:
raise ValueError("too long")
return make_job(interval=refresh_minutes)
def paged(request, service):
limit = _parse_int_param(
request.query_params.get("limit"), 20, min_val=1, max_val=100
)
return service.fetch(limit=limit)
"""
P5_KNOWN = {
(
"src/local_deep_research/web/routers/news_flask_api.py",
"create_subscription:refresh_minutes",
),
(
"src/local_deep_research/web/routers/news_flask_api.py",
"create_subscription:search_iterations",
),
}
P5_EXPECTED = P5_KNOWN
P5_REPAIRED_OFFSETS = {
("src/local_deep_research/web/routers/history.py", "get_history:offset"),
("src/local_deep_research/web/routers/research.py", "get_history:offset"),
(
"src/local_deep_research/web/routers/library.py",
"get_documents:offset",
),
}
def test_p5_positive_control():
sites, _ = scan_pattern_5(synthetic("p5_pos", P5_POSITIVE))
assert keys(sites) == {
("<synthetic:p5_pos>", "list_things:offset"),
("<synthetic:p5_pos>", "schedule:refresh_minutes"),
}, report("pattern 5 positive control", sites)
def test_p5_negative_control():
sites, _ = scan_pattern_5(synthetic("p5_neg", P5_NEGATIVE))
assert sites == [], report("pattern 5 negative control", sites)
def test_p5_examines_a_floor_of_sites():
_, stats = scan_pattern_5(modules())
assert stats["tainted_numeric_locals"] >= 30, stats
def test_p5_known_live_instances_are_refound():
sites, _ = scan_pattern_5(modules())
missing = P5_KNOWN - keys(sites)
assert not missing, report(f"pattern 5 lost {missing}", sites)
def test_p5_inventory_is_pinned():
sites, _ = scan_pattern_5(modules())
assert keys(sites) == P5_EXPECTED, report("pattern 5", sites)
def test_p5_repaired_pagination_offsets_stay_absent():
sites, _ = scan_pattern_5(modules())
regressed = P5_REPAIRED_OFFSETS & keys(sites)
assert not regressed, report(
f"pattern 5 repaired offsets returned: {regressed}", sites
)
# ===========================================================================
# PATTERN 6 -- a value interpolated into a header/log/prompt without the
# sanitiser its siblings use
#
# ``sanitize_for_log`` strips control characters and caps length.
# ``scrub_error``/``sanitize_error_message`` do not guarantee control-free
# output. This inventories call sites relying on a later sink backstop.
# The runtime premise for ``scrub_error`` is checked in
# tests/security/test_log_sanitizer.py: inspecting a stripped view for
# redaction analysis does not mean the returned text has been stripped.
# ===========================================================================
READER_CONTROLLED_RE = re.compile(
r"^(request\.(headers|cookies|query_params|path_params|form)\.get"
r"|request\.client\.host"
r"|request\.url(\.path)?"
r"|_get_client_ip"
r"|get_client_ip"
r"|get_remote_address)$"
)
CONTROL_CHAR_STRIPPERS = {
"sanitize_for_log",
"strip_control_chars",
"sanitize_error_for_client",
"sanitize_filename",
"quote",
"urlencode",
"json.dumps",
"dumps",
"repr",
}
CREDENTIAL_ONLY_SCRUBBERS = {
"scrub_error",
"_scrub_error",
"sanitize_error_message",
"redact_secrets",
}
LOG_METHODS = {
"debug",
"info",
"warning",
"error",
"exception",
"critical",
"log",
"success",
"trace",
}
LOGGER_NAMES = {"logger", "log", "logging", "_logger"}
def module_binds_http_request(tree: ast.Module) -> bool:
"""Does this module import ``Request`` from fastapi/starlette?
Without the gate, ``request.url`` also matches
``requests.PreparedRequest.url`` -- an OUTBOUND url, not reader input.
"""
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module:
if node.module.split(".")[0] in {"fastapi", "starlette"}:
if any(alias.name == "Request" for alias in node.names):
return True
return False
def _logger_sink(node: ast.Call) -> str | None:
if not isinstance(node.func, ast.Attribute):
return None
if node.func.attr not in LOG_METHODS:
return None
base = attr_chain(node.func.value) or ""
if base.split(".")[-1] not in LOGGER_NAMES:
return None
return f"{base}.{node.func.attr}"
def _wrapped_by(expr: ast.AST, names: set[str]) -> bool:
for sub in ast.walk(expr):
if isinstance(sub, ast.Call):
if (call_name(sub) or "").split(".")[-1] in names:
return True
return False
def _reader_controlled(expr: ast.AST) -> str | None:
for sub in ast.walk(expr):
if isinstance(sub, ast.Call):
name = call_name(sub)
if name and READER_CONTROLLED_RE.match(name):
key = (
repr(sub.args[0].value)
if sub.args and isinstance(sub.args[0], ast.Constant)
else ""
)
return f"{name}({key})"
chain = attr_chain(sub)
if chain and READER_CONTROLLED_RE.match(chain):
return chain
return None
def _interpolated(fstring: ast.JoinedStr) -> Iterator[ast.AST]:
for part in fstring.values:
if isinstance(part, ast.FormattedValue):
yield part.value
def scan_pattern_6_logs(mods):
"""Reader-controlled data written into a log line unstripped."""
sites: list[Site] = []
examined = 0
for path, tree in mods:
http = module_binds_http_request(tree)
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
sink = _logger_sink(node)
if sink is None:
continue
examined += 1
for arg in node.args:
if not isinstance(arg, ast.JoinedStr):
continue
for expr in _interpolated(arg):
source = _reader_controlled(expr)
if source is None:
continue
if source.startswith("request.") and not http:
continue
if _wrapped_by(expr, CONTROL_CHAR_STRIPPERS):
continue
sites.append(
Site(
path,
node.lineno,
source,
f"{sink}() interpolates {source} with no "
f"sanitize_for_log/strip_control_chars, unlike "
f"its siblings in web/routers/auth.py; the "
f"reader chooses these bytes and the length "
f"[SECURITY: audit-record forging]",
)
)
return ordered(sites), {"logger_calls": examined}
def scan_pattern_6_credential_only(mods):
"""A credential scrubber standing in for a control-character strip."""
sites: list[Site] = []
for path, tree in mods:
for fn in [tree, *functions(tree)]:
locs = {} if isinstance(fn, ast.Module) else local_assignments(fn)
for node in walk_own(fn):
if not isinstance(node, ast.Call):
continue
sink = _logger_sink(node)
if sink is None:
continue
for arg in node.args:
if not isinstance(arg, ast.JoinedStr):
continue
for expr in _interpolated(arg):
origin = (
resolve(expr, locs)
if isinstance(expr, ast.Name)
else expr
)
if not _wrapped_by(origin, CREDENTIAL_ONLY_SCRUBBERS):
continue
if _wrapped_by(origin, CONTROL_CHAR_STRIPPERS):
continue
label = (
expr.id if isinstance(expr, ast.Name) else "<expr>"
)
sites.append(
Site(
path,
node.lineno,
f"{getattr(fn, 'name', '<module>')}:{label}",
f"{sink}() interpolates a credential-scrubbed "
f"value; scrub_error()/"
f"sanitize_error_message() mask credential "
f"shapes but do not guarantee control-free output",
)
)
return ordered(sites)
def scan_pattern_6_headers(mods):
"""Reader-controlled data written into a response header value."""
sites: list[Site] = []
examined = 0
for path, tree in mods:
http = module_binds_http_request(tree)
for node in ast.walk(tree):
if isinstance(node, ast.Call):
for kw in node.keywords:
if kw.arg != "headers":
continue
if not isinstance(kw.value, ast.Dict):
continue
examined += 1
for key, value in zip(kw.value.keys, kw.value.values):
source = _reader_controlled(value)
if source is None:
continue
if source.startswith("request.") and not http:
continue
if _wrapped_by(value, CONTROL_CHAR_STRIPPERS):
continue
name = (
key.value
if isinstance(key, ast.Constant)
else "<dynamic>"
)
sites.append(
Site(
path,
node.lineno,
f"header:{name}",
f"header {name!r} is built from {source} with "
f"no sanitiser [SECURITY: header injection]",
)
)
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
if not isinstance(target, ast.Subscript):
continue
base = attr_chain(target.value) or ""
if not base.endswith("headers"):
continue
examined += 1
source = _reader_controlled(node.value)
if source is None:
continue
if source.startswith("request.") and not http:
continue
if _wrapped_by(node.value, CONTROL_CHAR_STRIPPERS):
continue
sites.append(
Site(
path,
node.lineno,
f"header-assign:{base}",
f"{base}[...] is built from {source} with no "
f"sanitiser [SECURITY: header injection]",
)
)
return ordered(sites), {"header_writes": examined}
P6_POSITIVE = """
from fastapi import Request
from loguru import logger
def audit(request: Request):
logger.warning(
f"denied path={request.url.path} agent={request.headers.get('UA')}"
)
def echo(request: Request):
return Response(headers={"X-Seen": request.headers.get("X-Trace")})
"""
P6_NEGATIVE = """
from fastapi import Request
from loguru import logger
from ...security.log_sanitizer import sanitize_for_log
def audit(request: Request):
logger.warning(
f"denied path={sanitize_for_log(request.url.path)} "
f"agent={sanitize_for_log(request.headers.get('UA'))}"
)
def echo(request: Request):
return Response(
headers={"X-Seen": sanitize_for_log(request.headers.get("X-Trace"))}
)
"""
# The 429 audit line in web/fastapi_app.py was the only live instance; it now
# wraps every field in sanitize_for_log (#5787), so the inventory is empty and
# the positive control above is what keeps the scanner honest.
P6_LOG_EXPECTED: set[tuple[str, str]] = set()
P6_HEADER_EXPECTED: set[tuple[str, str]] = set()
P6_CREDENTIAL_KNOWN = (
"src/local_deep_research/config/llm_config.py",
"_log_llm_error:safe_msg",
)
# Systemic, not a single site: pinned as a floor so the family cannot be
# quietly widened, and asserted to contain the known instance.
P6_CREDENTIAL_MIN = 150
def test_p6_positive_control():
logs, _ = scan_pattern_6_logs(synthetic("p6_pos", P6_POSITIVE))
headers, _ = scan_pattern_6_headers(synthetic("p6_pos", P6_POSITIVE))
assert keys(logs) == {
("<synthetic:p6_pos>", "request.url.path"),
("<synthetic:p6_pos>", "request.headers.get('UA')"),
}, report("pattern 6 log positive control", logs)
assert keys(headers) == {("<synthetic:p6_pos>", "header:X-Seen")}, report(
"pattern 6 header positive control", headers
)
def test_p6_negative_control():
logs, _ = scan_pattern_6_logs(synthetic("p6_neg", P6_NEGATIVE))
headers, _ = scan_pattern_6_headers(synthetic("p6_neg", P6_NEGATIVE))
assert logs == [], report("pattern 6 log negative control", logs)
assert headers == [], report("pattern 6 header negative control", headers)
def test_p6_examines_a_floor_of_sites():
_, log_stats = scan_pattern_6_logs(modules())
_, header_stats = scan_pattern_6_headers(modules())
assert log_stats["logger_calls"] >= 3000, log_stats
assert header_stats["header_writes"] >= 20, header_stats
def test_p6_known_live_instances_are_refound():
cred = scan_pattern_6_credential_only(modules())
assert P6_CREDENTIAL_KNOWN in keys(cred), report(
"pattern 6: _log_llm_error must still be flagged", cred[:20]
)
def test_p6_log_inventory_is_pinned():
logs, _ = scan_pattern_6_logs(modules())
assert keys(logs) == P6_LOG_EXPECTED, report("pattern 6 logs", logs)
def test_p6_header_inventory_is_pinned():
headers, _ = scan_pattern_6_headers(modules())
assert keys(headers) == P6_HEADER_EXPECTED, report(
"pattern 6 headers", headers
)
def test_p6_credential_only_family_is_systemic():
cred = scan_pattern_6_credential_only(modules())
assert len(cred) >= P6_CREDENTIAL_MIN, report(
"pattern 6 credential-only", cred[:20]
)
# ===========================================================================
# PATTERN 7 -- a ``try`` that wraps a ``yield``
#
# In a generator, an exception raised by the CONSUMER surfaces at the
# ``yield``. If the ``yield`` sits inside the ``try``, the setup handler
# catches it -- and a teardown failure is then reported, or swallowed, as a
# setup failure. A handler that re-raises the original exception is exempt:
# that is the idiomatic rollback-then-reraise context manager.
# ===========================================================================
CONTEXTMANAGER_DECORATORS = {
"contextmanager",
"contextlib.contextmanager",
"asynccontextmanager",
"contextlib.asynccontextmanager",
}
def _decorator_names(fn) -> set[str]:
out = set()
for dec in fn.decorator_list:
chain = attr_chain(dec)
if chain is None and isinstance(dec, ast.Call):
chain = attr_chain(dec.func)
if chain:
out.add(chain)
return out
def _yields_directly_in(body) -> list[ast.AST]:
found: list[ast.AST] = []
stack = list(body)
while stack:
node = stack.pop()
if isinstance(node, (ast.Yield, ast.YieldFrom)):
found.append(node)
if isinstance(
node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)
):
continue
stack.extend(ast.iter_child_nodes(node))
return found
def _handler_verdict(handler: ast.ExceptHandler) -> str:
"""``reraise`` / ``converts`` / ``swallows``."""
converts = False
for node in ast.walk(handler):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
if isinstance(node, ast.Raise):
if node.exc is None:
return "reraise"
converts = True
return "converts" if converts else "swallows"
def _handler_label(handler: ast.ExceptHandler) -> str:
if handler.type is None:
return "BareExcept"
chain = attr_chain(handler.type)
if chain:
return chain
return "|".join(
attr_chain(e) or "?" for e in getattr(handler.type, "elts", [])
)
def scan_pattern_7(mods):
sites: list[Site] = []
generators = 0
for path, tree in mods:
for fn in functions(tree):
own = list(walk_own(fn))
if not any(isinstance(n, (ast.Yield, ast.YieldFrom)) for n in own):
continue
generators += 1
decorators = _decorator_names(fn)
for node in own:
if not isinstance(node, ast.Try) or not node.handlers:
continue
yields = _yields_directly_in(node.body)
if not yields:
continue
bad = {}
for handler in node.handlers:
verdict = _handler_verdict(handler)
if verdict != "reraise":
bad[_handler_label(handler)] = verdict
if not bad:
continue
role = (
"contextmanager"
if decorators & CONTEXTMANAGER_DECORATORS
else "generator"
)
lines = sorted({y.lineno for y in yields})
sites.append(
Site(
path,
node.lineno,
fn.name,
f"{fn.name}() [{role}]: the `try:` at line "
f"{node.lineno} encloses `yield` at {lines}; its "
f"handlers {bad} sit on the SETUP path but also "
f"catch whatever the CONSUMER raises back through "
f"the yield, so teardown is handled by the setup "
f"handler",
)
)
return ordered(sites), {"generator_functions": generators}
P7_POSITIVE = """
def get_session_dep(request):
try:
with open_session() as session:
yield session
except SetupError:
request.session.clear()
raise HTTPException(status_code=401, detail="log in again")
def stream():
try:
yield "start"
yield "end"
except Exception:
logger.exception("boom")
"""
P7_NEGATIVE = """
from contextlib import contextmanager
def get_session_dep(request):
try:
session = open_session()
except SetupError:
request.session.clear()
raise HTTPException(status_code=401, detail="log in again")
try:
yield session
finally:
session.close()
@contextmanager
def scoped(session):
# rollback-then-reraise: the consumer's own exception still propagates.
try:
yield session
except Exception:
session.rollback()
raise
finally:
session.close()
"""
P7_KNOWN = (
"src/local_deep_research/web/dependencies/auth.py",
"get_db_session_dep",
)
P7_EXPECTED = {
P7_KNOWN,
("src/local_deep_research/web/routers/rag.py", "generate"),
("src/local_deep_research/web/routers/research.py", "generate"),
}
def test_p7_positive_control():
sites, _ = scan_pattern_7(synthetic("p7_pos", P7_POSITIVE))
assert keys(sites) == {
("<synthetic:p7_pos>", "get_session_dep"),
("<synthetic:p7_pos>", "stream"),
}, report("pattern 7 positive control", sites)
def test_p7_negative_control():
sites, stats = scan_pattern_7(synthetic("p7_neg", P7_NEGATIVE))
# Both functions must still be recognised as generators, or the control
# passes because the scanner stopped looking.
assert stats["generator_functions"] == 2, stats
assert sites == [], report("pattern 7 negative control", sites)
def test_p7_examines_a_floor_of_sites():
_, stats = scan_pattern_7(modules())
assert stats["generator_functions"] >= 30, stats
def test_p7_known_live_instance_is_refound():
sites, _ = scan_pattern_7(modules())
assert P7_KNOWN in keys(sites), report(
"pattern 7: get_db_session_dep must still be flagged", sites
)
def test_p7_inventory_is_pinned():
sites, _ = scan_pattern_7(modules())
assert keys(sites) == P7_EXPECTED, report("pattern 7", sites)
# ===========================================================================
# Sweep report -- run with `-s` to print every finding.
# ===========================================================================
@pytest.mark.parametrize(
"label,runner",
[
(
"1 enum written by VALUE into a name-keyed column",
lambda m: scan_pattern_1(m)[0],
),
(
"1b name-keyed enum column filtered by plain strings",
lambda m: (
scan_pattern_1_filters(m)[0] + scan_pattern_1_filters(m)[1]
),
),
(
"2 counter allocating a UNIQUE key, decremented elsewhere",
lambda m: scan_pattern_2(m)[0],
),
(
"3 guard blind to a referencing table",
lambda m: scan_pattern_3(m)[0],
),
(
"4 .first() where a filter should have come first",
lambda m: scan_pattern_4(m)[0],
),
(
"5 request-supplied cap with no upper bound",
lambda m: scan_pattern_5(m)[0],
),
(
"6 reader-controlled value into a log line",
lambda m: scan_pattern_6_logs(m)[0],
),
(
"6c reader-controlled value into a response header",
lambda m: scan_pattern_6_headers(m)[0],
),
("7 try wrapping a yield", lambda m: scan_pattern_7(m)[0]),
],
)
def test_sweep_report(label, runner, capsys):
"""Dump each pattern's findings, and verify every one of them points at a
real line of real source -- which is what makes a stale pin visible."""
sites = runner(modules())
with capsys.disabled():
print("\n" + report(f"PATTERN {label}", sites))
for site in sites:
target = REPO_ROOT / site.path
assert target.is_file(), f"{site} names a file that does not exist"
line_count = len(target.read_text(encoding="utf-8").splitlines())
assert 1 <= site.lineno <= line_count, (
f"{site} points past the end of {site.path} ({line_count} lines)"
)