Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(security): mask secrets in every stored snippet, and clear rows stored raw (#2496)
Raghav Chamadiya20987ccad135
diff --git a/docs/layers/SECURITY.md b/docs/layers/SECURITY.mdindex 6b5b47c3e..45499cdd2 100644--- a/docs/layers/SECURITY.md+++ b/docs/layers/SECURITY.md@@ -56,13 +56,13 @@ scored, ranked, or aggregated.| `subprocess_shell_true` | high | `subprocess.*` with `shell=True`, including across physical lines || `os_system` | high | `os.system` || `hardcoded_password` | high | an assignment of a quoted literal to a name containing `password`, any case |-| `hardcoded_secret` | high | the same for a name containing `api_key`, `apikey`, `secret`, `token` or `access_key` |+| `hardcoded_secret` | high | the same for a name containing `api_key`, `apikey`, `secret`, `token` or `access_key`. A snake_case value (a constant holding a key's name) or a template placeholder (`{{ ... }}`, `${...}`) is skipped, and a value one of the vendor kinds below already reports is not reported twice || `aws_access_key` | high | an AWS access key ID (`AKIA`/`ASIA` + 16 chars), regardless of variable name || `github_token` | high | a GitHub PAT, OAuth, app, or refresh token (`ghp_`/`gho_`/`ghu_`/`ghs_`/`ghr_`/`github_pat_`), regardless of variable name || `slack_token` | high | a Slack token (`xoxb-`, `xoxa-`, `xoxp-`, `xoxr-`, `xoxs-`), regardless of variable name |-| `google_api_key` | high | a Google API key (`AIza` + 35 chars), regardless of variable name |-| `stripe_key` | high | a Stripe secret or live publishable key (`sk_live_`, `sk_test_`, `pk_live_`), regardless of variable name |-| `private_key_pem` | high | a PEM header (`-----BEGIN ... PRIVATE KEY-----`) followed by a base64-looking body line, so assembling just the header string does not fire |+| `google_api_key` | high | a Google API key (`AIza` + 35 chars), regardless of variable name, but not from the middle of a base64 run such as a lockfile integrity hash |+| `stripe_key` | high | a Stripe secret or restricted key (`sk_` or `rk_`, then `live_`, `test_` or `prod_`), regardless of variable name. Publishable `pk_` keys are public by design and do not fire |+| `private_key_pem` | high | a PEM header (`-----BEGIN ... PRIVATE KEY-----`) followed by a base64-looking body line, so assembling just the header string does not fire. The break may be an escaped `\n`, which catches keys held on one line in JSON or `.env` files || `fstring_sql` | med | an f-string containing `SELECT` and an interpolation || `concat_sql` | med | `.execute("SELECT ... +` || `tls_verify_false` | med | `verify = False` |@@ -89,7 +89,8 @@ searched in raw source because the module usually arrives as a string literalthat masking would blank. Every other pattern in the table is a plain regex overone line of raw source, comments included.-**One pattern sees across lines.** `subprocess.run(` opening on one line with+**Two patterns see across lines.** `private_key_pem` needs the body line after+its header, and `subprocess.run(` opening on one line with`shell=True` several lines down is invisible to a per-line scan, so`subprocess_shell_true` gets a second pass over the whole source. Continuationis restricted to lines that begin with indentation and capped at roughly 200diff --git a/packages/core/alembic/versions/0077_clear_unmasked_credential_snippets.py b/packages/core/alembic/versions/0077_clear_unmasked_credential_snippets.pynew file mode 100644index 000000000..571a9a7ee--- /dev/null+++ b/packages/core/alembic/versions/0077_clear_unmasked_credential_snippets.py@@ -0,0 +1,40 @@+"""Clear credential snippets stored before full-line masking.++Rows written earlier can hold a raw value: a long secret cut before it was+masked, a ``public_env_secret`` value that was never masked, or a vendor-shape+secret sharing a line with another finding. Updates do not+rescan and history re-runs keep existing rows, so nothing else would replace+them. No downgrade: it would restore the secrets.++Revision ID: 0077+Revises: 0076+Create Date: 2026-09-23+"""++from __future__ import annotations++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++# revision identifiers+revision: str = "0077"+down_revision: str | None = "0076"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.get_bind().execute(+ sa.text(+ "UPDATE security_findings SET snippet = '' "+ "WHERE kind IN ('hardcoded_password', 'hardcoded_secret', 'public_env_secret', "+ "'aws_access_key', 'github_token', 'slack_token', 'google_api_key', "+ "'stripe_key', 'private_key_pem')"+ )+ )+++def downgrade() -> None:+ """No-op: restoring the snippets would re-expose the secrets."""diff --git a/packages/core/src/repowise/core/analysis/history_scan.py b/packages/core/src/repowise/core/analysis/history_scan.pyindex 73f199c0d..d90e1137d 100644--- a/packages/core/src/repowise/core/analysis/history_scan.py+++ b/packages/core/src/repowise/core/analysis/history_scan.py@@ -297,16 +297,6 @@ class HistorySecurityScanner:return kind in SECRET_KINDSreturn True- @staticmethod- def _is_placeholder(snippet: str | None) -> bool:- """True when the matched line is documentation, not a credential.-- ``api_key="sk-..."`` in a docstring or README is the shape every- provider example uses. An elided value is never a live secret, and on- this repo it accounted for every non-test history hit.- """- return "..." in (snippet or "")-# ------------------------------------------------------------------# Scan driver# ------------------------------------------------------------------@@ -370,12 +360,9 @@ class HistorySecurityScanner:if not findings:continue- kept = [- f- for f in findings- if self._passes_gate(f["kind"], secrets_only=secrets_only)- and not (secrets_only and self._is_placeholder(f.get("snippet")))- ]+ # Elided values ("sk-...") never get here: the scan's credential+ # check rejects them on the captured value, not the snippet.+ kept = [f for f in findings if self._passes_gate(f["kind"], secrets_only=secrets_only)]if not kept:continuediff --git a/packages/core/src/repowise/core/analysis/security_scan.py b/packages/core/src/repowise/core/analysis/security_scan.pyindex aee09eace..96cb1cfe9 100644--- a/packages/core/src/repowise/core/analysis/security_scan.py+++ b/packages/core/src/repowise/core/analysis/security_scan.py@@ -5,7 +5,7 @@ authentication, secret handling, raw SQL, dangerous deserialization, etc.Two scan surfaces share the same pattern registry and persistence layer:-* working-tree scans (during indexing) — ``SecurityScanner.scan_file`` ++* working-tree scans (during indexing) — ``scan_source`` / ``scan_source_map`` +``replace_findings`` with no commit provenance;* full-history scans (``repowise security scan --history``) — iterate everytracked revision of every source file and persist hits tagged with the@@ -21,6 +21,7 @@ from __future__ import annotationsimport astimport loggingimport re+from collections.abc import Iterable, Mappingfrom datetime import UTC, datetimefrom pathlib import PurePosixPathfrom typing import Any@@ -153,9 +154,16 @@ _PATTERNS: list[tuple[re.Pattern, str, str]] = ["high",),(re.compile(r"(?i:(xox[baprs]-[0-9a-zA-Z-]{10,72}))"), "slack_token", "high"),- (re.compile(r"\b(AIza[0-9A-Za-z_-]{35})\b"), "google_api_key", "high"),+ # Bounded by lookarounds, not ``\b``: a key can end in ``-``, and a base64 run+ # (a lockfile ``sha512-`` integrity hash) must not yield one from its middle.(- re.compile(r"\b((?:sk_(?:live|test)|pk_live)_[0-9A-Za-z]{10,99})\b"),+ re.compile(r"(?<![0-9A-Za-z+/_-])(AIza[0-9A-Za-z_-]{35})(?![0-9A-Za-z_-])"),+ "google_api_key",+ "high",+ ),+ # Secret and restricted keys only: a publishable ``pk_`` key is public by design.+ (+ re.compile(r"\b((?:sk|rk)_(?:live|test|prod)_[0-9A-Za-z]{10,99})\b"),"stripe_key","high",),@@ -237,8 +245,11 @@ _SPANNING_PATTERNS: list[tuple[re.Pattern, str, str]] = [# body line right after the header is what tells the two apart, and the# capture group around that body line is the value the SECRET_KINDS gate# below checks (length + placeholder), same as every other secret kind.+ # The break may be an escaped ``\n``: JSON and .env files hold the key on one line.(- re.compile(r"(?i:-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY-----)\r?\n([A-Za-z0-9+/=]{20,})"),+ re.compile(+ r"(?i:-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY-----)(?:\r?\n|\\n)([A-Za-z0-9+/=]{20,})"+ ),"private_key_pem","high",),@@ -274,44 +285,104 @@ SECRET_KINDS: frozenset[str] = frozenset(SYMBOL_NAME_KINDS: frozenset[str] = frozenset({"security_sensitive_symbol"})-def _mask_secret_snippet(snippet: str, val: str) -> str:- """Replace the first occurrence of *val* inside *snippet* with a redacted form.+_KEYWORD_KINDS: frozenset[str] = frozenset({"hardcoded_password", "hardcoded_secret"})- Keeps the variable name and surrounding text intact so the finding remains- locatable in the source file. The replacement is ``val[:4] + '****'``- (first four characters visible, everything else hidden). If the value is- shorter than four characters, the whole value is replaced with ``'****'``.+# A snake_case name is a key's name (a constant holding its own name) and a+# template placeholder is filled at render time; neither is a credential.+_KEY_NAME_VALUE = re.compile(r"[a-z_]*_[a-z_]*")+_TEMPLATE_VALUE = re.compile(r"\{\{.*\}\}|\$\{[^}]*\}")- Examples- --------- >>> _mask_secret_snippet("password = 'super_secret_pass_99'", "super_secret_pass_99")- "password = 'supe****'"- """- if not val:- return snippet- redacted = (val[:4] + "****") if len(val) >= 4 else "****"- return snippet.replace(val, redacted, 1)++def _is_secret_value(kind: str, val: str) -> bool:+ """True when *val*, captured by a *kind* pattern, looks like a real credential."""+ if not _is_valid_credential_value(val):+ return False+ if kind == "hardcoded_secret" and _KEY_NAME_VALUE.fullmatch(val):+ return False+ return not (kind in _KEYWORD_KINDS and _TEMPLATE_VALUE.fullmatch(val.strip()))-def _mask_findings(findings: list[dict]) -> None:- """Redact the captured secret value from every credential finding's snippet.+def _redaction(val: str) -> str:+ return (val[:4] + "****") if len(val) >= 4 else "****"- Operates in-place on *findings*. Only kinds listed in ``SECRET_KINDS`` are- touched; every other kind (code-smell patterns, symbol names, spanning- patterns not in SECRET_KINDS) is left unchanged.- This is called as a final post-pass inside ``scan_file`` before returning,- so it covers the per-line loop, the spanning-pattern loop, and the- ``HistorySecurityScanner`` path in one place.- """- for finding in findings:- if finding.get("kind") not in SECRET_KINDS:+# ``public_env_secret`` has no capture group, so its value is found here. In+# ``SECRET_KINDS`` the empty capture would fail validation and drop the finding.+_ASSIGNED_VALUE = re.compile(r"""\A['"]?\s*[:=]+\s*['"`]?([^'"`\s,;]+)""")++_SECRET_PATTERNS = [(p, kind) for p, kind, _ in _PATTERNS if kind in SECRET_KINDS]+_PUBLIC_ENV_PATTERN = next(p for p, kind, _ in _PATTERNS if kind == "public_env_secret")+# A PEM body written on its header's line (escaped ``\n`` breaks), up to ``-----END``.+_PEM_INLINE_BODY = re.compile(+ r"(?i:-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY-----)(?:\\[rn])*((?:[A-Za-z0-9+/=]|\\[rn])+)"+)+_SECRET_PREFILTER = re.compile(+ "|".join(+ f"(?:{p.pattern})"+ for p in [*(p for p, _ in _SECRET_PATTERNS), _PUBLIC_ENV_PATTERN, _PEM_INLINE_BODY]+ )+)++_SNIPPET_MAX = 120+_MARKER = "****"+++def _secret_spans(line: str) -> list[tuple[int, int]]:+ """Every credential value on *line*, including repeats of one elsewhere on it."""+ spans: list[tuple[int, int]] = []+ for pattern, kind in _SECRET_PATTERNS:+ for match in pattern.finditer(line):+ if _is_secret_value(kind, match.group(1)):+ spans.append(match.span(1))+ for match in _PEM_INLINE_BODY.finditer(line):+ if _is_valid_credential_value(match.group(1)):+ spans.append(match.span(1))+ for match in _PUBLIC_ENV_PATTERN.finditer(line):+ value = _ASSIGNED_VALUE.search(line[match.end() :])+ if value is not None:+ spans.append((match.end() + value.start(1), match.end() + value.end(1)))+ for val in {line[start:end] for start, end in spans}:+ if len(val) < 4:continue- val: str = finding.get("_secret_val", "")- if val:- finding["snippet"] = _mask_secret_snippet(finding.get("snippet", ""), val)- # Remove the internal key — it must never leave this module.- del finding["_secret_val"]+ at = line.find(val)+ while at != -1:+ spans.append((at, at + len(val)))+ at = line.find(val, at + 1)+ return spans+++def _mask_line(line: str) -> str:+ if not _SECRET_PREFILTER.search(line):+ return line+ merged: list[tuple[int, int]] = []+ for start, end in sorted(_secret_spans(line)):+ if merged and start < merged[-1][1]:+ merged[-1] = (merged[-1][0], max(end, merged[-1][1]))+ else:+ merged.append((start, end))+ out: list[str] = []+ pos = 0+ for start, end in merged:+ out.append(line[pos:start])+ out.append(_redaction(line[start:end]))+ pos = end+ out.append(line[pos:])+ return "".join(out)+++def _snippet(line: str) -> str:+ """*line* as stored: every secret on it masked, then stripped and trimmed.++ Masking comes first because a value cut by the trim no longer matches.+ """+ text = _mask_line(line).strip()+ cut = _SNIPPET_MAX+ if len(text) > cut:+ # Never end inside a marker: a bare ``*`` run defeats line verification.+ marker = text.rfind(_MARKER, 0, cut + len(_MARKER) - 1)+ if marker != -1 and marker < cut < marker + len(_MARKER):+ cut = marker+ return text[:cut]def _mask_comments_and_strings(source: str) -> str:@@ -437,12 +508,11 @@ def _call_findings(file_path: str, source: str) -> list[dict]:continueif call_name not in {"eval", "exec"}:continue- line = source.splitlines()[node.lineno - 1].strip()[:120]findings.append({"kind": f"{call_name}_call","severity": "high",- "snippet": line,+ "snippet": _snippet(source.splitlines()[node.lineno - 1]),"line": node.lineno,})@@ -454,7 +524,7 @@ def _call_findings(file_path: str, source: str) -> list[dict]:def add(kind: str, severity: str, offset: int) -> None:lineno = source.count("\n", 0, offset) + 1- snippet = lines[lineno - 1].strip()[:120] if lineno <= len(lines) else ""+ snippet = _snippet(lines[lineno - 1]) if lineno <= len(lines) else ""findings.append({"kind": kind, "severity": severity, "snippet": snippet, "line": lineno})is_python = file_path.lower().endswith((".py", ".pyi"))@@ -484,6 +554,126 @@ def _is_missing_table_error(exc: Exception) -> bool:return "no such table" in message or "does not exist" in message+def scan_source(file_path: str, source: str, symbols: Iterable[Any] = ()) -> list[dict]:+ """Scan *source* text and symbol names; return finding dicts. No I/O."""+ findings: list[dict] = []+ lines = source.splitlines()++ findings.extend(_call_findings(file_path, source))++ # Line-by-line pattern scan+ is_low_sev_file = _is_low_severity_path(file_path)+ for lineno, line in enumerate(lines, start=1):+ if not _ANY_PATTERN.search(line):+ continue+ snippet: str | None = None+ keyword_hits: list[tuple[dict, str]] = []+ vendor_values: list[str] = []+ for pattern, kind, severity in _PATTERNS:+ if kind in _CALL_KINDS:+ continue+ match = pattern.search(line)+ if match:+ if kind in SECRET_KINDS:+ if not _is_secret_value(kind, match.group(1)):+ continue+ if is_low_sev_file:+ severity = "low"+ if snippet is None:+ snippet = _snippet(line)+ finding = {+ "kind": kind,+ "severity": severity,+ "snippet": snippet,+ "line": lineno,+ }+ findings.append(finding)+ if kind in _KEYWORD_KINDS:+ keyword_hits.append((finding, match.group(1)))+ elif kind in SECRET_KINDS:+ vendor_values.append(match.group(1))+ # One secret, one finding: the vendor shape already names what the keyword saw.+ for finding, value in keyword_hits:+ if any(vendor in value for vendor in vendor_values):+ findings.remove(finding)++ # Calls that open on one line and set ``shell=True`` on a later one; reported+ # on the opening line, unless the per-line pass already did.+ pem_body_lines: set[int] = set()+ for pattern, kind, severity in _SPANNING_PATTERNS:+ for match in pattern.finditer(source):+ if kind in SECRET_KINDS:+ val = match.group(1) if match.groups() else ""+ if not _is_secret_value(kind, val):+ continue+ if is_low_sev_file:+ severity = "low"+ start_line = source.count("\n", 0, match.start()) + 1+ if kind == "private_key_pem":+ end = source.find("-----END", match.end())+ end_line = source.count("\n", 0, end if end != -1 else match.end()) + 1+ pem_body_lines.update(range(start_line + 1, end_line + (end == -1)))+ if any(f["kind"] == kind and f["line"] == start_line for f in findings):+ continue+ line_start = source.rfind("\n", 0, match.start()) + 1+ line_end = source.find("\n", match.start())+ if line_end == -1:+ line_end = len(source)+ findings.append(+ {+ "kind": kind,+ "severity": severity,+ "snippet": _snippet(source[line_start:line_end]),+ "line": start_line,+ }+ )++ # A key body line matches nothing of its own, so a hit there is stray and its+ # snippet would carry key material: mask the whole line.+ for finding in findings:+ if finding["line"] in pem_body_lines:+ finding["snippet"] = _redaction(lines[finding["line"] - 1].strip())++ # Symbol-name scan (informational / low)+ for sym in symbols:+ name = getattr(sym, "name", "") or getattr(sym, "qualified_name", "") or ""+ if name and _SYMBOL_KEYWORDS.search(name):+ findings.append(+ {+ "kind": "security_sensitive_symbol",+ "severity": "low",+ "snippet": name,+ "line": getattr(sym, "start_line", 0) or 0,+ }+ )++ return findings+++def scan_source_map(+ parsed_files: Iterable[Any],+ source_map: Mapping[str, bytes | str],+) -> tuple[dict[str, list[dict]], list[str]]:+ """Return ``(findings_by_file, scanned_paths)`` for ``replace_findings``.++ A path missing from *source_map* still gets the symbol-name scan.+ """+ findings_by_file: dict[str, list[dict]] = {}+ scanned_paths: list[str] = []+ for pf in parsed_files:+ path = pf.file_info.path+ raw = source_map.get(path, b"")+ if isinstance(raw, (bytes, bytearray)):+ source_text = raw.decode("utf-8", errors="replace")+ else:+ source_text = raw or ""+ scanned_paths.append(path)+ findings = scan_source(path, source_text, pf.symbols)+ if findings:+ findings_by_file[path] = findings+ return findings_by_file, scanned_paths++class SecurityScanner:"""Scan a single file for security signals and persist to the database."""@@ -497,113 +687,8 @@ class SecurityScanner:source: str,symbols: list[Any],) -> list[dict]:- """Scan *source* text and symbol names; return list of finding dicts.-- Parameters- ----------- file_path:- Relative path of the file (for reference only; not used in scan).- source:- Full text content of the file.- symbols:- List of symbol objects that have a ``name`` attribute (or similar).- """- findings: list[dict] = []- lines = source.splitlines()-- findings.extend(_call_findings(file_path, source))-- # Line-by-line pattern scan- is_low_sev_file = _is_low_severity_path(file_path)- for lineno, line in enumerate(lines, start=1):- if not _ANY_PATTERN.search(line):- continue- for pattern, kind, severity in _PATTERNS:- if kind in _CALL_KINDS:- continue- match = pattern.search(line)- if match:- if kind in SECRET_KINDS:- val = match.group(1) if match.groups() else ""- if not _is_valid_credential_value(val):- continue- if is_low_sev_file:- severity = "low"- # Trim snippet to keep it concise- snippet = line.strip()[:120]- findings.append(- {- "kind": kind,- "severity": severity,- "snippet": snippet,- "line": lineno,- # Internal: consumed by _mask_findings, never persisted.- "_secret_val": val,- }- )- continue- # Trim snippet to keep it concise- snippet = line.strip()[:120]- findings.append(- {- "kind": kind,- "severity": severity,- "snippet": snippet,- "line": lineno,- }- )-- # Whole-source pass for patterns that span physical lines- # (``subprocess.run(\n ...,\n shell=True,\n)``). The per-line- # loop above can never see the sink when the call opens on one line- # and ``shell=`` lands on another, so scan the full source once. The- # finding is reported on the line where the call starts, and a match- # the per-line pass already caught on that line is not duplicated.- for pattern, kind, severity in _SPANNING_PATTERNS:- for match in pattern.finditer(source):- if kind in SECRET_KINDS:- val = match.group(1) if match.groups() else ""- if not _is_valid_credential_value(val):- continue- if is_low_sev_file:- severity = "low"- start_line = source.count("\n", 0, match.start()) + 1- if any(f["kind"] == kind and f["line"] == start_line for f in findings):- continue- line_start = source.rfind("\n", 0, match.start()) + 1- line_end = source.find("\n", match.start())- if line_end == -1:- line_end = len(source)- snippet = source[line_start:line_end].strip()[:120]- findings.append(- {- "kind": kind,- "severity": severity,- "snippet": snippet,- "line": start_line,- }- )-- # Symbol-name scan (informational / low)- for sym in symbols:- name = getattr(sym, "name", "") or getattr(sym, "qualified_name", "") or ""- if name and _SYMBOL_KEYWORDS.search(name):- findings.append(- {- "kind": "security_sensitive_symbol",- "severity": "low",- "snippet": name,- "line": getattr(sym, "start_line", 0) or 0,- }- )-- # Mask secret values before any finding leaves this function.- # A single post-pass keyed on SECRET_KINDS covers the per-line loop,- # the spanning-pattern loop above, and HistorySecurityScanner (which- # calls this same method), so every present and future snippet site is- # covered by construction.- _mask_findings(findings)- return findings+ """Async wrapper over :func:`scan_source`."""+ return scan_source(file_path, source, symbols)def _uses_sqlite(self) -> bool:"""True when the bound session talks to SQLite (local/dev backend)."""diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 1e0e74ee3..d115eee1a 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -1613,24 +1613,11 @@ async def persist_security_findings(result: Any, session: Any, repo_id: str) ->it off ``file_info`` yields empty text and a scan that can never fire.Resume views without a ``source_map`` degrade to the symbol-name scan."""- from repowise.core.analysis.security_scan import SecurityScanner+ from repowise.core.analysis.security_scan import SecurityScanner, scan_source_map- scanner = SecurityScanner(session, repo_id)source_map = getattr(result, "source_map", None) or {}- findings_by_file: dict[str, list[dict]] = {}- scanned_paths: list[str] = []- for pf in result.parsed_files:- path = pf.file_info.path- raw = source_map.get(path, b"")- if isinstance(raw, (bytes, bytearray)):- source_text = raw.decode("utf-8", errors="replace")- else:- source_text = raw or ""- scanned_paths.append(path)- findings = await scanner.scan_file(path, source_text, pf.symbols)- if findings:- findings_by_file[path] = findings- await scanner.replace_findings(findings_by_file, scanned_paths)+ findings_by_file, scanned_paths = scan_source_map(result.parsed_files, source_map)+ await SecurityScanner(session, repo_id).replace_findings(findings_by_file, scanned_paths)async def persist_git(result: Any, session: Any, repo_id: str) -> None:diff --git a/packages/server/src/repowise/server/services/security_lines.py b/packages/server/src/repowise/server/services/security_lines.pyindex 5c15b8d4d..18c2bb6f7 100644--- a/packages/server/src/repowise/server/services/security_lines.py+++ b/packages/server/src/repowise/server/services/security_lines.py@@ -15,9 +15,9 @@ The contract, mirroring that module:* ``line_number: None`` — the snippet is gone from the file entirely. Thefinding is stale; a line here would point at unrelated code.-For the pattern scan the snippet is ``line.strip()[:120]`` (``security_scan.py``),-so it is always a substring of the line it came from and containment is a sound-gate. The symbol-name scan is the exception: its snippet is a bare identifier,+A pattern snippet is its trimmed source line, so containment is the gate; one+with a secret masked as ``****`` falls back to the text before the mask. The+symbol-name scan is the exception: its snippet is a bare identifier,which recurs all over a file, so those kinds are checked in place and neverrelocated or withdrawn — see ``SYMBOL_NAME_KINDS``."""@@ -53,25 +53,47 @@ def check_finding_line(if lines is None or not snippet:return LineCheck(line_number=line_number, verified=False)+ if kind in SYMBOL_NAME_KINDS:+ # A bare identifier is not discriminating enough to relocate on, and+ # its absence from the stored line does not mean the symbol is gone.+ found = _locate(lines, line_number, snippet, relocate=False)+ return found or LineCheck(line_number=line_number, verified=False)++ found = _locate(lines, line_number, snippet)+ if found is not None:+ return found+ # A masked secret leaves only the text before ``****`` verbatim (older rows+ # may end mid-mask). Too short a prefix would match unrelated lines.+ prefix = snippet.split("****", 1)[0].rstrip("*")+ if prefix != snippet and len("".join(prefix.split())) >= _MIN_PREFIX:+ found = _locate(lines, line_number, prefix)+ if found is not None:+ return found++ # Gone from the file: the finding outlived the code it describes.+ return LineCheck(line_number=None, verified=False)+++_MIN_PREFIX = 6+++def _locate(+ lines: list[str], line_number: int | None, needle: str, *, relocate: bool = True+) -> LineCheck | None:+ """The stored line if it holds *needle*, else a relocation; None if absent."""if (line_number is not Noneand 1 <= line_number <= len(lines)- and snippet in lines[line_number - 1]+ and needle in lines[line_number - 1]):return LineCheck(line_number=line_number, verified=True)-- if kind in SYMBOL_NAME_KINDS:- # A bare identifier is not discriminating enough to relocate on, and- # its absence from the stored line does not mean the symbol is gone.- return LineCheck(line_number=line_number, verified=False)-- matches = [i + 1 for i, text in enumerate(lines) if snippet in text]+ if not relocate:+ return None+ matches = [i + 1 for i, text in enumerate(lines) if needle in text]if len(matches) == 1:return LineCheck(line_number=matches[0], verified=True)if matches:# Ambiguous: the snippet recurs and the stored line is not one of the# hits. Serve the first as a pointer, but never as verified.return LineCheck(line_number=matches[0], verified=False)-- # Gone from the file: the finding outlived the code it describes.- return LineCheck(line_number=None, verified=False)+ return Nonediff --git a/packages/ui/src/security/findings-table.tsx b/packages/ui/src/security/findings-table.tsxindex aada4809d..867bb256f 100644--- a/packages/ui/src/security/findings-table.tsx+++ b/packages/ui/src/security/findings-table.tsx@@ -132,7 +132,7 @@ export function SecurityFindingsTable({ findings, onSelect, onGeneratePrompt }:className="block max-w-[320px] truncate font-mono text-xs text-[var(--color-text-tertiary)]"title={f.snippet ?? ""}>- {f.snippet ?? "—"}+ {f.snippet || "—"}</span>),},diff --git a/tests/unit/analysis/test_security_scan.py b/tests/unit/analysis/test_security_scan.pyindex 9f3b79b7b..1aec438cd 100644--- a/tests/unit/analysis/test_security_scan.py+++ b/tests/unit/analysis/test_security_scan.py@@ -17,7 +17,12 @@ from types import SimpleNamespaceimport pytestfrom sqlalchemy import text-from repowise.core.analysis.security_scan import SecurityScanner, _mask_secret_snippet+from repowise.core.analysis.security_scan import (+ SecurityScanner,+ _redaction,+ scan_source,+ scan_source_map,+)SNIPPY = b"""import pickle@@ -624,29 +629,16 @@ class TestPersistSecurityFindings:class TestSecretMasking:"""Credential snippets must be masked; non-credential snippets must not."""- # ------------------------------------------------------------------ #- # _mask_secret_snippet — pure-function unit tests #- # ------------------------------------------------------------------ #-@pytest.mark.parametrize(- ("snippet", "val", "expected"),+ ("val", "expected"),[- # Normal case: first 4 chars kept, rest replaced.- ("password = 'super_secret_pass_99'", "super_secret_pass_99", "password = 'supe****'"),- # Short-ish value: still 4 chars kept.- ("api_key = 'abcd1234'", "abcd1234", "api_key = 'abcd****'"),- # Value exactly 4 chars: keep all 4, append ****.- ("secret = 'abcd'", "abcd", "secret = 'abcd****'"),- # Value shorter than 4 chars: replace entirely.- ("secret = 'abc'", "abc", "secret = '****'"),- # Empty val: snippet unchanged.- ("password = 'something'", "", "password = 'something'"),+ ("super_secret_pass_99", "supe****"),+ ("abcd", "abcd****"),+ ("abc", "****"),],)- def test_mask_secret_snippet_helper(- self, snippet: str, val: str, expected: str- ) -> None:- assert _mask_secret_snippet(snippet, val) == expected+ def test_redaction_keeps_four_chars(self, val: str, expected: str) -> None:+ assert _redaction(val) == expected# ------------------------------------------------------------------ ## scan_file — credential kinds are masked #@@ -690,16 +682,11 @@ class TestSecretMasking:f"expected masked form {expected_mask!r} not found in {hit['snippet']!r}")- def test_credential_finding_has_no_internal_secret_val_key(self) -> None:- """The internal ``_secret_val`` key must be consumed by _mask_findings- and must never appear in a returned finding."""+ def test_credential_finding_carries_no_raw_value_field(self) -> None:+ """A finding holds only the persisted keys, so no raw value rides along."""source = "password = 'super_secret_password_123'\n"- scanner = SecurityScanner(session=None, repo_id="r1") # type: ignore[arg-type]- findings = asyncio.run(scanner.scan_file("config.py", source, symbols=[]))- for f in findings:- assert "_secret_val" not in f, (- f"internal key '_secret_val' leaked out of scan_file: {f}"- )+ for f in scan_source("config.py", source):+ assert set(f) == {"kind", "severity", "snippet", "line"}# ------------------------------------------------------------------ ## scan_file — non-credential kinds are NOT masked #@@ -729,3 +716,223 @@ class TestSecretMasking:assert hits[0]["severity"] == "low"assert "super_secret_real_password_99" not in hits[0]["snippet"]assert "supe****" in hits[0]["snippet"]+++class TestScanSource:+ """The session-free entry points, and masking that survives the length cap."""++ def test_long_token_is_masked_before_the_snippet_is_trimmed(self) -> None:+ """A value running past the 120-char cut must not ship its head raw."""+ token = "".join(chr(ord("a") + (i * 7) % 26) for i in range(220))+ findings = scan_source("config.py", f'API_KEY = "{token}"\n')+ hits = [f for f in findings if f["kind"] == "hardcoded_secret"]+ assert hits+ snippet = hits[0]["snippet"]+ assert len(snippet) <= 120+ assert token[:4] + "****" in snippet+ leaked = [token[i : i + 5] for i in range(len(token) - 4) if token[i : i + 5] in snippet]+ assert not leaked, f"raw token substrings in snippet: {leaked[:3]}"++ def test_recurring_value_is_masked_everywhere_on_the_line(self) -> None:+ source = "password = 'hunter2hunter2' # was hunter2hunter2\n"+ (hit,) = [f for f in scan_source("a.py", source) if f["kind"] == "hardcoded_password"]+ assert "hunter2hunter2" not in hit["snippet"]++ @pytest.mark.parametrize(+ ("source", "raw"),+ [+ ("NEXT_PUBLIC_STRIPE_SECRET_KEY=sk_live_0123456789abcdef\n", "sk_live_0123456789abcdef"),+ (' "VITE_API_KEY": "abcdef0123456789",\n', "abcdef0123456789"),+ ("export const NEXT_PUBLIC_AUTH_TOKEN = 'tok_9876543210';\n", "tok_9876543210"),+ ],+ )+ def test_public_env_secret_value_is_redacted(self, source: str, raw: str) -> None:+ hits = [f for f in scan_source("config.ts", source) if f["kind"] == "public_env_secret"]+ assert hits, "the finding must still be emitted"+ assert raw not in hits[0]["snippet"]+ assert raw[:4] + "****" in hits[0]["snippet"]++ def test_public_env_secret_without_a_value_is_unchanged(self) -> None:+ source = "const k = process.env.NEXT_PUBLIC_N8N_API_KEY;\n"+ (hit,) = [f for f in scan_source("config.ts", source) if f["kind"] == "public_env_secret"]+ assert hit["snippet"] == source.strip()++ def test_secret_kinds_and_history_gate_are_unchanged(self) -> None:+ from repowise.core.analysis.history_scan import HistorySecurityScanner+ from repowise.core.analysis.security_scan import SECRET_KINDS++ assert (+ frozenset(+ {+ "hardcoded_password",+ "hardcoded_secret",+ "aws_access_key",+ "github_token",+ "slack_token",+ "google_api_key",+ "stripe_key",+ "private_key_pem",+ }+ )+ == SECRET_KINDS+ )+ assert not HistorySecurityScanner._passes_gate("public_env_secret", secrets_only=True)++ def test_scan_file_wraps_scan_source(self) -> None:+ source = SNIPPY.decode()+ scanner = SecurityScanner(session=None, repo_id="r1") # type: ignore[arg-type]+ sym = SimpleNamespace(name="auth", start_line=2)+ assert asyncio.run(scanner.scan_file("a.py", source, [sym])) == scan_source(+ "a.py", source, [sym]+ )++ def test_scan_source_map_returns_replace_findings_inputs(self) -> None:+ result = _fake_result({"a.py": SNIPPY, "clean.py": CLEAN, "s.py": "x = 1\n"})+ findings_by_file, scanned = scan_source_map(result.parsed_files, result.source_map)+ assert scanned == ["a.py", "clean.py", "s.py"]+ assert set(findings_by_file) == {"a.py"}+ assert findings_by_file["a.py"] == scan_source("a.py", SNIPPY.decode())+++def _assert_no_raw(snippet: str, raw: str) -> None:+ leaked = [raw[i : i + 5] for i in range(len(raw) - 4) if raw[i : i + 5] in snippet]+ assert not leaked, f"raw value in {snippet!r}"+++class TestEverySnippetIsMasked:+ """A secret leaks through whichever finding on its line shows the line."""++ PW, KEY = "Qz8vK2mW9xL4pR7t", "Hj3nB6cX1vM5kS8d"++ def test_two_secrets_on_one_line(self) -> None:+ line = f'password = "{self.PW}"; api_key = "{self.KEY}"\n'+ findings = scan_source("a.py", line)+ assert {f["kind"] for f in findings} >= {"hardcoded_password", "hardcoded_secret"}+ for f in findings:+ _assert_no_raw(f["snippet"], self.PW)+ _assert_no_raw(f["snippet"], self.KEY)++ @pytest.mark.parametrize(+ ("source", "kind"),+ [+ (f'API_KEY = "{KEY}" # md5\n', "weak_hash"),+ (f'os.system(cmd); secret = "{KEY}"\n', "os_system"),+ (f'x = eval(y); secret = "{KEY}"\n', "eval_call"),+ (f'subprocess.run(cmd, secret = "{KEY}",\n shell=True)\n', "subprocess_shell_true"),+ ],+ )+ def test_secret_beside_another_pattern(self, source: str, kind: str) -> None:+ (hit,) = [f for f in scan_source("a.py", source) if f["kind"] == kind]+ _assert_no_raw(hit["snippet"], self.KEY)++ def test_two_public_env_values_on_one_line(self) -> None:+ line = "NEXT_PUBLIC_API_KEY=abcdef0123456789 VITE_AUTH_TOKEN=tok_9876543210\n"+ (hit,) = [f for f in scan_source("a.env.ts", line) if f["kind"] == "public_env_secret"]+ _assert_no_raw(hit["snippet"], "abcdef0123456789")+ _assert_no_raw(hit["snippet"], "tok_9876543210")++ def test_unterminated_value_with_trailing_space(self) -> None:+ (hit,) = scan_source("a.py", f'password = "{self.PW} \n')+ _assert_no_raw(hit["snippet"], self.PW)++ @pytest.mark.parametrize("pad", range(95, 120))+ def test_trim_never_ends_inside_a_mask(self, pad: int) -> None:+ from repowise.server.services.security_lines import check_finding_line++ line = f'{"x" * pad} password = "{self.PW}"'+ (hit,) = [f for f in scan_source("a.py", line + "\n") if f["kind"] == "hardcoded_password"]+ assert not hit["snippet"].endswith(("*", "**", "***")) or hit["snippet"].endswith("****")+ _assert_no_raw(hit["snippet"], self.PW)+ assert check_finding_line([line], 1, hit["snippet"], hit["kind"]).verified++ def test_a_repeated_value_masks_in_linear_time(self) -> None:+ import time++ line = " ".join(f'password = "{self.PW}";' for _ in range(5000))+ started = time.perf_counter()+ (hit,) = scan_source("a.py", line + "\n")+ assert time.perf_counter() - started < 1.0+ _assert_no_raw(hit["snippet"], self.PW)++ def test_long_github_pat_straddling_the_cut_is_masked(self) -> None:+ pat = "github_pat_" + "".join(chr(ord("A") + (i * 7) % 26) for i in range(82))+ line = f'{"x" * 80} = Client(auth="{pat}")'+ assert line.index(pat) < 120 < line.index(pat) + len(pat)+ (hit,) = [f for f in scan_source("a.py", line + "\n") if f["kind"] == "github_token"]+ assert "gith****" in hit["snippet"]+ _assert_no_raw(hit["snippet"], pat)++ def test_vendor_token_beside_a_weak_hash_is_masked_in_both(self) -> None:+ token = "ghp_" + "Kq7mZ2xV9pL4rT8wN3bY6cH1jF5dS0gA2eUo"+ findings = scan_source("a.py", f'digest = md5(body); auth = "{token}"\n')+ assert {f["kind"] for f in findings} == {"weak_hash", "github_token"}+ for f in findings:+ _assert_no_raw(f["snippet"], token)++ def test_inline_pem_body_is_masked(self) -> None:+ body = "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKj"+ line = (+ ' "private_key": "-----BEGIN PRIVATE KEY-----\\n'+ f'{body}\\n{body[::-1]}\\n-----END PRIVATE KEY-----\\n",'+ )+ (hit,) = [f for f in scan_source("sa.json", line + "\n") if f["kind"] == "private_key_pem"]+ _assert_no_raw(hit["snippet"], body)+ _assert_no_raw(hit["snippet"], body[::-1])++ def test_stray_hit_on_a_pem_body_line_is_masked(self) -> None:+ body = "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgE+md5/AoIBAQC7VJTUt9Us8cKj"+ source = f"-----BEGIN RSA PRIVATE KEY-----\n{body}\n-----END RSA PRIVATE KEY-----\n"+ (hit,) = [f for f in scan_source("id_rsa", source) if f["kind"] == "weak_hash"]+ assert hit["line"] == 2+ _assert_no_raw(hit["snippet"], body)+++class TestVendorShapeEdges:+ """Edges of the vendor-shape and keyword patterns."""++ @staticmethod+ def _kinds(source: str, path: str = "config.py") -> list[str]:+ return [f["kind"] for f in scan_source(path, source)]++ @pytest.mark.parametrize("prefix", ["sk_live_", "rk_live_", "sk_test_", "sk_prod_"])+ def test_stripe_secret_and_restricted_keys_fire(self, prefix: str) -> None:+ # Split so the contiguous prefix never sits in source for push protection.+ key = prefix[:3] + prefix[3:] + "A" * 24+ assert "stripe_key" in self._kinds(f'stripe_client = Stripe("{key}")\n')++ def test_stripe_publishable_key_does_not_fire(self) -> None:+ key = "pk_" + "live_" + "A" * 24+ assert "stripe_key" not in self._kinds(f'const stripe = loadStripe("{key}");\n', "a.ts")++ @pytest.mark.parametrize(+ "source",+ [+ 'ACCESS_TOKEN = "access_token"\n',+ 'const token = "auth_token_key";\n',+ 'token = "{{ csrf_token }}"\n',+ 'API_KEY = "${STRIPE_API_KEY}"\n',+ ],+ )+ def test_key_names_and_templates_are_not_secrets(self, source: str) -> None:+ assert "hardcoded_secret" not in self._kinds(source)++ def test_vendor_shape_and_keyword_report_one_finding(self) -> None:+ source = 'GITHUB_TOKEN = "ghp_abcdefghijklmnopqrstuvwxyzABCDEFGHIJ"\n'+ assert self._kinds(source) == ["github_token"]++ def test_pem_key_with_escaped_newline_is_flagged(self) -> None:+ source = (+ 'PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\\n'+ 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKj\\n"\n'+ )+ assert "private_key_pem" in self._kinds(source, ".env")++ def test_google_key_ending_in_a_dash_is_flagged(self) -> None:+ source = 'MAPS_KEY = "AIzaSyD9K2vQ7xR4mZ1pL8tY6wU3nB0cF5hD9a-"\n'+ assert "google_api_key" in self._kinds(source)++ def test_google_shape_inside_an_integrity_hash_is_ignored(self) -> None:+ run = "AIzaSyD9K2vQ7xR4mZ1pL8tY6wU3nB0cF5hD9aX"+ for integrity in (f"sha512-{run}+Q==", f"sha512-Zm9v/{run}/b2=="):+ source = f' "integrity": "{integrity}",\n'+ assert "google_api_key" not in self._kinds(source, "package-lock.json")diff --git a/tests/unit/analysis/test_security_scan_history.py b/tests/unit/analysis/test_security_scan_history.pyindex f684a6a04..100096bac 100644--- a/tests/unit/analysis/test_security_scan_history.py+++ b/tests/unit/analysis/test_security_scan_history.py@@ -163,6 +163,54 @@ def test_migration_0041_upgrades_sqlite() -> None:asyncio.run(engine.dispose())+def test_migration_0077_clears_credential_snippets_only() -> None:+ """Rows written before full-line masking lose their snippet; others keep it."""+ import sqlite3+ from contextlib import closing++ core_root = Path("packages/core")+ with tempfile.TemporaryDirectory() as tmp:+ db_path = Path(tmp) / "test.db"+ url = f"sqlite+aiosqlite:///{db_path}"+ prev_url = os.environ.get("DATABASE_URL")+ os.environ["DATABASE_URL"] = url+ prev_cwd = Path.cwd()+ try:+ os.chdir(core_root)+ alembic_cfg = Config("alembic.ini")+ alembic_cfg.set_main_option("sqlalchemy.url", url)+ with patch("logging.config.fileConfig"):+ command.upgrade(alembic_cfg, "0076")+ with closing(sqlite3.connect(db_path)) as conn, conn:+ conn.executemany(+ "INSERT INTO security_findings (repository_id, file_path, kind, "+ "severity, snippet, line_number, commit_sha, detected_at) "+ "VALUES ('r', 'a.ts', ?, 'high', ?, ?, ?, '2026-09-01')",+ [+ ("public_env_secret", "NEXT_PUBLIC_API_KEY=sk_live_raw", 1, ""),+ ("hardcoded_secret", 'API_KEY = "rawrawrawraw', 2, "abc123"),+ ("github_token", 'GH = "ghp_rawrawrawraw', 4, ""),+ ("eval_call", "eval(x)", 3, ""),+ ],+ )+ command.upgrade(alembic_cfg, "0077")+ finally:+ os.chdir(prev_cwd)+ if prev_url is None:+ os.environ.pop("DATABASE_URL", None)+ else:+ os.environ["DATABASE_URL"] = prev_url++ with closing(sqlite3.connect(db_path)) as conn:+ rows = dict(conn.execute("SELECT kind, snippet FROM security_findings").fetchall())+ assert rows == {+ "public_env_secret": "",+ "hardcoded_secret": "",+ "github_token": "",+ "eval_call": "eval(x)",+ }++async def test_history_gate_excludes_code_smells_by_default(session: AsyncSession) -> None:"""History mode keeps only secret kinds when secrets_only (default) is set."""assert HistorySecurityScanner._passes_gate("hardcoded_password", secrets_only=True)@@ -359,6 +407,23 @@ async def test_history_default_skips_test_fixtures_and_placeholders(assert "docs.py" not in paths, "elided placeholder reported as a credential"+async def test_history_keeps_a_real_secret_beside_an_elided_comment(+ session: AsyncSession, secret_repo: Path+) -> None:+ """Placeholder filtering reads the captured value, not the rest of the line."""+ (secret_repo / "cfg.py").write_text('password = "a9f3kQ2mZx7LpW1v" # rotate ...\n')+ _git(secret_repo, "add", ".")+ _git(secret_repo, "commit", "-m", "add cfg")++ await HistorySecurityScanner(session, "repo-1").scan_history(secret_repo, secrets_only=True)++ paths = {+ r._mapping["file_path"]+ for r in (await session.execute(text("SELECT file_path FROM security_findings"))).all()+ }+ assert "cfg.py" in paths++async def test_all_patterns_lifts_the_noise_gate(session: AsyncSession, secret_repo: Path) -> None:"""The filtering is the default's promise, not a hard exclusion."""(secret_repo / "tests").mkdir(exist_ok=True)diff --git a/tests/unit/server/test_security.py b/tests/unit/server/test_security.pyindex f60b9491d..18a9cc16f 100644--- a/tests/unit/server/test_security.py+++ b/tests/unit/server/test_security.py@@ -150,6 +150,27 @@ async def test_symbol_name_finding_is_never_relocated_or_withdrawn(assert row["line_verified"] is False+async def test_masked_credential_snippet_still_locates_its_line(+ client: AsyncClient, app+) -> None:+ """A masked snippet is not verbatim on its line, but the code is still there."""+ repo = await create_test_repo(client)+ src = Path(repo["local_path"]) / "app.py"+ src.write_text("import os\n\nAPI_KEY = 'sk_live_0123456789'\n")++ await _insert(+ app.state.session_factory,+ repo["id"],+ snippet="API_KEY = 'sk_l****'",+ line_number=1,+ )++ row = (await client.get(f"/api/repos/{repo['id']}/security")).json()[0]+ assert row["line_number"] == 3+ assert row["line_verified"] is True++@pytest.mark.asyncioasync def test_path_outside_the_repo_is_not_read(client: AsyncClient, app) -> None:"""A finding path is not a licence to read anywhere on disk."""@@ -216,3 +237,30 @@ async def test_working_tree_finding_has_no_commit_date(client: AsyncClient, app)row = (await client.get(f"/api/repos/{repo['id']}/security")).json()[0]assert row["found_in_history"] is Falseassert row["commit_at"] is None+++def test_snippet_cut_inside_a_mask_still_locates_its_line() -> None:+ from repowise.server.services.security_lines import check_finding_line++ line = 'API_KEY = "sk_live_0123456789"'+ check = check_finding_line(["", line], 1, 'API_KEY = "sk_l**', "hardcoded_secret")+ assert check.line_number == 2+ assert check.verified is True+++def test_a_literal_star_run_is_matched_verbatim_first() -> None:+ from repowise.server.services.security_lines import check_finding_line++ snippet = "****md5**** is weak"+ check = check_finding_line(["x = 1", snippet], 1, snippet, "weak_hash")+ assert (check.line_number, check.verified) == (2, True)+++def test_a_short_prefix_before_stars_is_not_a_match() -> None:+ """A deleted banner must not verify against any line starting with '/'."""+ from repowise.server.services.security_lines import check_finding_line++ snippet = "/******** md5 helpers ********/"+ check = check_finding_line(["/* other */", "int x;"], 1, snippet, "weak_hash")+ assert check.line_number is None+ assert check.verified is False