Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(security): make the history scanner actually scan (#1667)
Raghav Chamadiya7d35ada808de
diff --git a/packages/core/src/repowise/core/analysis/history_scan.py b/packages/core/src/repowise/core/analysis/history_scan.pyindex 2c0977d66..32d5f342a 100644--- a/packages/core/src/repowise/core/analysis/history_scan.py+++ b/packages/core/src/repowise/core/analysis/history_scan.py@@ -44,6 +44,7 @@ from repowise.core.analysis.security_scan import (SecurityScanner,)from repowise.core.ingestion.models import EXTENSION_TO_LANGUAGE+from repowise.core.test_paths import is_test_related_pathdef _run_git(repo_path: Path, args: list[str], *, timeout: float = 30.0) -> str:@@ -216,9 +217,12 @@ class HistorySecurityScanner:else:rev_range = "--all"+ # --no-abbrev is load-bearing: --raw abbreviates blob SHAs to 8 chars+ # while ``rev-list --objects`` emits the full 40, so no lookup below+ # could ever hit. (--full-index does not affect --raw, only patches.)raw = _run_git(repo_path,- ["log", "--reverse", "--format=%H%x1f%aI", "--raw", rev_range],+ ["log", "--reverse", "--format=%H%x1f%aI", "--raw", "--no-abbrev", rev_range],timeout=120.0,)blob_introduced_at: dict[str, str] = {}@@ -232,9 +236,14 @@ class HistorySecurityScanner:if current_commit is None:continueparts = line.split()- if len(parts) < 3:+ if len(parts) < 4:continue- blob_sha = parts[2]+ # parts[2] is the pre-image blob, parts[3] the post-image one.+ # Keying on the pre-image attributed the *replaced* content and+ # was all-zero for every added file.+ blob_sha = parts[3]+ if blob_sha.strip("0") == "":+ continue # deletion: no post-image blobblob_introduced_at.setdefault(blob_sha, current_commit)continueif "\x1f" not in line:@@ -268,9 +277,13 @@ class HistorySecurityScanner:@staticmethoddef _is_source(path: str) -> bool:- """True when *path* has a language we scan (mirrors the indexer)."""- suffix = Path(path).suffix.lower().lstrip(".")- return suffix in EXTENSION_TO_LANGUAGE+ """True when *path* has a language we scan (mirrors the indexer).++ ``EXTENSION_TO_LANGUAGE`` is keyed *with* the leading dot, so stripping+ it here matched nothing and every blob was filtered out — the history+ scan walked the object graph and scanned zero files for its whole life.+ """+ return Path(path).suffix.lower() in EXTENSION_TO_LANGUAGE@staticmethoddef _passes_gate(kind: str, *, secrets_only: bool) -> bool:@@ -284,6 +297,16 @@ class HistorySecurityScanner:return kind in SECRET_KINDSreturn True+ @staticmethod+ def _is_placeholder(snippet: str | None) -> bool:+ """True when the matched line is documentation, not a credential.++ ``api_key="sk-..."`` in a docstring or README is the shape every+ provider example uses. An elided value is never a live secret, and on+ this repo it accounted for every non-test history hit.+ """+ return "..." in (snippet or "")+# ------------------------------------------------------------------# Scan driver# ------------------------------------------------------------------@@ -325,10 +348,17 @@ class HistorySecurityScanner:summary.blobs_scanned = len(blobs)blob_introduced_at, commit_dates = self._blob_introductions(repo_path, since, to)+ # Test material is excluded from history mode, not merely down-ranked.+ # A committed secret matters because it leaked a live credential, and a+ # fixture is not one: on this repo 730 of 832 history hits (88%) were+ # `api_key="sk-test"` in provider unit tests. That ratio makes the whole+ # report untrustworthy, which is worse than reporting nothing.+ # `--all-patterns` lifts this along with the kind gate.source_items = [(blob_sha, path)for blob_sha, path in blobs.items()- if not path or self._is_source(path)+ if (not path or self._is_source(path))+ and not (secrets_only and path and is_test_related_path(path))]contents_map = self._read_blobs_batch(repo_path, [blob_sha for blob_sha, _ in source_items]@@ -345,8 +375,10 @@ class HistorySecurityScanner:continuekept = [- f for f in findings+ f+ for f in findingsif self._passes_gate(f["kind"], secrets_only=secrets_only)+ and not (secrets_only and self._is_placeholder(f.get("snippet")))]if not kept:continuediff --git a/tests/unit/analysis/test_security_scan_history.py b/tests/unit/analysis/test_security_scan_history.pyindex 5664ab5d5..30544434e 100644--- a/tests/unit/analysis/test_security_scan_history.py+++ b/tests/unit/analysis/test_security_scan_history.py@@ -19,7 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom alembic import commandfrom alembic.config import Config-from sqlalchemy import inspect+from sqlalchemy import inspect, textfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_enginefrom repowise.core.analysis.history_scan import HistorySecurityScanner@@ -253,3 +253,128 @@ async def test_history_progress_fires_for_clean_repo(session: AsyncSession) -> Ndef test_secret_kinds_are_the_two_secret_patterns() -> None:"""Guard against the registry drifting away from the history gate."""assert {"hardcoded_password", "hardcoded_secret"} == SECRET_KINDS+++# ---------------------------------------------------------------------------+# End-to-end against a real git repo.+#+# Every test above stubs `_is_source` to `lambda p: True` and `_blob_+# introductions` to a literal, which is precisely why two bugs lived here+# undetected for the feature's whole life: `_is_source` stripped the leading+# dot off a suffix while EXTENSION_TO_LANGUAGE is keyed with it, so no blob was+# ever scanned; and `_blob_introductions` keyed the *pre-image* blob from an+# abbreviated `--raw` SHA, so no lookup could ever match and every history+# finding was written with an empty commit. These run the real helpers.+# ---------------------------------------------------------------------------+++def _git(repo: Path, *args: str) -> None:+ import subprocess++ subprocess.run(["git", *args], cwd=str(repo), check=True, capture_output=True)+++def secret_repo() -> Path:+ """A real one-commit git repo carrying a hardcoded secret in a .py file."""+ with tempfile.TemporaryDirectory() as tmp:+ repo = Path(tmp)+ _git(repo, "init")+ _git(repo, "config", "user.email", "[email protected]")+ _git(repo, "config", "user.name", "T")+ (repo / "leak.py").write_text('password = "hunter2-not-a-real-secret"\n')+ (repo / "notes.md").write_text("# just prose\n")+ _git(repo, "add", ".")+ _git(repo, "commit", "-m", "add leak")+ yield repo+++def test_is_source_accepts_real_source_paths() -> None:+ """The extension gate must match EXTENSION_TO_LANGUAGE's dotted keys."""+ assert HistorySecurityScanner._is_source("pkg/mod.py")+ assert HistorySecurityScanner._is_source("src/app.tsx")+ assert not HistorySecurityScanner._is_source("docs")+++async def test_history_scan_on_real_repo_records_commit_provenance(+ session: AsyncSession, secret_repo: Path+) -> None:+ """A real scan finds the secret and dates it to the commit that introduced it."""+ scanner = HistorySecurityScanner(session, "repo-1")+ summary = await scanner.scan_history(secret_repo, secrets_only=True)++ assert summary.files_scanned > 0, "the extension gate rejected every blob"+ assert summary.findings_inserted > 0, "the secret in leak.py was not found"++ rows = (+ await session.execute(+ text("SELECT file_path, line_number, commit_sha, commit_at FROM security_findings")+ )+ ).all()+ assert rows+ for row in rows:+ m = row._mapping+ assert m["file_path"] == "leak.py"+ assert m["line_number"] == 1+ assert m["commit_sha"], "no introducing commit recorded"+ assert m["commit_at"] is not None, "introducing commit has no date"+++async def test_history_default_skips_test_fixtures_and_placeholders(+ session: AsyncSession, secret_repo: Path+) -> None:+ """Secrets mode reports credentials, not fixtures and docs.++ Unfiltered, this repo's own history produced 832 hits of which 730 were+ ``api_key="sk-test"`` in unit tests and the rest were ``api_key="sk-..."``+ in docstrings. A report that is 100% noise is worse than no report.+ """+ _git(secret_repo, "config", "user.email", "[email protected]")+ (secret_repo / "tests").mkdir(exist_ok=True)+ (secret_repo / "tests" / "test_client.py").write_text('password = "fixture-value"\n')+ (secret_repo / "docs.py").write_text('# example: password = "..."\n')+ _git(secret_repo, "add", ".")+ _git(secret_repo, "commit", "-m", "add fixture and docs")++ scanner = HistorySecurityScanner(session, "repo-1")+ await scanner.scan_history(secret_repo, secrets_only=True)++ paths = {+ r._mapping["file_path"]+ for r in (await session.execute(text("SELECT file_path FROM security_findings"))).all()+ }+ assert "leak.py" in paths, "the real secret must still be reported"+ assert "tests/test_client.py" not in paths, "test fixture leaked into a secrets report"+ assert "docs.py" not in paths, "elided placeholder reported as a credential"+++async def test_all_patterns_lifts_the_noise_gate(+ session: AsyncSession, secret_repo: Path+) -> None:+ """The filtering is the default's promise, not a hard exclusion."""+ (secret_repo / "tests").mkdir(exist_ok=True)+ (secret_repo / "tests" / "test_client.py").write_text('password = "fixture-value"\n')+ _git(secret_repo, "add", ".")+ _git(secret_repo, "commit", "-m", "add fixture")++ scanner = HistorySecurityScanner(session, "repo-1")+ await scanner.scan_history(secret_repo, secrets_only=False)++ paths = {+ r._mapping["file_path"]+ for r in (await session.execute(text("SELECT file_path FROM security_findings"))).all()+ }+ assert "tests/test_client.py" in paths+++async def test_blob_introductions_resolves_real_blobs(secret_repo: Path) -> None:+ """Every source blob rev-list reports must resolve to an introducing commit."""+ scanner = HistorySecurityScanner.__new__(HistorySecurityScanner)+ blobs = scanner._unique_blobs(secret_repo, None, None)+ intro, dates = scanner._blob_introductions(secret_repo, None, None)++ source = [sha for sha, path in blobs.items() if HistorySecurityScanner._is_source(path)]+ assert source, "no source blobs found in the fixture repo"+ for sha in source:+ assert sha in intro, f"blob {sha} has no introducing commit (SHA width mismatch?)"+ assert dates.get(intro[sha]), "introducing commit has no author date"