Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

13.5 KB
Raw Blame
#!/usr/bin/env python3
"""Scan added lines for customer PII and high-signal secrets.

Default mode reads staged additions. During a merge, it scans only lines added
relative to every parent, excluding unchanged imported content. `--changed`
covers the same scope as `just check`. `--text` scans a PR title/body or other
free text.

The optional Gitleaks pass conservatively excludes only entire staged files
identical to the same path in an incoming parent; modified files stay scanned.

Only humans should add `tooling/pii/exceptions` entries, unless they ask an
agent to.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
EXCEPTIONS_PATH = Path(__file__).resolve().with_name("exceptions")
MAX_FILE_BYTES = 1_000_000

SAFE_DOMAINS = frozenset(
    {
        "example.com",
        "example.org",
        "example.net",
        "example.edu",
        "macro.com",
        "localhost",
        "invalid",
        "test",
        "local",
        "users.noreply.github.com",
    }
)
SAFE_TLDS = frozenset({"example", "test", "localhost", "invalid", "local"})
SKIP_PATH_PREFIXES = (
    ".sqlx/",
    "packages/sdk/generated/",
)
SKIP_PATH_SUFFIXES = (
    "bun.lock",
    "package-lock.json",
    "Cargo.lock",
    "pnpm-lock.yaml",
    "flake.lock",
)
SKIP_PATHS = frozenset(
    {
        "tooling/pii/exceptions",
        "tooling/pii/test_scan.py",
    }
)

EMAIL_RE = re.compile(
    r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"
)
PHONE_RE = re.compile(
    r"(?<!\w)(?:\+\d{1,3}[\s.-]*)?(?:\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}|\+\d{8,15})(?!\w)"
)
ADDRESS_RE = re.compile(
    r"\b\d{1,5}[A-Za-z]?\s+(?:[A-Z][A-Za-z0-9.'-]+\s+){0,3}"
    r"(?:Street|St\.?|Avenue|Ave\.?|Road|Rd\.?|Boulevard|Blvd\.?|"
    r"Lane|Ln\.?|Drive|Dr\.?|Court|Ct\.?|Way|Place|Pl\.?)\b"
)
SECRET_PATTERNS = (
    ("secret", re.compile(r"-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----")),
    ("secret", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
    ("secret", re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}\b")),
    ("secret", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b")),
    ("secret", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b")),
)


@dataclass(frozen=True)
class Finding:
    path: str
    line_no: int
    kind: str
    value: str

    def render(self) -> str:
        location = self.path if self.line_no <= 0 else f"{self.path}:{self.line_no}"
        return f"{location}: {self.kind} [redacted]"


@dataclass
class Exceptions:
    values: set[str]
    path_values: set[str]
    domains: set[str]


def load_exceptions(path: Path = EXCEPTIONS_PATH) -> Exceptions:
    values: set[str] = set()
    path_values: set[str] = set()
    domains: set[str] = set()
    if not path.is_file():
        return Exceptions(values, path_values, domains)
    for raw in path.read_text(encoding="utf-8").splitlines():
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        lowered = line.lower()
        if lowered.startswith("@") and "/" not in lowered:
            domains.add(lowered[1:])
            continue
        if ":" in line:
            file_path, _, value = line.partition(":")
            if "/" in file_path or file_path.endswith((".rs", ".ts", ".tsx", ".py", ".md")):
                path_values.add(f"{file_path}:{value}".lower())
                continue
        values.add(lowered)
    return Exceptions(values, path_values, domains)


def email_domain(value: str) -> str:
    return value.rsplit("@", 1)[-1].lower().rstrip(".")


def is_safe_email(value: str) -> bool:
    domain = email_domain(value)
    if domain in SAFE_DOMAINS:
        return True
    tld = domain.rsplit(".", 1)[-1]
    return tld in SAFE_TLDS or domain.endswith(".example.com")


def is_excepted(path: str, value: str, exceptions: Exceptions) -> bool:
    lowered = value.lower()
    if lowered in exceptions.values:
        return True
    if f"{path}:{value}".lower() in exceptions.path_values:
        return True
    if "@" in value:
        domain = email_domain(value)
        if domain in exceptions.domains:
            return True
    return False


def skip_path(path: str) -> bool:
    if path in SKIP_PATHS:
        return True
    if path.startswith(SKIP_PATH_PREFIXES) or path.endswith(SKIP_PATH_SUFFIXES):
        return True
    return False


def scan_line(path: str, line_no: int, line: str, exceptions: Exceptions) -> list[Finding]:
    findings: list[Finding] = []

    def add(kind: str, value: str) -> None:
        if not is_excepted(path, value, exceptions):
            findings.append(Finding(path, line_no, kind, value))

    for match in EMAIL_RE.finditer(line):
        value = match.group(0)
        if not is_safe_email(value):
            add("email", value)
    for match in PHONE_RE.finditer(line):
        add("phone", re.sub(r"\s+", " ", match.group(0)).strip())
    for match in ADDRESS_RE.finditer(line):
        add("address", match.group(0))
    for kind, pattern in SECRET_PATTERNS:
        for match in pattern.finditer(line):
            add(kind, match.group(0))
    return findings


def parse_unified_diff(diff: str) -> list[Finding]:
    return list(iter_diff_findings(diff, load_exceptions()))


def iter_diff_findings(diff: str, exceptions: Exceptions) -> Iterable[Finding]:
    path = ""
    new_line_no = 0
    in_hunk = False
    for raw in diff.splitlines():
        if raw.startswith("diff --git "):
            path = ""
            in_hunk = False
            continue
        if not in_hunk and raw.startswith("+++ "):
            plus_path = raw[4:]
            if plus_path == "/dev/null":
                path = ""
                continue
            path = plus_path[2:] if plus_path.startswith("b/") else plus_path
            continue
        if raw.startswith("@@"):
            in_hunk = True
            match = re.search(r"\+(\d+)", raw)
            new_line_no = int(match.group(1)) if match else 0
            continue
        if not path or skip_path(path) or raw.startswith("Binary files "):
            continue
        if in_hunk and raw.startswith("+"):
            yield from scan_line(path, new_line_no, raw[1:], exceptions)
            new_line_no += 1
        elif raw.startswith("-") and not raw.startswith("---"):
            continue
        else:
            new_line_no += 1


def git(*args: str) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        ["git", *args],
        cwd=ROOT,
        check=False,
        text=True,
        capture_output=True,
    )


def staged_diff(parent: str | None = None) -> str:
    args = ["diff", "--cached", "-U0", "--diff-filter=AMR", "--no-color"]
    if parent is not None:
        args.extend([parent, "--"])
    result = git(*args)
    result.check_returncode()
    return result.stdout


def merge_parents() -> list[str]:
    """Verify incoming merge parents before excluding imported content."""
    result = git("rev-parse", "--git-path", "MERGE_HEAD")
    result.check_returncode()
    merge_head = ROOT / result.stdout.strip()
    if not merge_head.exists():
        return []
    parents = merge_head.read_text(encoding="utf-8").splitlines()
    if not parents:
        raise ValueError("MERGE_HEAD contains no parents")
    verified_parents = []
    for parent in parents:
        verified = git("rev-parse", "--verify", "--end-of-options", f"{parent}^{{commit}}")
        verified.check_returncode()
        verified_parents.append(verified.stdout.strip())
    return verified_parents


def staged_findings(exceptions: Exceptions) -> list[Finding]:
    """Keep only staged findings on lines newly added relative to every parent."""
    findings = list(iter_diff_findings(staged_diff(), exceptions))
    for parent in merge_parents():
        parent_findings = set(iter_diff_findings(staged_diff(parent), exceptions))
        findings = [finding for finding in findings if finding in parent_findings]
    return findings


def merge_base() -> str:
    result = git("merge-base", "HEAD", "origin/main")
    return result.stdout.strip()


def changed_diff() -> str:
    base = os.environ.get("CHECK_BASE") or merge_base() or "HEAD"
    args = ["diff", "-U0", "--diff-filter=AMR", "--no-color", base]
    result = git(*args)
    result.check_returncode()
    return result.stdout


def untracked_findings(exceptions: Exceptions) -> list[Finding]:
    result = git("ls-files", "--others", "--exclude-standard")
    findings: list[Finding] = []
    for path in result.stdout.splitlines():
        if skip_path(path):
            continue
        file_path = ROOT / path
        if not file_path.is_file():
            continue
        try:
            if file_path.stat().st_size > MAX_FILE_BYTES:
                continue
            text = file_path.read_text(encoding="utf-8")
        except (OSError, UnicodeDecodeError):
            continue
        if "\0" in text:
            continue
        for line_no, line in enumerate(text.splitlines(), start=1):
            findings.extend(scan_line(path, line_no, line, exceptions))
    return findings


def scan_text(text: str, exceptions: Exceptions, path: str = "<text>") -> list[Finding]:
    findings: list[Finding] = []
    for line_no, line in enumerate(text.splitlines() or [text], start=1):
        findings.extend(scan_line(path, line_no, line, exceptions))
    return findings


def unchanged_import(path: str, parents: list[str]) -> bool:
    """Exclude only a same-path staged blob imported unchanged from a parent."""
    if not path or Path(path).is_absolute() or ".." in Path(path).parts or "\0" in path:
        raise ValueError("Invalid Gitleaks finding path")
    staged = git("rev-parse", "--verify", "--end-of-options", f":0:{path}")
    staged.check_returncode()
    for parent in parents:
        entry = git("ls-tree", "-z", parent, "--", f":(literal){path}")
        entry.check_returncode()
        if not entry.stdout:
            continue
        metadata, found_path = entry.stdout.rstrip("\0").split("\t", 1)
        _, kind, oid = metadata.split()
        if found_path == path and kind == "blob" and oid == staged.stdout.strip():
            return True
    return False


def run_gitleaks() -> int:
    gitleaks = shutil.which("gitleaks")
    if not gitleaks:
        return 0
    command = [gitleaks, "protect", "--staged", "--redact", "--no-banner"]
    try:
        parents = merge_parents()
        if not parents:
            return subprocess.run(command, cwd=ROOT, check=False).returncode
        with tempfile.TemporaryDirectory(prefix="macro-pii-") as directory:
            report_path = Path(directory) / "gitleaks.json"
            result = subprocess.run(
                [*command, "--exit-code", "42", "--report-format", "json", "--report-path", str(report_path)],
                cwd=ROOT,
                check=False,
            )
            if result.returncode not in (0, 42):
                return result.returncode
            findings = json.loads(report_path.read_text(encoding="utf-8"))
        if not isinstance(findings, list) or bool(findings) != (result.returncode == 42):
            raise ValueError("Invalid Gitleaks report")
        blocked = 0
        for finding in findings:
            if (not isinstance(finding, dict)
                    or not isinstance(finding.get("File"), str)
                    or not isinstance(finding.get("RuleID"), str)
                    or type(finding.get("StartLine")) is not int):
                raise ValueError("Invalid Gitleaks finding")
            if unchanged_import(finding["File"], parents):
                continue
            print(f'{finding["File"]}:{finding["StartLine"]}: {finding["RuleID"]} [redacted]', file=sys.stderr)
            blocked += 1
        if not blocked and findings:
            print("Gitleaks findings are in unchanged files imported from a merge parent.", file=sys.stderr)
        return int(blocked != 0)
    except (OSError, ValueError, subprocess.CalledProcessError):
        print("Gitleaks merge scan failed; commit remains blocked.", file=sys.stderr)
        return 2


def report(findings: list[Finding]) -> int:
    if not findings:
        return 0
    print("PII or secret found in added lines:", file=sys.stderr)
    for finding in findings:
        print(finding.render(), file=sys.stderr)
    print(
        "Redact the value, or ask a human to add an exception in "
        "tooling/pii/exceptions if it is intentional.",
        file=sys.stderr,
    )
    return 1


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--changed",
        action="store_true",
        help="Scan added lines vs origin/main (and untracked files), like just check.",
    )
    parser.add_argument(
        "--text",
        metavar="TEXT",
        help="Scan a PR title/body or other free text. Use - to read stdin.",
    )
    parser.add_argument(
        "--skip-gitleaks",
        action="store_true",
        help="Skip the optional gitleaks pass.",
    )
    args = parser.parse_args(argv)
    exceptions = load_exceptions()

    if args.text is not None:
        text = sys.stdin.read() if args.text == "-" else args.text
        return report(scan_text(text, exceptions))

    if args.changed:
        findings = list(iter_diff_findings(changed_diff(), exceptions))
        findings.extend(untracked_findings(exceptions))
        return report(findings)

    findings = staged_findings(exceptions)
    status = report(findings)
    if status != 0 or args.skip_gitleaks:
        return status
    return run_gitleaks()


if __name__ == "__main__":
    sys.exit(main())