Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.3 KB
Raw Blame
#!/usr/bin/env python3
# ruff: noqa: T201 allow print statements
"""Gate a branch on new code duplication in Python and TypeScript.

Runs jscpd over this tree and, when it has reportable candidates, over a
worktree of the branch point. Diffs the two clone sets so only duplication
the branch introduces can fail the gate. Existing duplication is grandfathered
in. App code and test code are held to different bars: a clone between two test files may
run longer before it fails, because repeated setup is idiomatic in tests.

jscpd pairs files only within one format, so a clone the branch introduces
carries the format of a file the branch changed, and a change set that touches
one format can scan that format alone.

With --report-dir, also writes one findings file per language for the CI
report comment (.github/scripts/post-duplication-section.mjs).
"""

import re
import sys
import json
import hashlib
import argparse
import tempfile
import subprocess
import collections
from dataclasses import dataclass
from pathlib import Path, PurePosixPath

JSCPD_VERSION = "5.1.1"

FORMAT_EXTENSIONS: dict[str, tuple[str, ...]] = {
    "python": (".py", ".pyx", ".pxd", ".pxi"),
    "typescript": (".ts", ".mts", ".cts"),
    "tsx": (".tsx",),
}

# Migrations and generated code duplicate by design; .depot/ is a vendored
# mirror of .github/actions/.
IGNORE = "**/migrations/**,**/generated/**,**/generated.*,.depot/**"

# jscpd scan floor. Anything smaller is never reported at all.
SCAN_MIN_LINES = 10
SCAN_MIN_TOKENS = 70

TEST_PATH = re.compile(
    r"(^|/)(test|tests|__tests__|__mocks__)/"
    r"|(^|/)test_[^/]*\.py$"
    r"|_test\.py$"
    r"|conftest\.py$"
    r"|\.test\.tsx?$"
    r"|\.spec\.tsx?$"
    r"|\.stories\.tsx?$"
)

LIMITS_PATH = Path(__file__).with_name("lint-duplication.limits.json")
SCAN_TIMEOUT_SECONDS = 240


@dataclass(frozen=True, kw_only=True, slots=True)
class Limits:
    production: int
    test: int


def load_limits() -> Limits:
    data = json.loads(LIMITS_PATH.read_text())
    return Limits(production=int(data["production"]), test=int(data["test"]))


LIMITS = load_limits()
APP_MAX_NEW_CLONE_TOKENS = LIMITS.production
TEST_MAX_NEW_CLONE_TOKENS = LIMITS.test

LANGUAGES = ("python", "typescript")


def is_test_file(path: str) -> bool:
    return bool(TEST_PATH.search(path))


def formats_touched(changed_files: set[str]) -> list[str]:
    """Return the jscpd formats a clone in this change set can have.

    Two or more touched formats keep the full scan: a two-format run measured
    no cheaper than the three-format one. So does any changed file with no
    extension, which jscpd reads by shebang (`bin/sandbox` is Python) and which
    the extension lists above therefore cannot place.
    """
    if any(not PurePosixPath(path).suffix for path in changed_files):
        return list(FORMAT_EXTENSIONS)
    touched = [
        format_name
        for format_name, extensions in FORMAT_EXTENSIONS.items()
        if any(path.endswith(extensions) for path in changed_files)
    ]
    if len(touched) <= 1:
        return touched
    return list(FORMAT_EXTENSIONS)


def clone_language(clone: dict) -> str:
    return "python" if clone.get("format") == "python" else "typescript"


def find_gate_failures(clones: list[dict]) -> list[tuple[dict, bool]]:
    """Keep the new clones that fail the gate, worst first.

    A clone between two test files gets the looser bar; anything touching
    app code is held to the app bar.
    """
    failures = []
    for clone in clones:
        if not clone.get("isNew"):
            continue
        both_tests = is_test_file(clone["firstFile"]["name"]) and is_test_file(clone["secondFile"]["name"])
        bar = TEST_MAX_NEW_CLONE_TOKENS if both_tests else APP_MAX_NEW_CLONE_TOKENS
        if clone["tokens"] >= bar:
            failures.append((clone, both_tests))
    failures.sort(key=lambda item: -item[0]["tokens"])
    return failures


def build_findings(failures: list[tuple[dict, bool]]) -> dict[str, list[dict]]:
    """Shape the failures as per-language findings for the CI report files."""
    findings: dict[str, list[dict]] = {language: [] for language in LANGUAGES}
    for clone, _ in failures:
        findings[clone_language(clone)].append(
            {
                "first_file": clone["firstFile"]["name"],
                "first_start": clone["firstFile"]["start"],
                "second_file": clone["secondFile"]["name"],
                "second_start": clone["secondFile"]["start"],
                "lines": clone["lines"],
                "tokens": clone["tokens"],
            }
        )
    return findings


def write_findings(report_dir: str | None, findings: dict[str, list[dict]]) -> None:
    """Write the per-language findings files the CI report poster reads."""
    if not report_dir:
        return
    (Path(report_dir) / "duplication-scan-status.json").write_text(json.dumps({"status": "ok"}) + "\n")
    report_names = {"python": "duplication-findings-python.json", "typescript": "duplication-findings-ts.json"}
    for language, language_findings in findings.items():
        (Path(report_dir) / report_names[language]).write_text(json.dumps(language_findings, indent=2) + "\n")


def run_jscpd(scan_root: Path, out_dir: Path, formats: list[str]) -> list[dict]:
    """Scan one tree with jscpd and return its clone list."""
    try:
        proc = subprocess.run(
            [
                "npx",
                "--yes",
                f"jscpd@{JSCPD_VERSION}",
                "--format",
                ",".join(formats),
                "--min-lines",
                str(SCAN_MIN_LINES),
                "--min-tokens",
                str(SCAN_MIN_TOKENS),
                "--skip-comments",
                "--reporters",
                "json",
                "--output",
                str(out_dir),
                "--ignore",
                IGNORE,
                ".",
            ],
            capture_output=True,
            text=True,
            # Two scans must finish inside the CI job's timeout-minutes, or the
            # job dies before the failure section can post.
            timeout=SCAN_TIMEOUT_SECONDS,
            cwd=scan_root,
        )
    except subprocess.TimeoutExpired:
        print(f"duplication lint could not run: jscpd exceeded {SCAN_TIMEOUT_SECONDS}s scanning {scan_root}")
        raise SystemExit(2) from None
    report_path = out_dir / "jscpd-report.json"
    if proc.returncode != 0 or not report_path.exists():
        print(proc.stdout[-3000:])
        print(proc.stderr[-3000:])
        print(f"duplication lint could not run: jscpd exited {proc.returncode} scanning {scan_root}")
        raise SystemExit(2)
    return json.loads(report_path.read_text())["duplicates"]


def clone_key(clone: dict) -> tuple[frozenset, str]:
    """Pair key, insensitive to which side jscpd calls first and to where in
    the files the fragment sits.

    Keying on the fragment text (not the span) means edits elsewhere in the
    files do not re-flag an old clone as new; editing the duplicated block
    itself does, which is what the gate is for.
    """
    pair = frozenset((clone["firstFile"]["name"], clone["secondFile"]["name"]))
    return pair, hashlib.sha256(clone["fragment"].encode()).hexdigest()


def mark_new_clones(current: list[dict], baseline: list[dict], changed_files: set[str]) -> None:
    """Flag clones the baseline cannot account for, counting occurrences.

    A fragment already copied once between two files is grandfathered only
    for as many copies as the baseline holds: a third copy of the same
    fragment in the same file pair is new duplication and must be flagged.
    """
    available = collections.Counter(clone_key(clone) for clone in baseline)
    for clone in current:
        key = clone_key(clone)
        if key[0].isdisjoint(changed_files):
            clone["isNew"] = False
        elif available[key] > 0:
            available[key] -= 1
            clone["isNew"] = False
        else:
            clone["isNew"] = True


def resolve_baseline(base: str, repo: Path) -> str:
    """Return the ref to compare clones against.

    The branch point, not the base tip: jscpd matches clones with their
    locations, so a file that moved on the base since the branch forked
    re-flags every old clone inside it as new. Comparing against the
    merge-base keeps pre-existing duplication out of the gate no matter
    how far behind the branch falls.
    """
    merge_base = subprocess.run(
        ["git", "merge-base", base, "HEAD"], capture_output=True, text=True, cwd=repo
    ).stdout.strip()
    if merge_base:
        return merge_base
    print(f"Could not resolve the merge-base with {base!r} (shallow history?). Falling back to {base!r}.")
    return base


def changed_files_between(baseline: str, current: str, repo: Path) -> set[str]:
    return set(
        filter(
            None,
            subprocess.check_output(["git", "diff", "--name-only", "-z", baseline, current], cwd=repo, text=True).split(
                "\0"
            ),
        )
    )


def scan_revision(repo: Path, revision: str, scan_dir: Path, formats: list[str]) -> list[dict]:
    worktree = scan_dir / "worktree"
    scan_dir.mkdir()
    try:
        add = subprocess.run(
            ["git", "worktree", "add", "--detach", str(worktree), revision],
            capture_output=True,
            text=True,
            cwd=repo,
        )
        if add.returncode != 0:
            print(add.stderr[-2000:])
            print(f"duplication lint could not check out {revision}")
            raise SystemExit(2)
        return run_jscpd(worktree, scan_dir / "report", formats)
    finally:
        subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], capture_output=True, cwd=repo)


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--base", default="origin/master", help="git ref to compare clones against")
    parser.add_argument("--path", default=".", help="repository root to scan")
    parser.add_argument(
        "--report-dir",
        default=None,
        help="write duplication-findings-<language>.json files for the CI report into this directory",
    )
    args = parser.parse_args()

    if args.report_dir:
        # Written before any check or scan so an early exit can never leave a
        # stale "ok": the CI report can then tell "the scan failed" from "the
        # branch predates the check" (no files at all).
        (Path(args.report_dir) / "duplication-scan-status.json").write_text(json.dumps({"status": "failed"}) + "\n")

    if subprocess.run(["git", "rev-parse", "--verify", "--quiet", args.base], capture_output=True).returncode != 0:
        print(f"Base ref {args.base!r} not found. Fetch it first, e.g.:")
        print("  git fetch --no-tags --depth=1 origin master:refs/remotes/origin/master")
        return 2

    repo = Path(args.path).resolve()
    if not repo.is_dir():
        print(f"Scan path {repo} does not exist.")
        return 2

    baseline = resolve_baseline(args.base, repo)
    print(f"Comparing clones against {baseline}")
    changed_files = changed_files_between(baseline, "HEAD", repo)

    # Read before the scans: they skip the formats this change set cannot clone in.
    changed_files = changed_files_between(baseline, "HEAD", repo)
    formats = formats_touched(changed_files)
    if not formats:
        print("No changed file in a scanned format, so no clone can be new.")
        write_findings(args.report_dir, {language: [] for language in LANGUAGES})
        return 0
    print(f"Scanning {', '.join(formats)}")

    # jscpd's own --baseline-from-ref mismatches clones whose files moved on
    # the base since the branch forked, and some stable pairs it re-flags
    # with no visible cause. Scan both trees with identical flags and diff
    # the clone sets ourselves instead: a clone is new only when no clone in
    # the baseline pairs the same files over the same fragment. jscpd merges
    # repeated occurrences of one fragment into a single clone, so a repeat
    # copy inside an already-duplicated file pair is not detectable here;
    # new file pairs and edited blocks are.
    scan_failed = False
    with tempfile.TemporaryDirectory(prefix="jscpd-") as tmp:
        tmp_path = Path(tmp)
        # Registrations from runs killed mid-scan point at paths that no
        # longer exist; drop them before adding a fresh one.
        subprocess.run(["git", "worktree", "prune"], capture_output=True, cwd=repo)
        try:
            current_clones = scan_revision(repo, "HEAD", tmp_path / "current", formats)
            # A baseline can only grandfather candidates; it cannot add a finding.
            mark_new_clones(current_clones, [], changed_files)
            if find_gate_failures(current_clones):
                baseline_clones = scan_revision(repo, baseline, tmp_path / "baseline", formats)
                mark_new_clones(current_clones, baseline_clones, changed_files)
                print(
                    f"{len(current_clones)} clones in this tree, "
                    f"{sum(1 for c in current_clones if c['isNew'])} not in the baseline"
                )
            else:
                print("No reportable clones touch changed files; skipping the baseline scan.")
        except SystemExit:
            scan_failed = True
            current_clones = []

    if scan_failed:
        return 2

    if args.report_dir:
        (Path(args.report_dir) / "duplication-scan-status.json").write_text(json.dumps({"status": "ok"}) + "\n")

    failures = find_gate_failures(current_clones)
    write_findings(args.report_dir, build_findings(failures))
    if not failures:
        print("No new code duplication above the bars.")
        return 0

    print("This branch adds new code duplication:")
    for clone, both_tests in failures:
        first, second = clone["firstFile"], clone["secondFile"]
        side = "test" if both_tests else "app"
        print(
            f"  [{clone_language(clone)}] {first['name']}:{first['start']}-{first['end']} <-> "
            f"{second['name']}:{second['start']}-{second['end']} "
            f"({clone['lines']} lines, {clone['tokens']} tokens, {side} code)"
        )
    print()
    print(
        f"New duplication fails at {APP_MAX_NEW_CLONE_TOKENS}+ tokens in app code, "
        f"or {TEST_MAX_NEW_CLONE_TOKENS}+ tokens when both copies live in test files."
    )
    print("Extract the repeated block into a shared helper instead of copying it.")
    return 1


if __name__ == "__main__":
    sys.exit(main())