Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): skip duplication baselines without candidates (#126664)

posthog[bot]f56994e88e04
diff --git a/bin/lint_duplication.py b/bin/lint_duplication.py
index 36d3aa4f8c48..cd0ede18c9c6 100644
--- a/bin/lint_duplication.py
+++ b/bin/lint_duplication.py
@@ -2,10 +2,10 @@
# ruff: noqa: T201 allow print statements
"""Gate a branch on new code duplication in Python and TypeScript.
-Runs jscpd over this tree and over a worktree of the branch point, and
-diffs the two clone sets, so only duplication the branch introduces can
-fail the gate. Existing duplication is grandfathered in. App code and
-test code are held to different bars: a clone between two test files may
+Runs jscpd over this tree and, when it has reportable candidates, over a
+worktree of the branch point. Diffs the two clone sets so only duplication
+the branch introduces can fail the gate. Existing duplication is grandfathered
+in. App code and test code are held to different bars: a clone between two test files may
run longer before it fails, because repeated setup is idiomatic in tests.
With --report-dir, also writes one findings file per language for the CI
@@ -212,6 +212,25 @@ def changed_files_between(baseline: str, current: str, repo: Path) -> set[str]:
)
+def scan_revision(repo: Path, revision: str, scan_dir: Path) -> list[dict]:
+ worktree = scan_dir / "worktree"
+ scan_dir.mkdir()
+ try:
+ add = subprocess.run(
+ ["git", "worktree", "add", "--detach", str(worktree), revision],
+ capture_output=True,
+ text=True,
+ cwd=repo,
+ )
+ if add.returncode != 0:
+ print(add.stderr[-2000:])
+ print(f"duplication lint could not check out {revision}")
+ raise SystemExit(2)
+ return run_jscpd(worktree, scan_dir / "report")
+ finally:
+ subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], capture_output=True, cwd=repo)
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base", default="origin/master", help="git ref to compare clones against")
@@ -241,6 +260,7 @@ def main() -> int:
baseline = resolve_baseline(args.base, repo)
print(f"Comparing clones against {baseline}")
+ changed_files = changed_files_between(baseline, "HEAD", repo)
# jscpd's own --baseline-from-ref mismatches clones whose files moved on
# the base since the branch forked, and some stable pairs it re-flags
@@ -253,41 +273,29 @@ def main() -> int:
scan_failed = False
with tempfile.TemporaryDirectory(prefix="jscpd-") as tmp:
tmp_path = Path(tmp)
- current_worktree = tmp_path / "current-worktree"
- baseline_worktree = tmp_path / "baseline-worktree"
- scan_worktrees = ((current_worktree, "HEAD"), (baseline_worktree, baseline))
# Registrations from runs killed mid-scan point at paths that no
# longer exist; drop them before adding a fresh one.
subprocess.run(["git", "worktree", "prune"], capture_output=True, cwd=repo)
try:
- for worktree, revision in scan_worktrees:
- add = subprocess.run(
- ["git", "worktree", "add", "--detach", str(worktree), revision],
- capture_output=True,
- text=True,
- cwd=repo,
+ current_clones = scan_revision(repo, "HEAD", tmp_path / "current")
+ # A baseline can only grandfather candidates; it cannot add a finding.
+ mark_new_clones(current_clones, [], changed_files)
+ if find_gate_failures(current_clones):
+ baseline_clones = scan_revision(repo, baseline, tmp_path / "baseline")
+ mark_new_clones(current_clones, baseline_clones, changed_files)
+ print(
+ f"{len(current_clones)} clones in this tree, "
+ f"{sum(1 for c in current_clones if c['isNew'])} not in the baseline"
)
- if add.returncode != 0:
- print(add.stderr[-2000:])
- print(f"duplication lint could not check out {revision}")
- return 2
- current_clones = run_jscpd(current_worktree, tmp_path / "current-report")
- baseline_clones = run_jscpd(baseline_worktree, tmp_path / "baseline-report")
+ else:
+ print("No reportable clones touch changed files; skipping the baseline scan.")
except SystemExit:
scan_failed = True
current_clones = []
- finally:
- for worktree, _ in scan_worktrees:
- subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], capture_output=True, cwd=repo)
if scan_failed:
return 2
- changed_files = changed_files_between(baseline, "HEAD", repo)
- mark_new_clones(current_clones, baseline_clones, changed_files)
- print(
- f"{len(current_clones)} clones in this tree, {sum(1 for c in current_clones if c['isNew'])} not in the baseline"
- )
if args.report_dir:
(Path(args.report_dir) / "duplication-scan-status.json").write_text(json.dumps({"status": "ok"}) + "\n")
diff --git a/bin/test/test_lint_duplication.py b/bin/test/test_lint_duplication.py
index 56522ab242b3..b8e746cbb983 100644
--- a/bin/test/test_lint_duplication.py
+++ b/bin/test/test_lint_duplication.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import sys
+import json
import tempfile
import subprocess
from pathlib import Path
@@ -227,7 +228,30 @@ class TestChangedFilesBetween(unittest.TestCase):
class TestCommittedScanTree(unittest.TestCase):
- def test_scans_head_without_worktree_changes(self) -> None:
+ @parameterized.expand(
+ [
+ ("no_clones", [], [], 1, 0),
+ ("below_bar", [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS - 1)], [], 1, 0),
+ ("unchanged_pair", [make_clone("unchanged.py", "other.py", 900)], [], 1, 0),
+ (
+ "grandfathered",
+ [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)],
+ [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)],
+ 2,
+ 0,
+ ),
+ ("new_clone", [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)], [], 2, 1),
+ ("baseline_failed", [make_clone("tracked.py", "other.py", 900)], None, 2, 2),
+ ]
+ )
+ def test_scans_head_without_worktree_changes(
+ self,
+ _name: str,
+ current_clones: list[dict],
+ baseline_clones: list[dict] | None,
+ expected_scans: int,
+ expected_exit: int,
+ ) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
repo = Path(temp_dir)
subprocess.run(["git", "init", "--quiet"], check=True, cwd=repo)
@@ -245,13 +269,73 @@ class TestCommittedScanTree(unittest.TestCase):
def record_scan(scan_root: Path, _out_dir: Path) -> list[dict]:
scans.append({path.name: path.read_text() for path in scan_root.glob("*.py")})
- return []
+ if len(scans) == 1:
+ return current_clones
+ if baseline_clones is None:
+ raise SystemExit(2)
+ return baseline_clones
with patch("bin.lint_duplication.run_jscpd", side_effect=record_scan):
- with patch.object(sys, "argv", ["lint_duplication.py", "--base", baseline, "--path", str(repo)]):
- self.assertEqual(main(), 0)
+ with patch.object(
+ sys,
+ "argv",
+ ["lint_duplication.py", "--base", baseline, "--path", str(repo), "--report-dir", str(repo)],
+ ):
+ self.assertEqual(main(), expected_exit)
+
+ self.assertEqual(scans, [{"tracked.py": "committed\n"}, {"tracked.py": "before\n"}][:expected_scans])
+ self.assertEqual(
+ json.loads((repo / "duplication-scan-status.json").read_text()),
+ {"status": "failed" if expected_exit == 2 else "ok"},
+ )
+ if expected_exit != 2:
+ findings = json.loads((repo / "duplication-findings-python.json").read_text())
+ self.assertEqual(json.loads((repo / "duplication-findings-ts.json").read_text()), [])
+ self.assertEqual(bool(findings), expected_exit == 1)
+ if findings:
+ self.assertEqual(findings[0]["first_file"], "tracked.py")
+ self.assertEqual(findings[0]["tokens"], APP_MAX_NEW_CLONE_TOKENS)
+ self.assertEqual(
+ subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True).count(
+ "worktree "
+ ),
+ 1,
+ )
+
+ def test_failed_post_checkout_hook_does_not_leave_a_worktree(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ repo = Path(temp_dir)
+ subprocess.run(["git", "init", "--quiet"], check=True, cwd=repo)
+ subprocess.run(["git", "config", "user.email", "[email protected]"], check=True, cwd=repo)
+ subprocess.run(["git", "config", "user.name", "Test User"], check=True, cwd=repo)
+ (repo / "tracked.py").write_text("committed\n")
+ subprocess.run(["git", "add", "tracked.py"], check=True, cwd=repo)
+ subprocess.run(["git", "commit", "--quiet", "-m", "initial"], check=True, cwd=repo)
+ baseline = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, cwd=repo).strip()
+
+ hook = repo / ".git" / "hooks" / "post-checkout"
+ hook.write_text("#!/bin/sh\nexit 1\n")
+ hook.chmod(0o755)
+
+ with patch("bin.lint_duplication.run_jscpd", return_value=[]) as run_jscpd:
+ with patch.object(
+ sys,
+ "argv",
+ ["lint_duplication.py", "--base", baseline, "--path", str(repo), "--report-dir", str(repo)],
+ ):
+ self.assertEqual(main(), 2)
- self.assertEqual(scans, [{"tracked.py": "committed\n"}, {"tracked.py": "before\n"}])
+ run_jscpd.assert_not_called()
+ self.assertEqual(
+ json.loads((repo / "duplication-scan-status.json").read_text()),
+ {"status": "failed"},
+ )
+ self.assertEqual(
+ subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True).count(
+ "worktree "
+ ),
+ 1,
+ )
if __name__ == "__main__":