:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): skip duplication baselines without candidates (#126664)
posthog[bot]f56994e88e04
diff --git a/bin/lint_duplication.py b/bin/lint_duplication.pyindex 36d3aa4f8c48..cd0ede18c9c6 100644--- a/bin/lint_duplication.py+++ b/bin/lint_duplication.py@@ -2,10 +2,10 @@# ruff: noqa: T201 allow print statements"""Gate a branch on new code duplication in Python and TypeScript.-Runs jscpd over this tree and over a worktree of the branch point, and-diffs the two clone sets, so only duplication the branch introduces can-fail the gate. Existing duplication is grandfathered in. App code and-test code are held to different bars: a clone between two test files may+Runs jscpd over this tree and, when it has reportable candidates, over a+worktree of the branch point. Diffs the two clone sets so only duplication+the branch introduces can fail the gate. Existing duplication is grandfathered+in. App code and test code are held to different bars: a clone between two test files mayrun longer before it fails, because repeated setup is idiomatic in tests.With --report-dir, also writes one findings file per language for the CI@@ -212,6 +212,25 @@ def changed_files_between(baseline: str, current: str, repo: Path) -> set[str]:)+def scan_revision(repo: Path, revision: str, scan_dir: Path) -> list[dict]:+ worktree = scan_dir / "worktree"+ scan_dir.mkdir()+ try:+ add = subprocess.run(+ ["git", "worktree", "add", "--detach", str(worktree), revision],+ capture_output=True,+ text=True,+ cwd=repo,+ )+ if add.returncode != 0:+ print(add.stderr[-2000:])+ print(f"duplication lint could not check out {revision}")+ raise SystemExit(2)+ return run_jscpd(worktree, scan_dir / "report")+ finally:+ subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], capture_output=True, cwd=repo)++def main() -> int:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--base", default="origin/master", help="git ref to compare clones against")@@ -241,6 +260,7 @@ def main() -> int:baseline = resolve_baseline(args.base, repo)print(f"Comparing clones against {baseline}")+ changed_files = changed_files_between(baseline, "HEAD", repo)# jscpd's own --baseline-from-ref mismatches clones whose files moved on# the base since the branch forked, and some stable pairs it re-flags@@ -253,41 +273,29 @@ def main() -> int:scan_failed = Falsewith tempfile.TemporaryDirectory(prefix="jscpd-") as tmp:tmp_path = Path(tmp)- current_worktree = tmp_path / "current-worktree"- baseline_worktree = tmp_path / "baseline-worktree"- scan_worktrees = ((current_worktree, "HEAD"), (baseline_worktree, baseline))# Registrations from runs killed mid-scan point at paths that no# longer exist; drop them before adding a fresh one.subprocess.run(["git", "worktree", "prune"], capture_output=True, cwd=repo)try:- for worktree, revision in scan_worktrees:- add = subprocess.run(- ["git", "worktree", "add", "--detach", str(worktree), revision],- capture_output=True,- text=True,- cwd=repo,+ current_clones = scan_revision(repo, "HEAD", tmp_path / "current")+ # A baseline can only grandfather candidates; it cannot add a finding.+ mark_new_clones(current_clones, [], changed_files)+ if find_gate_failures(current_clones):+ baseline_clones = scan_revision(repo, baseline, tmp_path / "baseline")+ mark_new_clones(current_clones, baseline_clones, changed_files)+ print(+ f"{len(current_clones)} clones in this tree, "+ f"{sum(1 for c in current_clones if c['isNew'])} not in the baseline")- if add.returncode != 0:- print(add.stderr[-2000:])- print(f"duplication lint could not check out {revision}")- return 2- current_clones = run_jscpd(current_worktree, tmp_path / "current-report")- baseline_clones = run_jscpd(baseline_worktree, tmp_path / "baseline-report")+ else:+ print("No reportable clones touch changed files; skipping the baseline scan.")except SystemExit:scan_failed = Truecurrent_clones = []- finally:- for worktree, _ in scan_worktrees:- subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], capture_output=True, cwd=repo)if scan_failed:return 2- changed_files = changed_files_between(baseline, "HEAD", repo)- mark_new_clones(current_clones, baseline_clones, changed_files)- print(- f"{len(current_clones)} clones in this tree, {sum(1 for c in current_clones if c['isNew'])} not in the baseline"- )if args.report_dir:(Path(args.report_dir) / "duplication-scan-status.json").write_text(json.dumps({"status": "ok"}) + "\n")diff --git a/bin/test/test_lint_duplication.py b/bin/test/test_lint_duplication.pyindex 56522ab242b3..b8e746cbb983 100644--- a/bin/test/test_lint_duplication.py+++ b/bin/test/test_lint_duplication.py@@ -1,6 +1,7 @@#!/usr/bin/env python3import sys+import jsonimport tempfileimport subprocessfrom pathlib import Path@@ -227,7 +228,30 @@ class TestChangedFilesBetween(unittest.TestCase):class TestCommittedScanTree(unittest.TestCase):- def test_scans_head_without_worktree_changes(self) -> None:+ @parameterized.expand(+ [+ ("no_clones", [], [], 1, 0),+ ("below_bar", [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS - 1)], [], 1, 0),+ ("unchanged_pair", [make_clone("unchanged.py", "other.py", 900)], [], 1, 0),+ (+ "grandfathered",+ [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)],+ [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)],+ 2,+ 0,+ ),+ ("new_clone", [make_clone("tracked.py", "other.py", APP_MAX_NEW_CLONE_TOKENS)], [], 2, 1),+ ("baseline_failed", [make_clone("tracked.py", "other.py", 900)], None, 2, 2),+ ]+ )+ def test_scans_head_without_worktree_changes(+ self,+ _name: str,+ current_clones: list[dict],+ baseline_clones: list[dict] | None,+ expected_scans: int,+ expected_exit: int,+ ) -> None:with tempfile.TemporaryDirectory() as temp_dir:repo = Path(temp_dir)subprocess.run(["git", "init", "--quiet"], check=True, cwd=repo)@@ -245,13 +269,73 @@ class TestCommittedScanTree(unittest.TestCase):def record_scan(scan_root: Path, _out_dir: Path) -> list[dict]:scans.append({path.name: path.read_text() for path in scan_root.glob("*.py")})- return []+ if len(scans) == 1:+ return current_clones+ if baseline_clones is None:+ raise SystemExit(2)+ return baseline_cloneswith patch("bin.lint_duplication.run_jscpd", side_effect=record_scan):- with patch.object(sys, "argv", ["lint_duplication.py", "--base", baseline, "--path", str(repo)]):- self.assertEqual(main(), 0)+ with patch.object(+ sys,+ "argv",+ ["lint_duplication.py", "--base", baseline, "--path", str(repo), "--report-dir", str(repo)],+ ):+ self.assertEqual(main(), expected_exit)++ self.assertEqual(scans, [{"tracked.py": "committed\n"}, {"tracked.py": "before\n"}][:expected_scans])+ self.assertEqual(+ json.loads((repo / "duplication-scan-status.json").read_text()),+ {"status": "failed" if expected_exit == 2 else "ok"},+ )+ if expected_exit != 2:+ findings = json.loads((repo / "duplication-findings-python.json").read_text())+ self.assertEqual(json.loads((repo / "duplication-findings-ts.json").read_text()), [])+ self.assertEqual(bool(findings), expected_exit == 1)+ if findings:+ self.assertEqual(findings[0]["first_file"], "tracked.py")+ self.assertEqual(findings[0]["tokens"], APP_MAX_NEW_CLONE_TOKENS)+ self.assertEqual(+ subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True).count(+ "worktree "+ ),+ 1,+ )++ def test_failed_post_checkout_hook_does_not_leave_a_worktree(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ repo = Path(temp_dir)+ subprocess.run(["git", "init", "--quiet"], check=True, cwd=repo)+ subprocess.run(["git", "config", "user.email", "[email protected]"], check=True, cwd=repo)+ subprocess.run(["git", "config", "user.name", "Test User"], check=True, cwd=repo)+ (repo / "tracked.py").write_text("committed\n")+ subprocess.run(["git", "add", "tracked.py"], check=True, cwd=repo)+ subprocess.run(["git", "commit", "--quiet", "-m", "initial"], check=True, cwd=repo)+ baseline = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, cwd=repo).strip()++ hook = repo / ".git" / "hooks" / "post-checkout"+ hook.write_text("#!/bin/sh\nexit 1\n")+ hook.chmod(0o755)++ with patch("bin.lint_duplication.run_jscpd", return_value=[]) as run_jscpd:+ with patch.object(+ sys,+ "argv",+ ["lint_duplication.py", "--base", baseline, "--path", str(repo), "--report-dir", str(repo)],+ ):+ self.assertEqual(main(), 2)- self.assertEqual(scans, [{"tracked.py": "committed\n"}, {"tracked.py": "before\n"}])+ run_jscpd.assert_not_called()+ self.assertEqual(+ json.loads((repo / "duplication-scan-status.json").read_text()),+ {"status": "failed"},+ )+ self.assertEqual(+ subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True).count(+ "worktree "+ ),+ 1,+ )if __name__ == "__main__":