Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
ci(tests): track flaky test files on main in one rolling issue (#435)
Ruben Lütkec4cc725bb385
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 928eea04cd..cfdcb6e9c4 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -495,6 +495,12 @@ jobs:merge-multiple: true- name: Min-passed floor across all Linux shards (BLOCKING)run: python scripts/ci/ratchet_tests.py summary --floor --expected-shards 6 --baseline scripts/ci/test-baseline-linux.json reports+ # Report only: flaky files (failed in a batch, passed alone) and new vs+ # known failures. On main the cross-OS view and the "Flaky tests on main"+ # issue come from .github/workflows/flaky-tests.yml.+ - name: Flaky files and failures (report only)+ if: ${{ !cancelled() }}+ run: python scripts/ci/flaky_report.py summary reports- name: Merge the duration cache (main only)if: github.event_name == 'push' && github.ref == 'refs/heads/main'run: python scripts/ci/ratchet_tests.py durations --out test_durations.json reportsdiff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.ymlnew file mode 100644index 0000000000..553d801d81--- /dev/null+++ b/.github/workflows/flaky-tests.yml@@ -0,0 +1,120 @@+# Flaky-test tracking for main.+#+# After every CI run on main (push, nightly, manual dispatch) this reads the+# shard reports that scripts/ci/run_tests_parallel.py uploaded, writes flaky+# files plus new and known failures per OS to the step summary, and keeps ONE+# issue, "Flaky tests on main", updated with a rolling table (test file, OS,+# count, last seen run). Logic: scripts/ci/flaky_report.py.+#+# Why a separate workflow: the issue needs `issues: write`, and job permissions+# cannot depend on the event. Keeping it out of ci.yml means no pull-request+# run, and no fork, ever holds a token that can write issues. This workflow+# always runs main's own code and treats the downloaded reports as data only.+name: Flaky tests++on:+ workflow_run:+ workflows: [CI]+ types: [completed]+ branches: [main]+ workflow_dispatch:+ inputs:+ run_id:+ description: "CI run id on main to record (empty = the latest completed main run)"+ required: false+ type: string++permissions:+ contents: read++# Issue updates are read-modify-write; never let two run at once.+concurrency:+ group: flaky-tests+ cancel-in-progress: false++jobs:+ record:+ name: record flaky tests+ # A fork pull request can be named `main` too: the repository and event+ # checks, not the branch filter alone, keep its runs out.+ if: >-+ (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') ||+ (github.event_name == 'workflow_run' &&+ github.event.workflow_run.head_repository.full_name == github.repository &&+ github.event.workflow_run.head_branch == 'main' &&+ contains(fromJSON('["push","schedule","workflow_dispatch"]'), github.event.workflow_run.event) &&+ github.event.workflow_run.conclusion != 'cancelled' &&+ github.event.workflow_run.conclusion != 'skipped')+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ actions: read+ issues: write+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ ref: main+ persist-credentials: false+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.11"+ - name: Resolve and vet the CI run+ id: run+ env:+ GH_TOKEN: ${{ github.token }}+ RUN_ID: ${{ inputs.run_id || github.event.workflow_run.id }}+ run: |+ mkdir -p out+ if [ -z "$RUN_ID" ]; then+ RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml --branch main \+ --status completed --limit 1 --json databaseId --jq '.[0].databaseId')"+ fi+ case "$RUN_ID" in (''|*[!0-9]*) echo "::error::not a run id: $RUN_ID"; exit 1 ;; esac+ gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" > out/run.json+ python scripts/ci/flaky_report.py vet-run out/run.json --repo "$GITHUB_REPOSITORY"+ - name: Download the run's shard reports+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8+ with:+ run-id: ${{ steps.run.outputs.id }}+ github-token: ${{ github.token }}+ pattern: tests-*+ path: reports+ merge-multiple: true+ - name: Flaky files and failures (step summary)+ run: python scripts/ci/flaky_report.py summary reports+ - name: Update the tracking issue+ env:+ GH_TOKEN: ${{ github.token }}+ RUN_ID: ${{ steps.run.outputs.id }}+ RUN_URL: ${{ steps.run.outputs.url }}+ RUN_AT: ${{ steps.run.outputs.at }}+ run: |+ # The label lists issues without the search index, which lags behind+ # a freshly created issue and would invite a duplicate.+ gh issue list --repo "$GITHUB_REPOSITORY" --label flaky-tests --state all \+ --limit 50 --json number,title,state,body > out/issues.json+ python scripts/ci/flaky_report.py issue --issues out/issues.json --out out/issue.md \+ --run-id "$RUN_ID" --run-url "$RUN_URL" --run-at "$RUN_AT" reports \+ | tee out/plan.txt+ action="$(sed -n 's/^action=//p' out/plan.txt)"+ number="$(sed -n 's/^number=//p' out/plan.txt)"+ case "$action" in+ create)+ gh label create flaky-tests --repo "$GITHUB_REPOSITORY" --force \+ --color FBCA04 --description "Tracking test-order flakiness on main"+ url="$(gh issue create --repo "$GITHUB_REPOSITORY" --title "Flaky tests on main" \+ --label flaky-tests --body-file out/issue.md)"+ echo "Created $url" | tee -a "$GITHUB_STEP_SUMMARY" ;;+ reopen)+ gh issue reopen "$number" --repo "$GITHUB_REPOSITORY"+ gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file out/issue.md+ echo "Reopened and updated issue #$number" | tee -a "$GITHUB_STEP_SUMMARY" ;;+ edit)+ gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file out/issue.md+ echo "Updated issue #$number" | tee -a "$GITHUB_STEP_SUMMARY" ;;+ none)+ echo "Tracking issue unchanged." | tee -a "$GITHUB_STEP_SUMMARY" ;;+ *)+ echo "::error::unknown plan '$action'"; exit 1 ;;+ esacdiff --git a/.github/zizmor.yml b/.github/zizmor.ymlindex 94985599c2..3fce53c102 100644--- a/.github/zizmor.yml+++ b/.github/zizmor.yml@@ -10,6 +10,10 @@ rules:# act on main / release tags; neither checks out or runs PR head code.- merge-train.yml- release-finalize.yml+ # Reacts to completed CI runs on main only (repository, branch and event+ # checked in `if:` and again by flaky_report.py vet-run), checks out+ # `main`, and reads the downloaded shard reports as JSON data.+ - flaky-tests.ymlartipacked:ignore:# These jobs push commits or tags with git after checkout, so thediff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.mdindex ec08c65927..e78d407e73 100644--- a/docs/ci-pipeline.md+++ b/docs/ci-pipeline.md@@ -65,6 +65,23 @@ python scripts/ci/ratchet_tests.py prune --baseline scripts/ci/test-baseline-linA missing list means no failures have been approved for that OS; new failuresblock immediately. Missing, empty or malformed reports also block.+### Flaky tests++A file that fails inside its batch and passes when re-run alone is reported+as flaky. That almost always means test-order dependence, and the ratchet does+not block on it, so it is tracked instead. `test report + floor` lists flaky+files and new versus known failures in its step summary on every run. After+each push, nightly and manual CI run on main, `.github/workflows/flaky-tests.yml`+does the same for every OS and keeps one issue, "Flaky tests on main", updated+with a rolling table (test file, OS, number of runs it was flaky in, last seen+run). Rows age out after 30 days without a sighting. That workflow is the only+place with `issues: write`; it never runs for pull requests and refuses any CI+run that is not a finished run of this repository's main branch. Record an+older run by hand with+`gh workflow run flaky-tests.yml -f run_id=<ci-run-id>`; a run already+recorded is never counted twice. Logic and tests: `scripts/ci/flaky_report.py`,+`tests/unit/ci/test_flaky_report.py`.+Static gates include repository workflow policy (immutable action pins, thecomplete aggregate dependency graph, tag-only PyPI publication and draft-onlyasset producers). CI additionally runs pinned Actionlint for workflow syntax,diff --git a/scripts/ci/flaky_report.py b/scripts/ci/flaky_report.pynew file mode 100644index 0000000000..3ddcaadc5a--- /dev/null+++ b/scripts/ci/flaky_report.py@@ -0,0 +1,418 @@+#!/usr/bin/env python3+"""Flaky-test visibility for the sharded suite.++``run_tests_parallel.py`` re-runs a failed file alone and records it under+``flaky_files`` when that second attempt passes. Such a file is almost always+order-dependent (it leaks or relies on state from a neighbour in its batch),+yet the ratchet never blocks on it, so it used to stay invisible. This script+makes it visible in two places:++* ``summary`` writes flaky files plus new and known failures per OS to the+ GitHub step summary (read-only; safe on every event).+* ``vet-run`` / ``issue`` keep ONE tracking issue, "Flaky tests on main", with+ a rolling table (test file, OS, count, last seen run). The issue body carries+ its own machine-readable state, so no storage outside GitHub is needed and a+ re-run of the same CI run never counts twice.++The GitHub calls stay in ``.github/workflows/flaky-tests.yml``; everything+here is pure data in, text out, so the aggregation is unit-tested.++Commands::++ flaky_report.py summary [--baseline-dir scripts/ci] reports...+ flaky_report.py vet-run run.json --repo OWNER/NAME # -> GITHUB_OUTPUT lines+ flaky_report.py issue --issues issues.json --out body.md --run-id N+ --run-url URL --run-at ISO [--keep-days 30] reports... # -> GITHUB_OUTPUT lines+"""++from __future__ import annotations++import argparse+import json+import os+import re+import sys+from dataclasses import dataclass, field+from datetime import UTC, datetime, timedelta+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parents[2]+if str(REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(REPO_ROOT))++from scripts.ci.ratchet_tests import load_baseline, normalize # noqa: E402++ISSUE_TITLE = "Flaky tests on main"+ISSUE_LABEL = "flaky-tests"+OS_ORDER = ("linux", "windows", "macos")+TRUSTED_EVENTS = frozenset({"push", "schedule", "workflow_dispatch"})+KEEP_DAYS = 30+MAX_LISTED = 40 # per section of the step summary+MAX_RUNS_REMEMBERED = 200+_STATE = re.compile(r"<!--\s*flaky-state:\s*(\{.*?\})\s*-->", re.DOTALL)+_REPORT_NAME = re.compile(r"test-report-(linux|windows|macos)-")+_DASH_DASH = "-" + chr(92) + "u002d" # JSON escape for "-", spelled out for editors+++# --------------------------------------------------------------------------- reports+++@dataclass+class OsResult:+ """Everything one OS's shards said in one CI run."""++ shards: int = 0+ tests: int = 0+ passed: int = 0+ flaky: set[str] = field(default_factory=set)+ failures: set[str] = field(default_factory=set)+++def os_of(report: dict, path: Path | None = None) -> str:+ """Map a report to linux / windows / macos (platform first, file name second)."""+ platform = str(report.get("platform", ""))+ if platform.startswith("linux"):+ return "linux"+ if platform in {"win32", "cygwin", "msys"}:+ return "windows"+ if platform == "darwin":+ return "macos"+ match = _REPORT_NAME.search(path.name) if path else None+ return match.group(1) if match else "unknown"+++def report_files(paths: list[Path]) -> list[Path]:+ found: list[Path] = []+ for path in paths:+ if path.is_dir():+ found.extend(sorted(path.rglob("test-report*.json")))+ elif path.is_file():+ found.append(path)+ return found+++def collect(paths: list[Path]) -> dict[str, OsResult]:+ """Group shard reports by OS; an unreadable report is skipped with a warning."""+ results: dict[str, OsResult] = {}+ for path in report_files(paths):+ try:+ report = json.loads(path.read_text(encoding="utf-8-sig"))+ except (OSError, ValueError) as exc:+ print(f"::warning::unreadable test report {path}: {exc}")+ continue+ if not isinstance(report, dict):+ print(f"::warning::test report {path} is not an object")+ continue+ result = results.setdefault(os_of(report, path), OsResult())+ counts = report.get("counts") if isinstance(report.get("counts"), dict) else {}+ result.shards += 1+ result.tests += int(counts.get("tests", 0) or 0)+ result.passed += int(counts.get("passed", 0) or 0)+ result.flaky.update(str(f) for f in report.get("flaky_files", []) or [])+ result.failures.update(normalize(str(f)) for f in report.get("failed_ids", []) or [])+ return results+++def split_failures(failures: set[str], baseline: set[str] | None) -> tuple[list[str], list[str]]:+ """(new, known) — the ratchet's rule: anything outside the baseline is new."""+ known_ids = baseline or set()+ return sorted(failures - known_ids), sorted(failures & known_ids)+++def _ordered(results: dict[str, OsResult]) -> list[str]:+ return sorted(results, key=lambda name: (OS_ORDER + (name,)).index(name))+++def _code(text: str) -> str:+ return "`" + text.replace("`", "'").replace("|", "\\|") + "`"+++def _bullets(items: list[str]) -> list[str]:+ lines = [f"- {_code(item)}" for item in items[:MAX_LISTED]]+ if len(items) > MAX_LISTED:+ lines.append(f"- ... and {len(items) - MAX_LISTED} more")+ return lines+++def summary_lines(results: dict[str, OsResult], baseline_dir: Path) -> list[str]:+ lines = ["### Flaky files and failures", ""]+ if not results:+ return [*lines, "No test reports in this run (the python lane did not run)."]+ for name in _ordered(results):+ result = results[name]+ baseline = load_baseline(baseline_dir / f"test-baseline-{name}.json")+ new, known = split_failures(result.failures, baseline)+ lines.append(+ f"**{name}** ({result.shards} shard(s), {result.passed}/{result.tests} passed): "+ f"{len(result.flaky)} flaky file(s), {len(new)} new failure(s), "+ f"{len(known)} known failure(s)"+ )+ if result.flaky:+ lines += ["", "Flaky (failed in a batch, passed alone - suspect test order):"]+ lines += _bullets(sorted(result.flaky))+ if new:+ lines += ["", "New failures (not in the baseline):"]+ lines += _bullets(new)+ if known:+ lines += ["", "<details><summary>Known (baselined) failures</summary>", ""]+ lines += _bullets(known)+ lines += ["", "</details>"]+ lines.append("")+ return lines+++# --------------------------------------------------------------------------- run vetting+++def vet_run(run: dict, repo: str) -> tuple[dict[str, str], str]:+ """Accept only a finished CI run of this repository's main branch.++ The issue job holds ``issues: write``; a manually supplied run id must never+ let a pull-request run (possibly from a fork) feed it data.+ """+ head_repo = ((run.get("head_repository") or {}).get("full_name")) or ""+ if head_repo != repo:+ return {}, f"run belongs to {head_repo or 'an unknown repository'}, not {repo}"+ if run.get("head_branch") != "main":+ return {}, f"run is on branch {run.get('head_branch')!r}, not main"+ if run.get("event") not in TRUSTED_EVENTS:+ return {}, f"run was triggered by {run.get('event')!r}, not a main push or nightly"+ if run.get("status") != "completed":+ return {}, "run has not completed"+ run_id = str(run.get("id", ""))+ if not run_id.isdigit():+ return {}, "run has no numeric id"+ return {+ "id": run_id,+ "url": str(run.get("html_url", "")),+ "at": str(run.get("run_started_at") or run.get("created_at") or ""),+ "event": str(run.get("event")),+ }, ""+++# --------------------------------------------------------------------------- tracking issue+++def parse_state(body: str | None) -> dict:+ """Read the state block; a missing or hand-mangled block starts fresh."""+ empty: dict = {"entries": {}, "runs": []}+ match = _STATE.search(body or "")+ if not match:+ return empty+ try:+ state = json.loads(match.group(1))+ except ValueError:+ return empty+ if not isinstance(state, dict):+ return empty+ entries = state.get("entries")+ runs = state.get("runs")+ clean: dict[str, dict] = {}+ for key, entry in (entries if isinstance(entries, dict) else {}).items():+ if not (isinstance(entry, dict) and entry.get("file") and entry.get("os")):+ continue+ count = entry.get("count", 0)+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:+ continue+ clean[str(key)] = entry+ return {+ "entries": clean,+ "runs": [str(r) for r in runs] if isinstance(runs, list) else [],+ }+++def _parse_time(value: str) -> datetime | None:+ try:+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))+ except (AttributeError, ValueError):+ return None+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)+++def merge_run(+ state: dict,+ results: dict[str, OsResult],+ *,+ run_id: str,+ run_url: str,+ run_at: str,+ keep_days: int = KEEP_DAYS,+) -> dict:+ """Fold one CI run into the rolling state.++ Count = number of distinct main/nightly runs in which the file was flaky on+ that OS. A run already folded in (a re-run attempt, a manual backfill) adds+ nothing. Rows not seen for ``keep_days`` drop out.+ """+ entries = {key: dict(entry) for key, entry in state.get("entries", {}).items()}+ runs = list(state.get("runs", []))+ if run_id not in runs:+ for os_name, result in results.items():+ for file in result.flaky:+ key = f"{os_name}|{file}"+ entry = entries.get(key) or {"file": file, "os": os_name, "count": 0}+ entry["count"] = int(entry.get("count", 0) or 0) + 1+ entry.setdefault("first_seen", run_at)+ entry.update(last_seen=run_at, last_run_id=run_id, last_run_url=run_url)+ entries[key] = entry+ runs = [*runs, run_id][-MAX_RUNS_REMEMBERED:]+ now = _parse_time(run_at) or datetime.now(UTC)+ cutoff = now - timedelta(days=keep_days)+ kept = {}+ for key, entry in entries.items():+ seen = _parse_time(str(entry.get("last_seen", "")))+ if seen is None or seen >= cutoff:+ kept[key] = entry+ return {"entries": kept, "runs": runs, "updated": run_at, "updated_run": run_url}+++def render_issue(state: dict, keep_days: int = KEEP_DAYS) -> str:+ # Most frequent first; ties by most recently seen, then by OS and file.+ rows = sorted(state["entries"].values(), key=lambda e: (e["os"], e["file"]))+ rows.sort(key=lambda e: str(e.get("last_seen", "")), reverse=True)+ rows.sort(key=lambda e: int(e.get("count", 0) or 0), reverse=True)+ lines = [+ "Test files that failed inside their batch on `main` and then passed when the",+ "parallel runner re-ran them alone (`scripts/ci/run_tests_parallel.py`). That",+ "pattern almost always means test-order dependence: shared module state, a",+ "leaked patch, a fixed port or a temp file another test also uses. The ratchet",+ "does not block on them, so this table is where they stay visible.",+ "",+ "Updated automatically by `.github/workflows/flaky-tests.yml` after every",+ f"push and nightly CI run on `main`. Rows not seen for {keep_days} days drop out;",+ "fix a test, and its row ages out. Edits to this body are overwritten.",+ "",+ ]+ if rows:+ lines += ["| Test file | OS | Count | Last seen |", "| --- | --- | ---: | --- |"]+ for entry in rows:+ seen = str(entry.get("last_seen", ""))[:10] or "?"+ url = str(entry.get("last_run_url", ""))+ last = f"[{seen}]({url})" if url.startswith("https://") else seen+ lines.append(+ f"| {_code(str(entry['file']))} | {entry['os']} | {int(entry.get('count', 0))} "+ f"| {last} |"+ )+ else:+ lines.append(f"No flaky test files on `main` in the last {keep_days} days.")+ updated_run = str(state.get("updated_run", ""))+ if updated_run.startswith("https://"):+ lines += ["", f"Last update: [{str(state.get('updated', ''))[:16]}]({updated_run})"]+ # An HTML comment may not contain "--"; inside JSON strings the escape+ # decodes back to the same text.+ payload = json.dumps(state, sort_keys=True, separators=(",", ":")).replace("--", _DASH_DASH)+ lines += ["", f"<!-- flaky-state: {payload} -->", ""]+ return "\n".join(lines)+++def pick_issue(issues: list[dict]) -> dict | None:+ """The tracking issue: exact title, an open one first, then the oldest."""+ matches = [i for i in issues if isinstance(i, dict) and i.get("title") == ISSUE_TITLE]+ if not matches:+ return None+ matches.sort(key=lambda i: (str(i.get("state", "")).upper() != "OPEN", int(i["number"])))+ return matches[0]+++def plan_issue(issue: dict | None, body: str, has_rows: bool) -> str:+ """create | edit | reopen | none."""+ if issue is None:+ return "create" if has_rows else "none"+ if str(issue.get("state", "")).upper() != "OPEN":+ return "reopen" if has_rows else "none"+ return "edit" if issue.get("body") != body else "none"+++# --------------------------------------------------------------------------- commands+++def _output(values: dict[str, str]) -> None:+ lines = [f"{key}={value}" for key, value in values.items()]+ print("\n".join(lines))+ target = os.environ.get("GITHUB_OUTPUT")+ if target:+ with open(target, "a", encoding="utf-8") as handle:+ handle.write("\n".join(lines) + "\n")+++def _step_summary(lines: list[str]) -> None:+ target = os.environ.get("GITHUB_STEP_SUMMARY")+ if target:+ with open(target, "a", encoding="utf-8") as handle:+ handle.write("\n".join(lines) + "\n")+++def cmd_summary(args: argparse.Namespace) -> int:+ lines = summary_lines(collect(args.reports), args.baseline_dir)+ print("\n".join(lines))+ _step_summary(lines)+ return 0+++def cmd_vet_run(args: argparse.Namespace) -> int:+ run = json.loads(args.run.read_text(encoding="utf-8-sig"))+ values, reason = vet_run(run if isinstance(run, dict) else {}, args.repo)+ if reason:+ print(f"::error::refusing CI run: {reason}")+ return 1+ _output(values)+ return 0+++def cmd_issue(args: argparse.Namespace) -> int:+ if not args.run_id.isdigit():+ print("::error::--run-id must be numeric")+ return 1+ issues = []+ if args.issues.is_file():+ issues = json.loads(args.issues.read_text(encoding="utf-8-sig"))+ issue = pick_issue(issues if isinstance(issues, list) else [])+ state = merge_run(+ parse_state(issue.get("body") if issue else ""),+ collect(args.reports),+ run_id=args.run_id,+ run_url=args.run_url,+ run_at=args.run_at,+ keep_days=args.keep_days,+ )+ body = render_issue(state, args.keep_days)+ args.out.parent.mkdir(parents=True, exist_ok=True)+ args.out.write_text(body, encoding="utf-8")+ action = plan_issue(issue, body, bool(state["entries"]))+ _output(+ {+ "action": action,+ "number": str(issue["number"]) if issue else "",+ "rows": str(len(state["entries"])),+ }+ )+ return 0+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ sub = parser.add_subparsers(dest="cmd", required=True)+ summary = sub.add_parser("summary")+ summary.add_argument("--baseline-dir", type=Path, default=REPO_ROOT / "scripts" / "ci")+ summary.add_argument("reports", nargs="*", type=Path)+ vet = sub.add_parser("vet-run")+ vet.add_argument("run", type=Path)+ vet.add_argument("--repo", required=True)+ issue = sub.add_parser("issue")+ issue.add_argument("--issues", type=Path, required=True)+ issue.add_argument("--out", type=Path, required=True)+ issue.add_argument("--run-id", required=True)+ issue.add_argument("--run-url", required=True)+ issue.add_argument("--run-at", required=True)+ issue.add_argument("--keep-days", type=int, default=KEEP_DAYS)+ issue.add_argument("reports", nargs="*", type=Path)+ args = parser.parse_args(argv)+ for stream in (sys.stdout, sys.stderr):+ if hasattr(stream, "reconfigure"):+ stream.reconfigure(encoding="utf-8", errors="replace")+ handler = {"summary": cmd_summary, "vet-run": cmd_vet_run, "issue": cmd_issue}[args.cmd]+ return handler(args)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tests/unit/ci/test_flaky_report.py b/tests/unit/ci/test_flaky_report.pynew file mode 100644index 0000000000..32e7e6ccae--- /dev/null+++ b/tests/unit/ci/test_flaky_report.py@@ -0,0 +1,340 @@+"""Flaky-test aggregation: step summary, run vetting and the rolling tracking issue."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from scripts.ci import flaky_report++REPO = "PersonalJarvis/PersonalJarvis"+RUN_URL = "https://github.com/PersonalJarvis/PersonalJarvis/actions/runs/{}"+++def _report(tmp: Path, name: str, platform: str, *, flaky=(), failed=(), passed=10) -> Path:+ path = tmp / name+ path.parent.mkdir(parents=True, exist_ok=True)+ payload = {+ "shard": "1/1",+ "platform": platform,+ "counts": {"tests": passed + len(failed), "passed": passed, "failed": len(failed)},+ "failed_ids": list(failed),+ "flaky_files": list(flaky),+ }+ path.write_text(json.dumps(payload), encoding="utf-8")+ return path+++def _baseline(tmp: Path, os_name: str, known: list[str]) -> None:+ payload = {"known_failures": known}+ (tmp / f"test-baseline-{os_name}.json").write_text(json.dumps(payload), encoding="utf-8")+++# --------------------------------------------------------------------------- collect+++def test_collect_groups_shards_by_os_and_normalizes_crashes(tmp_path):+ _report(tmp_path, "tests-linux-1/test-report-linux-1.json", "linux", flaky=["tests/a.py"])+ _report(+ tmp_path,+ "tests-linux-2/test-report-linux-2.json",+ "linux",+ failed=["tests/b.py::<timeout 300s>"],+ )+ _report(tmp_path, "test-report-windows-1.json", "win32", flaky=["tests/a.py"])+ results = flaky_report.collect([tmp_path])+ assert set(results) == {"linux", "windows"}+ assert results["linux"].shards == 2+ assert results["linux"].flaky == {"tests/a.py"}+ assert results["linux"].failures == {"tests/b.py::<crash>"}+ assert results["windows"].flaky == {"tests/a.py"}+++def test_os_falls_back_to_the_report_file_name():+ assert flaky_report.os_of({}, Path("test-report-macos-3.json")) == "macos"+ assert flaky_report.os_of({"platform": "darwin"}) == "macos"+ assert flaky_report.os_of({}) == "unknown"+++def test_unreadable_report_is_skipped_not_fatal(tmp_path, capsys):+ (tmp_path / "test-report-linux-1.json").write_text("{not json", encoding="utf-8")+ _report(tmp_path, "test-report-linux-2.json", "linux", flaky=["tests/a.py"])+ results = flaky_report.collect([tmp_path, tmp_path / "missing"])+ assert results["linux"].shards == 1+ assert "unreadable test report" in capsys.readouterr().out+++# --------------------------------------------------------------------------- summary+++def test_summary_splits_new_from_known_per_os_baseline(tmp_path):+ _baseline(tmp_path, "linux", ["tests/k.py::test_known", "tests/c.py::<timeout 1s>"])+ _report(+ tmp_path,+ "r/test-report-linux-1.json",+ "linux",+ flaky=["tests/order.py"],+ failed=["tests/k.py::test_known", "tests/n.py::test_new", "tests/c.py::<timeout 9s>"],+ )+ results = flaky_report.collect([tmp_path / "r"])+ new, known = flaky_report.split_failures(+ results["linux"].failures,+ flaky_report.load_baseline(tmp_path / "test-baseline-linux.json"),+ )+ assert new == ["tests/n.py::test_new"]+ assert known == ["tests/c.py::<crash>", "tests/k.py::test_known"]+ text = "\n".join(flaky_report.summary_lines(results, tmp_path))+ assert "1 flaky file(s), 1 new failure(s), 2 known failure(s)" in text+ assert "`tests/order.py`" in text+ assert "`tests/n.py::test_new`" in text+++def test_summary_without_reports_says_so(tmp_path):+ lines = flaky_report.summary_lines({}, tmp_path)+ assert "No test reports" in lines[-1]+++def test_summary_command_writes_the_step_summary(tmp_path, monkeypatch):+ target = tmp_path / "summary.md"+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(target))+ _report(tmp_path, "r/test-report-linux-1.json", "linux", flaky=["tests/x.py"])+ args = ["summary", "--baseline-dir", str(tmp_path), str(tmp_path / "r")]+ assert flaky_report.main(args) == 0+ assert "`tests/x.py`" in target.read_text(encoding="utf-8")+++def test_long_lists_are_capped():+ lines = flaky_report._bullets([f"t{i}" for i in range(flaky_report.MAX_LISTED + 5)])+ assert len(lines) == flaky_report.MAX_LISTED + 1+ assert lines[-1] == "- ... and 5 more"+++# --------------------------------------------------------------------------- run vetting+++def _run(**overrides) -> dict:+ run = {+ "id": 42,+ "html_url": RUN_URL.format(42),+ "run_started_at": "2026-10-07T02:17:00Z",+ "event": "schedule",+ "status": "completed",+ "head_branch": "main",+ "head_repository": {"full_name": REPO},+ }+ run.update(overrides)+ return run+++def test_vet_run_accepts_a_finished_main_run():+ values, reason = flaky_report.vet_run(_run(), REPO)+ assert reason == ""+ assert values["id"] == "42"+ assert values["at"] == "2026-10-07T02:17:00Z"+++ ("overrides", "why"),+ [+ ({"event": "pull_request"}, "triggered by"),+ ({"head_repository": {"full_name": "fork/PersonalJarvis"}}, "belongs to"),+ ({"head_branch": "feature"}, "not main"),+ ({"status": "in_progress"}, "not completed"),+ ({"id": "1; rm -rf /"}, "numeric id"),+ ],+)+def test_vet_run_refuses_anything_but_trusted_main_runs(overrides, why):+ values, reason = flaky_report.vet_run(_run(**overrides), REPO)+ assert values == {}+ assert why in reason+++# --------------------------------------------------------------------------- tracking issue+++def _results(**flaky_by_os) -> dict[str, flaky_report.OsResult]:+ return {+ os_name: flaky_report.OsResult(flaky=set(files)) for os_name, files in flaky_by_os.items()+ }+++def test_merge_counts_runs_per_file_and_os_and_is_idempotent():+ state = {"entries": {}, "runs": []}+ state = flaky_report.merge_run(+ state,+ _results(linux=["tests/a.py"], windows=["tests/a.py"]),+ run_id="1",+ run_url=RUN_URL.format(1),+ run_at="2026-10-01T02:00:00Z",+ )+ again = flaky_report.merge_run(+ state,+ _results(linux=["tests/a.py"]),+ run_id="1",+ run_url=RUN_URL.format(1),+ run_at="2026-10-01T02:00:00Z",+ )+ assert again["entries"]["linux|tests/a.py"]["count"] == 1+ later = flaky_report.merge_run(+ again,+ _results(linux=["tests/a.py"]),+ run_id="2",+ run_url=RUN_URL.format(2),+ run_at="2026-10-02T02:00:00Z",+ )+ entry = later["entries"]["linux|tests/a.py"]+ assert entry["count"] == 2+ assert entry["first_seen"] == "2026-10-01T02:00:00Z"+ assert entry["last_seen"] == "2026-10-02T02:00:00Z"+ assert entry["last_run_url"] == RUN_URL.format(2)+ assert later["entries"]["windows|tests/a.py"]["count"] == 1+ assert later["runs"] == ["1", "2"]+++def test_rows_not_seen_within_the_window_age_out():+ state = flaky_report.merge_run(+ {"entries": {}, "runs": []},+ _results(linux=["tests/old.py"]),+ run_id="1",+ run_url=RUN_URL.format(1),+ run_at="2026-08-01T00:00:00Z",+ )+ state = flaky_report.merge_run(+ state,+ _results(linux=["tests/new.py"]),+ run_id="2",+ run_url=RUN_URL.format(2),+ run_at="2026-10-01T00:00:00Z",+ keep_days=30,+ )+ assert set(state["entries"]) == {"linux|tests/new.py"}+++def test_remembered_runs_are_bounded():+ state = {"entries": {}, "runs": [str(i) for i in range(flaky_report.MAX_RUNS_REMEMBERED)]}+ state = flaky_report.merge_run(+ state, {}, run_id="new", run_url="", run_at="2026-10-01T00:00:00Z"+ )+ assert len(state["runs"]) == flaky_report.MAX_RUNS_REMEMBERED+ assert state["runs"][-1] == "new"+++def test_rendered_body_round_trips_its_state_and_sorts_by_count():+ state = flaky_report.merge_run(+ {"entries": {}, "runs": []},+ _results(linux=["tests/once.py", "tests/twice.py"]),+ run_id="1",+ run_url=RUN_URL.format(1),+ run_at="2026-10-01T00:00:00Z",+ )+ state = flaky_report.merge_run(+ state,+ _results(linux=["tests/twice.py", "tests/a--b|c.py"]),+ run_id="2",+ run_url=RUN_URL.format(2),+ run_at="2026-10-02T00:00:00Z",+ )+ body = flaky_report.render_issue(state)+ table = [line for line in body.splitlines() if line.startswith("| `")]+ assert table[0].startswith("| `tests/twice.py` | linux | 2 |")+ assert "`tests/a--b\\|c.py`" in body+ comment = body[body.index("<!--") + 4 :]+ assert "--" not in comment[: comment.index("-->")]+ parsed = flaky_report.parse_state(body)+ assert parsed["entries"] == state["entries"]+ assert parsed["runs"] == ["1", "2"]+++def test_empty_state_renders_a_quiet_body():+ body = flaky_report.render_issue({"entries": {}, "runs": []})+ assert "No flaky test files" in body+ assert "| Test file |" not in body+++ "body",+ [+ None,+ "",+ "edited by hand",+ "<!-- flaky-state: {broken -->",+ '<!-- flaky-state: {"entries": [], "runs": "x"} -->',+ '<!-- flaky-state: {"entries": {"k": {"file": "a", "os": "linux", "count": "x"}}} -->',+ ],+)+def test_missing_or_mangled_state_starts_fresh(body):+ assert flaky_report.parse_state(body) == {"entries": {}, "runs": []}+++def test_pick_issue_prefers_the_open_exact_title():+ issues = [+ {"number": 5, "title": "Flaky tests on main (old)", "state": "OPEN"},+ {"number": 7, "title": flaky_report.ISSUE_TITLE, "state": "CLOSED"},+ {"number": 9, "title": flaky_report.ISSUE_TITLE, "state": "OPEN"},+ ]+ assert flaky_report.pick_issue(issues)["number"] == 9+ assert flaky_report.pick_issue(issues[:2])["number"] == 7+ assert flaky_report.pick_issue([]) is None+++ ("issue", "has_rows", "expected"),+ [+ (None, True, "create"),+ (None, False, "none"),+ ({"number": 1, "state": "CLOSED", "body": ""}, True, "reopen"),+ ({"number": 1, "state": "CLOSED", "body": ""}, False, "none"),+ ({"number": 1, "state": "OPEN", "body": "old"}, False, "edit"),+ ({"number": 1, "state": "OPEN", "body": "same"}, True, "none"),+ ],+)+def test_plan_issue(issue, has_rows, expected):+ assert flaky_report.plan_issue(issue, "same", has_rows) == expected+++def test_issue_command_end_to_end(tmp_path, monkeypatch):+ output = tmp_path / "github_output"+ monkeypatch.setenv("GITHUB_OUTPUT", str(output))+ _report(tmp_path, "r/test-report-linux-1.json", "linux", flaky=["tests/a.py"])+ previous = flaky_report.render_issue(+ flaky_report.merge_run(+ {"entries": {}, "runs": []},+ _results(linux=["tests/a.py"]),+ run_id="1",+ run_url=RUN_URL.format(1),+ run_at="2026-10-06T02:00:00Z",+ )+ )+ issues = tmp_path / "issues.json"+ issues.write_text(+ json.dumps(+ [{"number": 12, "title": flaky_report.ISSUE_TITLE, "state": "OPEN", "body": previous}]+ ),+ encoding="utf-8",+ )+ body = tmp_path / "out" / "body.md"+ args = [+ "issue",+ "--issues",+ str(issues),+ "--out",+ str(body),+ "--run-id",+ "2",+ "--run-url",+ RUN_URL.format(2),+ "--run-at",+ "2026-10-07T02:00:00Z",+ str(tmp_path / "r"),+ ]+ assert flaky_report.main(args) == 0+ assert "action=edit\nnumber=12\nrows=1\n" == output.read_text(encoding="utf-8")+ assert "| `tests/a.py` | linux | 2 |" in body.read_text(encoding="utf-8")+++def test_issue_command_rejects_a_non_numeric_run_id(tmp_path):+ args = ["issue", "--issues", str(tmp_path / "none.json"), "--out", str(tmp_path / "b.md")]+ args += ["--run-id", "x", "--run-url", "u", "--run-at", "2026-10-07T00:00:00Z"]+ assert flaky_report.main(args) == 1