Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

ci(tests): track flaky test files on main in one rolling issue (#435)

Ruben Lütkec4cc725bb385
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 928eea04cd..cfdcb6e9c4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -495,6 +495,12 @@ jobs:
merge-multiple: true
- name: Min-passed floor across all Linux shards (BLOCKING)
run: python scripts/ci/ratchet_tests.py summary --floor --expected-shards 6 --baseline scripts/ci/test-baseline-linux.json reports
+ # Report only: flaky files (failed in a batch, passed alone) and new vs
+ # known failures. On main the cross-OS view and the "Flaky tests on main"
+ # issue come from .github/workflows/flaky-tests.yml.
+ - name: Flaky files and failures (report only)
+ if: ${{ !cancelled() }}
+ run: python scripts/ci/flaky_report.py summary reports
- name: Merge the duration cache (main only)
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: python scripts/ci/ratchet_tests.py durations --out test_durations.json reports
diff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.yml
new file mode 100644
index 0000000000..553d801d81
--- /dev/null
+++ b/.github/workflows/flaky-tests.yml
@@ -0,0 +1,120 @@
+# Flaky-test tracking for main.
+#
+# After every CI run on main (push, nightly, manual dispatch) this reads the
+# shard reports that scripts/ci/run_tests_parallel.py uploaded, writes flaky
+# files plus new and known failures per OS to the step summary, and keeps ONE
+# issue, "Flaky tests on main", updated with a rolling table (test file, OS,
+# count, last seen run). Logic: scripts/ci/flaky_report.py.
+#
+# Why a separate workflow: the issue needs `issues: write`, and job permissions
+# cannot depend on the event. Keeping it out of ci.yml means no pull-request
+# run, and no fork, ever holds a token that can write issues. This workflow
+# always runs main's own code and treats the downloaded reports as data only.
+name: Flaky tests
+
+on:
+ workflow_run:
+ workflows: [CI]
+ types: [completed]
+ branches: [main]
+ workflow_dispatch:
+ inputs:
+ run_id:
+ description: "CI run id on main to record (empty = the latest completed main run)"
+ required: false
+ type: string
+
+permissions:
+ contents: read
+
+# Issue updates are read-modify-write; never let two run at once.
+concurrency:
+ group: flaky-tests
+ cancel-in-progress: false
+
+jobs:
+ record:
+ name: record flaky tests
+ # A fork pull request can be named `main` too: the repository and event
+ # checks, not the branch filter alone, keep its runs out.
+ if: >-
+ (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') ||
+ (github.event_name == 'workflow_run' &&
+ github.event.workflow_run.head_repository.full_name == github.repository &&
+ github.event.workflow_run.head_branch == 'main' &&
+ contains(fromJSON('["push","schedule","workflow_dispatch"]'), github.event.workflow_run.event) &&
+ github.event.workflow_run.conclusion != 'cancelled' &&
+ github.event.workflow_run.conclusion != 'skipped')
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ actions: read
+ issues: write
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ ref: main
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.11"
+ - name: Resolve and vet the CI run
+ id: run
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_ID: ${{ inputs.run_id || github.event.workflow_run.id }}
+ run: |
+ mkdir -p out
+ if [ -z "$RUN_ID" ]; then
+ RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml --branch main \
+ --status completed --limit 1 --json databaseId --jq '.[0].databaseId')"
+ fi
+ case "$RUN_ID" in (''|*[!0-9]*) echo "::error::not a run id: $RUN_ID"; exit 1 ;; esac
+ gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" > out/run.json
+ python scripts/ci/flaky_report.py vet-run out/run.json --repo "$GITHUB_REPOSITORY"
+ - name: Download the run's shard reports
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+ with:
+ run-id: ${{ steps.run.outputs.id }}
+ github-token: ${{ github.token }}
+ pattern: tests-*
+ path: reports
+ merge-multiple: true
+ - name: Flaky files and failures (step summary)
+ run: python scripts/ci/flaky_report.py summary reports
+ - name: Update the tracking issue
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_ID: ${{ steps.run.outputs.id }}
+ RUN_URL: ${{ steps.run.outputs.url }}
+ RUN_AT: ${{ steps.run.outputs.at }}
+ run: |
+ # The label lists issues without the search index, which lags behind
+ # a freshly created issue and would invite a duplicate.
+ gh issue list --repo "$GITHUB_REPOSITORY" --label flaky-tests --state all \
+ --limit 50 --json number,title,state,body > out/issues.json
+ python scripts/ci/flaky_report.py issue --issues out/issues.json --out out/issue.md \
+ --run-id "$RUN_ID" --run-url "$RUN_URL" --run-at "$RUN_AT" reports \
+ | tee out/plan.txt
+ action="$(sed -n 's/^action=//p' out/plan.txt)"
+ number="$(sed -n 's/^number=//p' out/plan.txt)"
+ case "$action" in
+ create)
+ gh label create flaky-tests --repo "$GITHUB_REPOSITORY" --force \
+ --color FBCA04 --description "Tracking test-order flakiness on main"
+ url="$(gh issue create --repo "$GITHUB_REPOSITORY" --title "Flaky tests on main" \
+ --label flaky-tests --body-file out/issue.md)"
+ echo "Created $url" | tee -a "$GITHUB_STEP_SUMMARY" ;;
+ reopen)
+ gh issue reopen "$number" --repo "$GITHUB_REPOSITORY"
+ gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file out/issue.md
+ echo "Reopened and updated issue #$number" | tee -a "$GITHUB_STEP_SUMMARY" ;;
+ edit)
+ gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file out/issue.md
+ echo "Updated issue #$number" | tee -a "$GITHUB_STEP_SUMMARY" ;;
+ none)
+ echo "Tracking issue unchanged." | tee -a "$GITHUB_STEP_SUMMARY" ;;
+ *)
+ echo "::error::unknown plan '$action'"; exit 1 ;;
+ esac
diff --git a/.github/zizmor.yml b/.github/zizmor.yml
index 94985599c2..3fce53c102 100644
--- a/.github/zizmor.yml
+++ b/.github/zizmor.yml
@@ -10,6 +10,10 @@ rules:
# act on main / release tags; neither checks out or runs PR head code.
- merge-train.yml
- release-finalize.yml
+ # Reacts to completed CI runs on main only (repository, branch and event
+ # checked in `if:` and again by flaky_report.py vet-run), checks out
+ # `main`, and reads the downloaded shard reports as JSON data.
+ - flaky-tests.yml
artipacked:
ignore:
# These jobs push commits or tags with git after checkout, so the
diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md
index ec08c65927..e78d407e73 100644
--- a/docs/ci-pipeline.md
+++ b/docs/ci-pipeline.md
@@ -65,6 +65,23 @@ python scripts/ci/ratchet_tests.py prune --baseline scripts/ci/test-baseline-lin
A missing list means no failures have been approved for that OS; new failures
block immediately. Missing, empty or malformed reports also block.
+### Flaky tests
+
+A file that fails inside its batch and passes when re-run alone is reported
+as flaky. That almost always means test-order dependence, and the ratchet does
+not block on it, so it is tracked instead. `test report + floor` lists flaky
+files and new versus known failures in its step summary on every run. After
+each push, nightly and manual CI run on main, `.github/workflows/flaky-tests.yml`
+does the same for every OS and keeps one issue, "Flaky tests on main", updated
+with a rolling table (test file, OS, number of runs it was flaky in, last seen
+run). Rows age out after 30 days without a sighting. That workflow is the only
+place with `issues: write`; it never runs for pull requests and refuses any CI
+run that is not a finished run of this repository's main branch. Record an
+older run by hand with
+`gh workflow run flaky-tests.yml -f run_id=<ci-run-id>`; a run already
+recorded is never counted twice. Logic and tests: `scripts/ci/flaky_report.py`,
+`tests/unit/ci/test_flaky_report.py`.
+
Static gates include repository workflow policy (immutable action pins, the
complete aggregate dependency graph, tag-only PyPI publication and draft-only
asset producers). CI additionally runs pinned Actionlint for workflow syntax,
diff --git a/scripts/ci/flaky_report.py b/scripts/ci/flaky_report.py
new file mode 100644
index 0000000000..3ddcaadc5a
--- /dev/null
+++ b/scripts/ci/flaky_report.py
@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+"""Flaky-test visibility for the sharded suite.
+
+``run_tests_parallel.py`` re-runs a failed file alone and records it under
+``flaky_files`` when that second attempt passes. Such a file is almost always
+order-dependent (it leaks or relies on state from a neighbour in its batch),
+yet the ratchet never blocks on it, so it used to stay invisible. This script
+makes it visible in two places:
+
+* ``summary`` writes flaky files plus new and known failures per OS to the
+ GitHub step summary (read-only; safe on every event).
+* ``vet-run`` / ``issue`` keep ONE tracking issue, "Flaky tests on main", with
+ a rolling table (test file, OS, count, last seen run). The issue body carries
+ its own machine-readable state, so no storage outside GitHub is needed and a
+ re-run of the same CI run never counts twice.
+
+The GitHub calls stay in ``.github/workflows/flaky-tests.yml``; everything
+here is pure data in, text out, so the aggregation is unit-tested.
+
+Commands::
+
+ flaky_report.py summary [--baseline-dir scripts/ci] reports...
+ flaky_report.py vet-run run.json --repo OWNER/NAME # -> GITHUB_OUTPUT lines
+ flaky_report.py issue --issues issues.json --out body.md --run-id N
+ --run-url URL --run-at ISO [--keep-days 30] reports... # -> GITHUB_OUTPUT lines
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPO_ROOT))
+
+from scripts.ci.ratchet_tests import load_baseline, normalize # noqa: E402
+
+ISSUE_TITLE = "Flaky tests on main"
+ISSUE_LABEL = "flaky-tests"
+OS_ORDER = ("linux", "windows", "macos")
+TRUSTED_EVENTS = frozenset({"push", "schedule", "workflow_dispatch"})
+KEEP_DAYS = 30
+MAX_LISTED = 40 # per section of the step summary
+MAX_RUNS_REMEMBERED = 200
+_STATE = re.compile(r"<!--\s*flaky-state:\s*(\{.*?\})\s*-->", re.DOTALL)
+_REPORT_NAME = re.compile(r"test-report-(linux|windows|macos)-")
+_DASH_DASH = "-" + chr(92) + "u002d" # JSON escape for "-", spelled out for editors
+
+
+# --------------------------------------------------------------------------- reports
+
+
+@dataclass
+class OsResult:
+ """Everything one OS's shards said in one CI run."""
+
+ shards: int = 0
+ tests: int = 0
+ passed: int = 0
+ flaky: set[str] = field(default_factory=set)
+ failures: set[str] = field(default_factory=set)
+
+
+def os_of(report: dict, path: Path | None = None) -> str:
+ """Map a report to linux / windows / macos (platform first, file name second)."""
+ platform = str(report.get("platform", ""))
+ if platform.startswith("linux"):
+ return "linux"
+ if platform in {"win32", "cygwin", "msys"}:
+ return "windows"
+ if platform == "darwin":
+ return "macos"
+ match = _REPORT_NAME.search(path.name) if path else None
+ return match.group(1) if match else "unknown"
+
+
+def report_files(paths: list[Path]) -> list[Path]:
+ found: list[Path] = []
+ for path in paths:
+ if path.is_dir():
+ found.extend(sorted(path.rglob("test-report*.json")))
+ elif path.is_file():
+ found.append(path)
+ return found
+
+
+def collect(paths: list[Path]) -> dict[str, OsResult]:
+ """Group shard reports by OS; an unreadable report is skipped with a warning."""
+ results: dict[str, OsResult] = {}
+ for path in report_files(paths):
+ try:
+ report = json.loads(path.read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError) as exc:
+ print(f"::warning::unreadable test report {path}: {exc}")
+ continue
+ if not isinstance(report, dict):
+ print(f"::warning::test report {path} is not an object")
+ continue
+ result = results.setdefault(os_of(report, path), OsResult())
+ counts = report.get("counts") if isinstance(report.get("counts"), dict) else {}
+ result.shards += 1
+ result.tests += int(counts.get("tests", 0) or 0)
+ result.passed += int(counts.get("passed", 0) or 0)
+ result.flaky.update(str(f) for f in report.get("flaky_files", []) or [])
+ result.failures.update(normalize(str(f)) for f in report.get("failed_ids", []) or [])
+ return results
+
+
+def split_failures(failures: set[str], baseline: set[str] | None) -> tuple[list[str], list[str]]:
+ """(new, known) — the ratchet's rule: anything outside the baseline is new."""
+ known_ids = baseline or set()
+ return sorted(failures - known_ids), sorted(failures & known_ids)
+
+
+def _ordered(results: dict[str, OsResult]) -> list[str]:
+ return sorted(results, key=lambda name: (OS_ORDER + (name,)).index(name))
+
+
+def _code(text: str) -> str:
+ return "`" + text.replace("`", "'").replace("|", "\\|") + "`"
+
+
+def _bullets(items: list[str]) -> list[str]:
+ lines = [f"- {_code(item)}" for item in items[:MAX_LISTED]]
+ if len(items) > MAX_LISTED:
+ lines.append(f"- ... and {len(items) - MAX_LISTED} more")
+ return lines
+
+
+def summary_lines(results: dict[str, OsResult], baseline_dir: Path) -> list[str]:
+ lines = ["### Flaky files and failures", ""]
+ if not results:
+ return [*lines, "No test reports in this run (the python lane did not run)."]
+ for name in _ordered(results):
+ result = results[name]
+ baseline = load_baseline(baseline_dir / f"test-baseline-{name}.json")
+ new, known = split_failures(result.failures, baseline)
+ lines.append(
+ f"**{name}** ({result.shards} shard(s), {result.passed}/{result.tests} passed): "
+ f"{len(result.flaky)} flaky file(s), {len(new)} new failure(s), "
+ f"{len(known)} known failure(s)"
+ )
+ if result.flaky:
+ lines += ["", "Flaky (failed in a batch, passed alone - suspect test order):"]
+ lines += _bullets(sorted(result.flaky))
+ if new:
+ lines += ["", "New failures (not in the baseline):"]
+ lines += _bullets(new)
+ if known:
+ lines += ["", "<details><summary>Known (baselined) failures</summary>", ""]
+ lines += _bullets(known)
+ lines += ["", "</details>"]
+ lines.append("")
+ return lines
+
+
+# --------------------------------------------------------------------------- run vetting
+
+
+def vet_run(run: dict, repo: str) -> tuple[dict[str, str], str]:
+ """Accept only a finished CI run of this repository's main branch.
+
+ The issue job holds ``issues: write``; a manually supplied run id must never
+ let a pull-request run (possibly from a fork) feed it data.
+ """
+ head_repo = ((run.get("head_repository") or {}).get("full_name")) or ""
+ if head_repo != repo:
+ return {}, f"run belongs to {head_repo or 'an unknown repository'}, not {repo}"
+ if run.get("head_branch") != "main":
+ return {}, f"run is on branch {run.get('head_branch')!r}, not main"
+ if run.get("event") not in TRUSTED_EVENTS:
+ return {}, f"run was triggered by {run.get('event')!r}, not a main push or nightly"
+ if run.get("status") != "completed":
+ return {}, "run has not completed"
+ run_id = str(run.get("id", ""))
+ if not run_id.isdigit():
+ return {}, "run has no numeric id"
+ return {
+ "id": run_id,
+ "url": str(run.get("html_url", "")),
+ "at": str(run.get("run_started_at") or run.get("created_at") or ""),
+ "event": str(run.get("event")),
+ }, ""
+
+
+# --------------------------------------------------------------------------- tracking issue
+
+
+def parse_state(body: str | None) -> dict:
+ """Read the state block; a missing or hand-mangled block starts fresh."""
+ empty: dict = {"entries": {}, "runs": []}
+ match = _STATE.search(body or "")
+ if not match:
+ return empty
+ try:
+ state = json.loads(match.group(1))
+ except ValueError:
+ return empty
+ if not isinstance(state, dict):
+ return empty
+ entries = state.get("entries")
+ runs = state.get("runs")
+ clean: dict[str, dict] = {}
+ for key, entry in (entries if isinstance(entries, dict) else {}).items():
+ if not (isinstance(entry, dict) and entry.get("file") and entry.get("os")):
+ continue
+ count = entry.get("count", 0)
+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+ continue
+ clean[str(key)] = entry
+ return {
+ "entries": clean,
+ "runs": [str(r) for r in runs] if isinstance(runs, list) else [],
+ }
+
+
+def _parse_time(value: str) -> datetime | None:
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except (AttributeError, ValueError):
+ return None
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
+
+
+def merge_run(
+ state: dict,
+ results: dict[str, OsResult],
+ *,
+ run_id: str,
+ run_url: str,
+ run_at: str,
+ keep_days: int = KEEP_DAYS,
+) -> dict:
+ """Fold one CI run into the rolling state.
+
+ Count = number of distinct main/nightly runs in which the file was flaky on
+ that OS. A run already folded in (a re-run attempt, a manual backfill) adds
+ nothing. Rows not seen for ``keep_days`` drop out.
+ """
+ entries = {key: dict(entry) for key, entry in state.get("entries", {}).items()}
+ runs = list(state.get("runs", []))
+ if run_id not in runs:
+ for os_name, result in results.items():
+ for file in result.flaky:
+ key = f"{os_name}|{file}"
+ entry = entries.get(key) or {"file": file, "os": os_name, "count": 0}
+ entry["count"] = int(entry.get("count", 0) or 0) + 1
+ entry.setdefault("first_seen", run_at)
+ entry.update(last_seen=run_at, last_run_id=run_id, last_run_url=run_url)
+ entries[key] = entry
+ runs = [*runs, run_id][-MAX_RUNS_REMEMBERED:]
+ now = _parse_time(run_at) or datetime.now(UTC)
+ cutoff = now - timedelta(days=keep_days)
+ kept = {}
+ for key, entry in entries.items():
+ seen = _parse_time(str(entry.get("last_seen", "")))
+ if seen is None or seen >= cutoff:
+ kept[key] = entry
+ return {"entries": kept, "runs": runs, "updated": run_at, "updated_run": run_url}
+
+
+def render_issue(state: dict, keep_days: int = KEEP_DAYS) -> str:
+ # Most frequent first; ties by most recently seen, then by OS and file.
+ rows = sorted(state["entries"].values(), key=lambda e: (e["os"], e["file"]))
+ rows.sort(key=lambda e: str(e.get("last_seen", "")), reverse=True)
+ rows.sort(key=lambda e: int(e.get("count", 0) or 0), reverse=True)
+ lines = [
+ "Test files that failed inside their batch on `main` and then passed when the",
+ "parallel runner re-ran them alone (`scripts/ci/run_tests_parallel.py`). That",
+ "pattern almost always means test-order dependence: shared module state, a",
+ "leaked patch, a fixed port or a temp file another test also uses. The ratchet",
+ "does not block on them, so this table is where they stay visible.",
+ "",
+ "Updated automatically by `.github/workflows/flaky-tests.yml` after every",
+ f"push and nightly CI run on `main`. Rows not seen for {keep_days} days drop out;",
+ "fix a test, and its row ages out. Edits to this body are overwritten.",
+ "",
+ ]
+ if rows:
+ lines += ["| Test file | OS | Count | Last seen |", "| --- | --- | ---: | --- |"]
+ for entry in rows:
+ seen = str(entry.get("last_seen", ""))[:10] or "?"
+ url = str(entry.get("last_run_url", ""))
+ last = f"[{seen}]({url})" if url.startswith("https://") else seen
+ lines.append(
+ f"| {_code(str(entry['file']))} | {entry['os']} | {int(entry.get('count', 0))} "
+ f"| {last} |"
+ )
+ else:
+ lines.append(f"No flaky test files on `main` in the last {keep_days} days.")
+ updated_run = str(state.get("updated_run", ""))
+ if updated_run.startswith("https://"):
+ lines += ["", f"Last update: [{str(state.get('updated', ''))[:16]}]({updated_run})"]
+ # An HTML comment may not contain "--"; inside JSON strings the escape
+ # decodes back to the same text.
+ payload = json.dumps(state, sort_keys=True, separators=(",", ":")).replace("--", _DASH_DASH)
+ lines += ["", f"<!-- flaky-state: {payload} -->", ""]
+ return "\n".join(lines)
+
+
+def pick_issue(issues: list[dict]) -> dict | None:
+ """The tracking issue: exact title, an open one first, then the oldest."""
+ matches = [i for i in issues if isinstance(i, dict) and i.get("title") == ISSUE_TITLE]
+ if not matches:
+ return None
+ matches.sort(key=lambda i: (str(i.get("state", "")).upper() != "OPEN", int(i["number"])))
+ return matches[0]
+
+
+def plan_issue(issue: dict | None, body: str, has_rows: bool) -> str:
+ """create | edit | reopen | none."""
+ if issue is None:
+ return "create" if has_rows else "none"
+ if str(issue.get("state", "")).upper() != "OPEN":
+ return "reopen" if has_rows else "none"
+ return "edit" if issue.get("body") != body else "none"
+
+
+# --------------------------------------------------------------------------- commands
+
+
+def _output(values: dict[str, str]) -> None:
+ lines = [f"{key}={value}" for key, value in values.items()]
+ print("\n".join(lines))
+ target = os.environ.get("GITHUB_OUTPUT")
+ if target:
+ with open(target, "a", encoding="utf-8") as handle:
+ handle.write("\n".join(lines) + "\n")
+
+
+def _step_summary(lines: list[str]) -> None:
+ target = os.environ.get("GITHUB_STEP_SUMMARY")
+ if target:
+ with open(target, "a", encoding="utf-8") as handle:
+ handle.write("\n".join(lines) + "\n")
+
+
+def cmd_summary(args: argparse.Namespace) -> int:
+ lines = summary_lines(collect(args.reports), args.baseline_dir)
+ print("\n".join(lines))
+ _step_summary(lines)
+ return 0
+
+
+def cmd_vet_run(args: argparse.Namespace) -> int:
+ run = json.loads(args.run.read_text(encoding="utf-8-sig"))
+ values, reason = vet_run(run if isinstance(run, dict) else {}, args.repo)
+ if reason:
+ print(f"::error::refusing CI run: {reason}")
+ return 1
+ _output(values)
+ return 0
+
+
+def cmd_issue(args: argparse.Namespace) -> int:
+ if not args.run_id.isdigit():
+ print("::error::--run-id must be numeric")
+ return 1
+ issues = []
+ if args.issues.is_file():
+ issues = json.loads(args.issues.read_text(encoding="utf-8-sig"))
+ issue = pick_issue(issues if isinstance(issues, list) else [])
+ state = merge_run(
+ parse_state(issue.get("body") if issue else ""),
+ collect(args.reports),
+ run_id=args.run_id,
+ run_url=args.run_url,
+ run_at=args.run_at,
+ keep_days=args.keep_days,
+ )
+ body = render_issue(state, args.keep_days)
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ args.out.write_text(body, encoding="utf-8")
+ action = plan_issue(issue, body, bool(state["entries"]))
+ _output(
+ {
+ "action": action,
+ "number": str(issue["number"]) if issue else "",
+ "rows": str(len(state["entries"])),
+ }
+ )
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ sub = parser.add_subparsers(dest="cmd", required=True)
+ summary = sub.add_parser("summary")
+ summary.add_argument("--baseline-dir", type=Path, default=REPO_ROOT / "scripts" / "ci")
+ summary.add_argument("reports", nargs="*", type=Path)
+ vet = sub.add_parser("vet-run")
+ vet.add_argument("run", type=Path)
+ vet.add_argument("--repo", required=True)
+ issue = sub.add_parser("issue")
+ issue.add_argument("--issues", type=Path, required=True)
+ issue.add_argument("--out", type=Path, required=True)
+ issue.add_argument("--run-id", required=True)
+ issue.add_argument("--run-url", required=True)
+ issue.add_argument("--run-at", required=True)
+ issue.add_argument("--keep-days", type=int, default=KEEP_DAYS)
+ issue.add_argument("reports", nargs="*", type=Path)
+ args = parser.parse_args(argv)
+ for stream in (sys.stdout, sys.stderr):
+ if hasattr(stream, "reconfigure"):
+ stream.reconfigure(encoding="utf-8", errors="replace")
+ handler = {"summary": cmd_summary, "vet-run": cmd_vet_run, "issue": cmd_issue}[args.cmd]
+ return handler(args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/unit/ci/test_flaky_report.py b/tests/unit/ci/test_flaky_report.py
new file mode 100644
index 0000000000..32e7e6ccae
--- /dev/null
+++ b/tests/unit/ci/test_flaky_report.py
@@ -0,0 +1,340 @@
+"""Flaky-test aggregation: step summary, run vetting and the rolling tracking issue."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.ci import flaky_report
+
+REPO = "PersonalJarvis/PersonalJarvis"
+RUN_URL = "https://github.com/PersonalJarvis/PersonalJarvis/actions/runs/{}"
+
+
+def _report(tmp: Path, name: str, platform: str, *, flaky=(), failed=(), passed=10) -> Path:
+ path = tmp / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {
+ "shard": "1/1",
+ "platform": platform,
+ "counts": {"tests": passed + len(failed), "passed": passed, "failed": len(failed)},
+ "failed_ids": list(failed),
+ "flaky_files": list(flaky),
+ }
+ path.write_text(json.dumps(payload), encoding="utf-8")
+ return path
+
+
+def _baseline(tmp: Path, os_name: str, known: list[str]) -> None:
+ payload = {"known_failures": known}
+ (tmp / f"test-baseline-{os_name}.json").write_text(json.dumps(payload), encoding="utf-8")
+
+
+# --------------------------------------------------------------------------- collect
+
+
+def test_collect_groups_shards_by_os_and_normalizes_crashes(tmp_path):
+ _report(tmp_path, "tests-linux-1/test-report-linux-1.json", "linux", flaky=["tests/a.py"])
+ _report(
+ tmp_path,
+ "tests-linux-2/test-report-linux-2.json",
+ "linux",
+ failed=["tests/b.py::<timeout 300s>"],
+ )
+ _report(tmp_path, "test-report-windows-1.json", "win32", flaky=["tests/a.py"])
+ results = flaky_report.collect([tmp_path])
+ assert set(results) == {"linux", "windows"}
+ assert results["linux"].shards == 2
+ assert results["linux"].flaky == {"tests/a.py"}
+ assert results["linux"].failures == {"tests/b.py::<crash>"}
+ assert results["windows"].flaky == {"tests/a.py"}
+
+
+def test_os_falls_back_to_the_report_file_name():
+ assert flaky_report.os_of({}, Path("test-report-macos-3.json")) == "macos"
+ assert flaky_report.os_of({"platform": "darwin"}) == "macos"
+ assert flaky_report.os_of({}) == "unknown"
+
+
+def test_unreadable_report_is_skipped_not_fatal(tmp_path, capsys):
+ (tmp_path / "test-report-linux-1.json").write_text("{not json", encoding="utf-8")
+ _report(tmp_path, "test-report-linux-2.json", "linux", flaky=["tests/a.py"])
+ results = flaky_report.collect([tmp_path, tmp_path / "missing"])
+ assert results["linux"].shards == 1
+ assert "unreadable test report" in capsys.readouterr().out
+
+
+# --------------------------------------------------------------------------- summary
+
+
+def test_summary_splits_new_from_known_per_os_baseline(tmp_path):
+ _baseline(tmp_path, "linux", ["tests/k.py::test_known", "tests/c.py::<timeout 1s>"])
+ _report(
+ tmp_path,
+ "r/test-report-linux-1.json",
+ "linux",
+ flaky=["tests/order.py"],
+ failed=["tests/k.py::test_known", "tests/n.py::test_new", "tests/c.py::<timeout 9s>"],
+ )
+ results = flaky_report.collect([tmp_path / "r"])
+ new, known = flaky_report.split_failures(
+ results["linux"].failures,
+ flaky_report.load_baseline(tmp_path / "test-baseline-linux.json"),
+ )
+ assert new == ["tests/n.py::test_new"]
+ assert known == ["tests/c.py::<crash>", "tests/k.py::test_known"]
+ text = "\n".join(flaky_report.summary_lines(results, tmp_path))
+ assert "1 flaky file(s), 1 new failure(s), 2 known failure(s)" in text
+ assert "`tests/order.py`" in text
+ assert "`tests/n.py::test_new`" in text
+
+
+def test_summary_without_reports_says_so(tmp_path):
+ lines = flaky_report.summary_lines({}, tmp_path)
+ assert "No test reports" in lines[-1]
+
+
+def test_summary_command_writes_the_step_summary(tmp_path, monkeypatch):
+ target = tmp_path / "summary.md"
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(target))
+ _report(tmp_path, "r/test-report-linux-1.json", "linux", flaky=["tests/x.py"])
+ args = ["summary", "--baseline-dir", str(tmp_path), str(tmp_path / "r")]
+ assert flaky_report.main(args) == 0
+ assert "`tests/x.py`" in target.read_text(encoding="utf-8")
+
+
+def test_long_lists_are_capped():
+ lines = flaky_report._bullets([f"t{i}" for i in range(flaky_report.MAX_LISTED + 5)])
+ assert len(lines) == flaky_report.MAX_LISTED + 1
+ assert lines[-1] == "- ... and 5 more"
+
+
+# --------------------------------------------------------------------------- run vetting
+
+
+def _run(**overrides) -> dict:
+ run = {
+ "id": 42,
+ "html_url": RUN_URL.format(42),
+ "run_started_at": "2026-10-07T02:17:00Z",
+ "event": "schedule",
+ "status": "completed",
+ "head_branch": "main",
+ "head_repository": {"full_name": REPO},
+ }
+ run.update(overrides)
+ return run
+
+
+def test_vet_run_accepts_a_finished_main_run():
+ values, reason = flaky_report.vet_run(_run(), REPO)
+ assert reason == ""
+ assert values["id"] == "42"
+ assert values["at"] == "2026-10-07T02:17:00Z"
+
+
+ ("overrides", "why"),
+ [
+ ({"event": "pull_request"}, "triggered by"),
+ ({"head_repository": {"full_name": "fork/PersonalJarvis"}}, "belongs to"),
+ ({"head_branch": "feature"}, "not main"),
+ ({"status": "in_progress"}, "not completed"),
+ ({"id": "1; rm -rf /"}, "numeric id"),
+ ],
+)
+def test_vet_run_refuses_anything_but_trusted_main_runs(overrides, why):
+ values, reason = flaky_report.vet_run(_run(**overrides), REPO)
+ assert values == {}
+ assert why in reason
+
+
+# --------------------------------------------------------------------------- tracking issue
+
+
+def _results(**flaky_by_os) -> dict[str, flaky_report.OsResult]:
+ return {
+ os_name: flaky_report.OsResult(flaky=set(files)) for os_name, files in flaky_by_os.items()
+ }
+
+
+def test_merge_counts_runs_per_file_and_os_and_is_idempotent():
+ state = {"entries": {}, "runs": []}
+ state = flaky_report.merge_run(
+ state,
+ _results(linux=["tests/a.py"], windows=["tests/a.py"]),
+ run_id="1",
+ run_url=RUN_URL.format(1),
+ run_at="2026-10-01T02:00:00Z",
+ )
+ again = flaky_report.merge_run(
+ state,
+ _results(linux=["tests/a.py"]),
+ run_id="1",
+ run_url=RUN_URL.format(1),
+ run_at="2026-10-01T02:00:00Z",
+ )
+ assert again["entries"]["linux|tests/a.py"]["count"] == 1
+ later = flaky_report.merge_run(
+ again,
+ _results(linux=["tests/a.py"]),
+ run_id="2",
+ run_url=RUN_URL.format(2),
+ run_at="2026-10-02T02:00:00Z",
+ )
+ entry = later["entries"]["linux|tests/a.py"]
+ assert entry["count"] == 2
+ assert entry["first_seen"] == "2026-10-01T02:00:00Z"
+ assert entry["last_seen"] == "2026-10-02T02:00:00Z"
+ assert entry["last_run_url"] == RUN_URL.format(2)
+ assert later["entries"]["windows|tests/a.py"]["count"] == 1
+ assert later["runs"] == ["1", "2"]
+
+
+def test_rows_not_seen_within_the_window_age_out():
+ state = flaky_report.merge_run(
+ {"entries": {}, "runs": []},
+ _results(linux=["tests/old.py"]),
+ run_id="1",
+ run_url=RUN_URL.format(1),
+ run_at="2026-08-01T00:00:00Z",
+ )
+ state = flaky_report.merge_run(
+ state,
+ _results(linux=["tests/new.py"]),
+ run_id="2",
+ run_url=RUN_URL.format(2),
+ run_at="2026-10-01T00:00:00Z",
+ keep_days=30,
+ )
+ assert set(state["entries"]) == {"linux|tests/new.py"}
+
+
+def test_remembered_runs_are_bounded():
+ state = {"entries": {}, "runs": [str(i) for i in range(flaky_report.MAX_RUNS_REMEMBERED)]}
+ state = flaky_report.merge_run(
+ state, {}, run_id="new", run_url="", run_at="2026-10-01T00:00:00Z"
+ )
+ assert len(state["runs"]) == flaky_report.MAX_RUNS_REMEMBERED
+ assert state["runs"][-1] == "new"
+
+
+def test_rendered_body_round_trips_its_state_and_sorts_by_count():
+ state = flaky_report.merge_run(
+ {"entries": {}, "runs": []},
+ _results(linux=["tests/once.py", "tests/twice.py"]),
+ run_id="1",
+ run_url=RUN_URL.format(1),
+ run_at="2026-10-01T00:00:00Z",
+ )
+ state = flaky_report.merge_run(
+ state,
+ _results(linux=["tests/twice.py", "tests/a--b|c.py"]),
+ run_id="2",
+ run_url=RUN_URL.format(2),
+ run_at="2026-10-02T00:00:00Z",
+ )
+ body = flaky_report.render_issue(state)
+ table = [line for line in body.splitlines() if line.startswith("| `")]
+ assert table[0].startswith("| `tests/twice.py` | linux | 2 |")
+ assert "`tests/a--b\\|c.py`" in body
+ comment = body[body.index("<!--") + 4 :]
+ assert "--" not in comment[: comment.index("-->")]
+ parsed = flaky_report.parse_state(body)
+ assert parsed["entries"] == state["entries"]
+ assert parsed["runs"] == ["1", "2"]
+
+
+def test_empty_state_renders_a_quiet_body():
+ body = flaky_report.render_issue({"entries": {}, "runs": []})
+ assert "No flaky test files" in body
+ assert "| Test file |" not in body
+
+
+ "body",
+ [
+ None,
+ "",
+ "edited by hand",
+ "<!-- flaky-state: {broken -->",
+ '<!-- flaky-state: {"entries": [], "runs": "x"} -->',
+ '<!-- flaky-state: {"entries": {"k": {"file": "a", "os": "linux", "count": "x"}}} -->',
+ ],
+)
+def test_missing_or_mangled_state_starts_fresh(body):
+ assert flaky_report.parse_state(body) == {"entries": {}, "runs": []}
+
+
+def test_pick_issue_prefers_the_open_exact_title():
+ issues = [
+ {"number": 5, "title": "Flaky tests on main (old)", "state": "OPEN"},
+ {"number": 7, "title": flaky_report.ISSUE_TITLE, "state": "CLOSED"},
+ {"number": 9, "title": flaky_report.ISSUE_TITLE, "state": "OPEN"},
+ ]
+ assert flaky_report.pick_issue(issues)["number"] == 9
+ assert flaky_report.pick_issue(issues[:2])["number"] == 7
+ assert flaky_report.pick_issue([]) is None
+
+
+ ("issue", "has_rows", "expected"),
+ [
+ (None, True, "create"),
+ (None, False, "none"),
+ ({"number": 1, "state": "CLOSED", "body": ""}, True, "reopen"),
+ ({"number": 1, "state": "CLOSED", "body": ""}, False, "none"),
+ ({"number": 1, "state": "OPEN", "body": "old"}, False, "edit"),
+ ({"number": 1, "state": "OPEN", "body": "same"}, True, "none"),
+ ],
+)
+def test_plan_issue(issue, has_rows, expected):
+ assert flaky_report.plan_issue(issue, "same", has_rows) == expected
+
+
+def test_issue_command_end_to_end(tmp_path, monkeypatch):
+ output = tmp_path / "github_output"
+ monkeypatch.setenv("GITHUB_OUTPUT", str(output))
+ _report(tmp_path, "r/test-report-linux-1.json", "linux", flaky=["tests/a.py"])
+ previous = flaky_report.render_issue(
+ flaky_report.merge_run(
+ {"entries": {}, "runs": []},
+ _results(linux=["tests/a.py"]),
+ run_id="1",
+ run_url=RUN_URL.format(1),
+ run_at="2026-10-06T02:00:00Z",
+ )
+ )
+ issues = tmp_path / "issues.json"
+ issues.write_text(
+ json.dumps(
+ [{"number": 12, "title": flaky_report.ISSUE_TITLE, "state": "OPEN", "body": previous}]
+ ),
+ encoding="utf-8",
+ )
+ body = tmp_path / "out" / "body.md"
+ args = [
+ "issue",
+ "--issues",
+ str(issues),
+ "--out",
+ str(body),
+ "--run-id",
+ "2",
+ "--run-url",
+ RUN_URL.format(2),
+ "--run-at",
+ "2026-10-07T02:00:00Z",
+ str(tmp_path / "r"),
+ ]
+ assert flaky_report.main(args) == 0
+ assert "action=edit\nnumber=12\nrows=1\n" == output.read_text(encoding="utf-8")
+ assert "| `tests/a.py` | linux | 2 |" in body.read_text(encoding="utf-8")
+
+
+def test_issue_command_rejects_a_non_numeric_run_id(tmp_path):
+ args = ["issue", "--issues", str(tmp_path / "none.json"), "--out", str(tmp_path / "b.md")]
+ args += ["--run-id", "x", "--run-url", "u", "--run-at", "2026-10-07T00:00:00Z"]
+ assert flaky_report.main(args) == 1