Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci: per-file test times, weekly CI speed report, faster ci-gate finish (CUA-3558 #12) (#6604)

Francesco Bonaccica2fdb4fd1e4
diff --git a/.github/actions/ci-timings-upload/action.yml b/.github/actions/ci-timings-upload/action.yml
new file mode 100644
index 0000000000..a09589368a
--- /dev/null
+++ b/.github/actions/ci-timings-upload/action.yml
@@ -0,0 +1,15 @@
+name: Upload test timings
+description: "Uploads the per-file test times that .github/scripts/flaky/timings.py wrote, for the weekly CI speed report (ci_weekly_report.py). Run with if: always()."
+inputs:
+ suite:
+ description: "The suite name given to flaky.py run or timings.py (unique per job, matrix legs included)."
+ required: true
+runs:
+ using: composite
+ steps:
+ - uses: actions/upload-artifact@v4
+ with:
+ name: timings-${{ inputs.suite }}
+ path: ${{ runner.temp }}/ci-timings/
+ if-no-files-found: ignore
+ retention-days: 10
diff --git a/.github/actions/flaky-upload/action.yml b/.github/actions/flaky-upload/action.yml
index 047898e14b..732adeecdf 100644
--- a/.github/actions/flaky-upload/action.yml
+++ b/.github/actions/flaky-upload/action.yml
@@ -13,3 +13,7 @@ runs:
path: ${{ runner.temp }}/flaky-results/
if-no-files-found: ignore
retention-days: 10
+ # Per-file test times written by flaky.py run (timings.py), for the weekly CI speed report.
+ - uses: ./.github/actions/ci-timings-upload
+ with:
+ suite: ${{ inputs.suite }}
diff --git a/.github/flaky/vitest-reporter.mjs b/.github/flaky/vitest-reporter.mjs
index 777cf4a3f7..360b5c9cbb 100644
--- a/.github/flaky/vitest-reporter.mjs
+++ b/.github/flaky/vitest-reporter.mjs
@@ -1,15 +1,23 @@
// Vitest reporter for CI flaky-test handling (.github/scripts/flaky/flaky.py).
// Writes one row per test to $FLAKY_RESULTS_OUT: "flaky" when the test passed
// only on a retry, "failed" when it failed after its retries. Does nothing
-// when FLAKY_RESULTS_OUT is unset, so local runs are unchanged.
+// when FLAKY_RESULTS_OUT is unset, so local runs are unchanged. Also writes
+// per-file test times next to it (<out>.timings.json), read by
+// .github/scripts/flaky/timings.py through flaky.py.
import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, relative } from "node:path";
export default class FlakyReporter {
rows = [];
+ files = new Map();
onTestCaseResult(testCase) {
const result = testCase.result();
+ const where = relative(process.cwd(), testCase.module.moduleId);
+ const seen = this.files.get(where) ?? { file: where, seconds: 0, tests: 0 };
+ seen.seconds += (testCase.diagnostic()?.duration ?? 0) / 1000;
+ seen.tests += 1;
+ this.files.set(where, seen);
if (result.state !== "passed" && result.state !== "failed") return;
const retried = (testCase.diagnostic()?.retryCount ?? 0) > 0;
const status = result.state === "failed" ? "failed" : retried ? "flaky" : "passed";
@@ -27,5 +35,9 @@ export default class FlakyReporter {
if (!out) return;
mkdirSync(dirname(out), { recursive: true });
writeFileSync(out, JSON.stringify(this.rows));
+ const timings = [...this.files.values()]
+ .map((f) => ({ ...f, seconds: Math.round(f.seconds * 1000) / 1000 }))
+ .sort((a, b) => b.seconds - a.seconds);
+ writeFileSync(`${out}.timings.json`, JSON.stringify(timings));
}
}
diff --git a/.github/scripts/ci_weekly_report.py b/.github/scripts/ci_weekly_report.py
new file mode 100644
index 0000000000..843718396c
--- /dev/null
+++ b/.github/scripts/ci_weekly_report.py
@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""Weekly CI speed report: opens or updates ONE issue, "CI speed this week" (CUA-3558, cut #12).
+
+It measures what the CI audit of 5 Nov 2026 had to estimate by hand, so the next
+round of cuts can be ranked from numbers:
+
+ * push-to-green: from a PR's last push (its first check starting) to a green
+ `ci-gate`, for PRs merged in the last 7 days;
+ * the gate tail: how long `ci-gate` kept running after the last other check
+ finished, split into "waiting for the local-ci sign-off" and "other";
+ * merge wait: from a green `ci-gate` to the merge;
+ * release-PR wait: release-please PRs, opened to merged and green to merged;
+ * CD time: the median and p90 duration of every `CD:` workflow run;
+ * the slowest test files, from the `timings-*` artifacts that CI jobs upload
+ (.github/scripts/flaky/timings.py).
+
+It follows the weekly flaky-test issue (.github/scripts/flaky/flaky.py digest):
+one issue, found by a marker, rewritten each Monday by .github/workflows/ci-weekly-report.yml.
+
+ ci_weekly_report.py [--days 7] [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import io
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+import zipfile
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional
+
+TITLE = "CI speed this week"
+MARKER = "<!-- ci-speed-weekly -->"
+LABEL = "ci-speed"
+GATE = "ci-gate"
+SIGNOFF = "local-ci"
+RERUN_SLACK_MIN = 2.0 # a gate that starts this long after the last check is a re-run, not a wait
+
+PR_QUERY = """
+query($q: String!, $after: String) {
+ search(query: $q, type: ISSUE, first: 25, after: $after) {
+ pageInfo { hasNextPage endCursor }
+ nodes { ... on PullRequest {
+ number createdAt mergedAt headRefName
+ commits(last: 1) { nodes { commit { oid status { contexts { context createdAt } } } } }
+ } }
+ }
+}
+"""
+
+# Workflow runs the gate never waits for (see ci-gate.yml); they must not count as "the last check".
+GATE_IGNORES = ("claude-auto-fix", "ci-release-reminder", "review-cua-perception-candidates", "cla.yml")
+
+
+def gh(*args: str, check: bool = True) -> str:
+ for attempt in range(3): # the API sometimes answers 5xx or an empty body
+ done = subprocess.run(["gh", *args], capture_output=True, text=True)
+ if done.returncode == 0:
+ break
+ time.sleep(5 * (attempt + 1))
+ if done.returncode != 0 and check:
+ raise RuntimeError(f"gh {' '.join(args[:3])} failed: {done.stderr.strip()[:300]}")
+ return done.stdout
+
+
+def ts(value: Optional[str]) -> Optional[dt.datetime]:
+ return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None
+
+
+def minutes(a: dt.datetime, b: dt.datetime) -> float:
+ return (b - a).total_seconds() / 60
+
+
+def pct(values: List[float], p: float) -> float:
+ ordered = sorted(values)
+ return ordered[min(len(ordered) - 1, max(0, int(round(p * len(ordered))) - 1))]
+
+
+def summarize(values: List[float]) -> str:
+ if not values:
+ return "no data"
+ return f"median {statistics.median(values):.1f} min, p90 {pct(values, 0.9):.1f} min (n={len(values)})"
+
+
+# --- pull requests -----------------------------------------------------------
+
+
+def pr_metrics(pr: dict, runs: List[dict]) -> Optional[dict]:
+ """The timings of one merged PR. `runs` are the workflow runs of its last commit
+ ({name, path, started, updated, conclusion}). None when it has no finished gate.
+
+ The gate tail is a lower bound: a run the gate skipped by scope that finished late still counts as
+ "the last check", which can only shorten it.
+ """
+ done = [r for r in runs if r.get("started") and r.get("updated") and r.get("conclusion")]
+ gates = [r for r in done if r["name"] == "CI: gate" and r["conclusion"] == "success"]
+ if not gates:
+ return None
+ gate = min(gates, key=lambda r: ts(r["updated"])) # the first green; later ones are label or edit re-runs
+ gate_start, gate_end = ts(gate["started"]), ts(gate["updated"])
+ others = [r for r in done if r["name"] != "CI: gate" and not any(i in (r.get("path") or "") for i in GATE_IGNORES)
+ and r["conclusion"] != "cancelled"]
+ if not others:
+ return None
+ first = min(ts(r["started"]) for r in others)
+ ended = [ts(r["updated"]) for r in others if ts(r["updated"]) <= gate_end]
+ last = max(ended) if ended else gate_end
+ rerun = minutes(last, gate_start) > RERUN_SLACK_MIN # label or edit event, after everything finished
+ commit = ((pr.get("commits") or {}).get("nodes") or [{}])[0].get("commit") or {}
+ contexts = ((commit.get("status") or {}).get("contexts")) or []
+ out = {
+ "number": pr["number"],
+ "release": (pr.get("headRefName") or "").startswith("release-please--"),
+ "push_to_green": minutes(first, last if rerun else gate_end),
+ "merge_wait": max(0.0, minutes(gate_end, ts(pr["mergedAt"]))),
+ "opened_to_merged": minutes(ts(pr["createdAt"]), ts(pr["mergedAt"])),
+ "tail": None,
+ "tail_signoff": None,
+ }
+ if not rerun:
+ tail = max(0.0, minutes(last, gate_end))
+ signoff = next((ts(c["createdAt"]) for c in contexts if c.get("context") == SIGNOFF and c.get("createdAt")), None)
+ out["tail"] = tail
+ out["tail_signoff"] = bool(signoff and last <= signoff <= gate_end and tail > 3)
+ return out
+
+
+def fetch_runs(oid: str) -> List[dict]:
+ repo = os.environ["GITHUB_REPOSITORY"]
+ out = gh("api", f"repos/{repo}/actions/runs?head_sha={oid}&per_page=100", "--jq",
+ '.workflow_runs[] | select(.event == "pull_request" or .event == "pull_request_target")'
+ ' | {name, path, started: .run_started_at, updated: .updated_at, conclusion}')
+ return [json.loads(line) for line in out.splitlines() if line.strip()]
+
+
+def fetch_prs(days: int) -> List[dict]:
+ since = (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days)).strftime("%Y-%m-%d")
+ repo = os.environ["GITHUB_REPOSITORY"]
+ q = f"repo:{repo} is:pr is:merged base:main merged:>={since}"
+ out, after = [], None
+ for _ in range(30):
+ args = ["api", "graphql", "-f", f"query={PR_QUERY}", "-f", f"q={q}"]
+ if after:
+ args += ["-f", f"after={after}"]
+ data = json.loads(gh(*args))["data"]["search"]
+ out += [n for n in data["nodes"] if n]
+ if not data["pageInfo"]["hasNextPage"]:
+ break
+ after = data["pageInfo"]["endCursor"]
+ return out
+
+
+# --- CD workflows ------------------------------------------------------------
+
+
+def cd_durations(days: int) -> Dict[str, dict]:
+ repo = os.environ["GITHUB_REPOSITORY"]
+ since = (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days)).strftime("%Y-%m-%d")
+ lines = gh("api", f"repos/{repo}/actions/workflows?per_page=100", "--paginate", "--jq",
+ '.workflows[] | select(.name | startswith("CD:")) | {id, name}')
+ workflows = [json.loads(line) for line in lines.splitlines() if line.strip()]
+ out: Dict[str, dict] = {}
+ for wf in workflows:
+ runs = json.loads(gh("api", f"repos/{repo}/actions/workflows/{wf['id']}/runs?status=completed"
+ f"&created=>={since}&per_page=100", "--jq", "[.workflow_runs[] | {c: .conclusion, s: .run_started_at, u: .updated_at}]"))
+ durs = [minutes(ts(r["s"]), ts(r["u"])) for r in runs if r["c"] in ("success", "failure") and r["s"]]
+ if durs:
+ out[wf["name"]] = {"runs": len(durs), "fail": sum(1 for r in runs if r["c"] == "failure"), "durations": durs}
+ return out
+
+
+# --- test timings --------------------------------------------------------------
+
+
+def timing_workflows() -> List[str]:
+ """Workflow files that upload `timings-*` artifacts (directly or through flaky-upload)."""
+ root = Path(__file__).resolve().parents[1] / "workflows"
+ return sorted(p.name for p in root.glob("*.yml") if any(
+ needle in p.read_text() for needle in ("ci-timings-upload", "flaky-upload")))
+
+
+def fetch_timings(days: int, runs_per_workflow: int = 10) -> List[dict]:
+ """The per-file timings of the latest main-branch runs of each workflow that uploads them.
+
+ The repository-wide artifact listing (`actions/artifacts`) answers 500 on a repository this
+ size, so this goes through each workflow's runs instead.
+ """
+ repo = os.environ["GITHUB_REPOSITORY"]
+ since = (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days)).strftime("%Y-%m-%d")
+ reports: List[dict] = []
+ for wf in timing_workflows():
+ try:
+ run_ids = gh("api", f"repos/{repo}/actions/workflows/{wf}/runs?branch=main&event=push&status=completed"
+ f"&created=>={since}&per_page={runs_per_workflow}", "--jq", ".workflow_runs[].id").split()
+ for run_id in run_ids:
+ arts = gh("api", f"repos/{repo}/actions/runs/{run_id}/artifacts?per_page=100", "--jq",
+ '.artifacts[] | select((.name | startswith("timings-")) and (.expired | not)) | .id').split()
+ for art in arts:
+ done = subprocess.run(["gh", "api", f"repos/{repo}/actions/artifacts/{art}/zip"], capture_output=True)
+ if done.returncode != 0:
+ continue
+ with zipfile.ZipFile(io.BytesIO(done.stdout)) as z:
+ for name in z.namelist():
+ if name.endswith(".json"):
+ try:
+ reports.append(json.loads(z.read(name)))
+ except ValueError:
+ pass
+ except RuntimeError as exc: # one workflow's failure must not lose the whole report
+ print(f"warning: no timings from {wf}: {exc}", file=sys.stderr)
+ return reports
+
+
+def slowest_files(reports: Iterable[dict], top: int = 20) -> List[dict]:
+ seen: Dict[tuple, List[float]] = defaultdict(list)
+ for r in reports:
+ for f in r.get("files", []):
+ seen[(r.get("suite", ""), f["file"])].append(float(f["seconds"]))
+ rows = [{"suite": s, "file": f, "median": statistics.median(v), "max": max(v), "runs": len(v)}
+ for (s, f), v in seen.items()]
+ return sorted(rows, key=lambda r: -r["median"])[:top]
+
+
+# --- report -------------------------------------------------------------------
+
+
+def build_report(metrics: List[dict], cd: Dict[str, dict], timings: List[dict], since: str) -> str:
+ normal = [m for m in metrics if not m["release"]]
+ release = [m for m in metrics if m["release"]]
+ tails = [m["tail"] for m in normal if m["tail"] is not None]
+ signoff_tails = [m["tail"] for m in normal if m["tail"] is not None and m["tail_signoff"]]
+ other_tails = [m["tail"] for m in normal if m["tail"] is not None and not m["tail_signoff"]]
+ out = [MARKER, f"Measured over the last 7 days (since {since}). Rewritten every Monday by `ci-weekly-report.yml`; "
+ "numbers are for merged PRs into `main` and from the last commit's checks.", ""]
+ out += ["## Pull requests", "",
+ "| Measure | Result |", "|---|---|",
+ f"| Push to green (first check starts to green `ci-gate`) | {summarize([m['push_to_green'] for m in normal])} |",
+ f"| Green to merged | {summarize([m['merge_wait'] for m in normal])} |",
+ f"| `ci-gate` after the last other check, all | {summarize(tails)} |",
+ f"| ...of which waiting for the `local-ci` sign-off | {summarize(signoff_tails)} |",
+ f"| ...of which other (poll interval, flake reruns) | {summarize(other_tails)} |", ""]
+ out += ["## Release PRs (release-please)", "",
+ "| Measure | Result |", "|---|---|",
+ f"| Opened to merged | {summarize([m['opened_to_merged'] for m in release])} |",
+ f"| Green to merged (the wait for a human) | {summarize([m['merge_wait'] for m in release])} |", ""]
+ out += ["## CD time", "", "| Workflow | Runs | Failed | Median | p90 |", "|---|---:|---:|---:|---:|"]
+ for name, d in sorted(cd.items(), key=lambda kv: -statistics.median(kv[1]["durations"])):
+ out.append(f"| {name} | {d['runs']} | {d['fail']} | {statistics.median(d['durations']):.0f} min | {pct(d['durations'], 0.9):.0f} min |")
+ if not cd:
+ out.append("| no CD runs this week | | | | |")
+ out += ["", "## Slowest test files", ""]
+ rows = slowest_files(timings)
+ if rows:
+ out += ["| Suite | File | Median | Max | Runs |", "|---|---|---:|---:|---:|"]
+ out += [f"| {r['suite']} | `{r['file']}` | {r['median']:.0f} s | {r['max']:.0f} s | {r['runs']} |" for r in rows]
+ else:
+ out.append("No `timings-*` artifacts yet (CI jobs upload them for 10 days).")
+ return "\n".join(out) + "\n"
+
+
+def collect_metrics(prs: List[dict]) -> List[dict]:
+ out = []
+ for pr in prs:
+ commit = ((pr.get("commits") or {}).get("nodes") or [{}])[0].get("commit") or {}
+ if not commit.get("oid"):
+ continue
+ try:
+ m = pr_metrics(pr, fetch_runs(commit["oid"]))
+ except RuntimeError as exc:
+ print(f"warning: PR #{pr['number']}: {exc}", file=sys.stderr)
+ continue
+ if m:
+ out.append(m)
+ return out
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--days", type=int, default=7)
+ p.add_argument("--dry-run", action="store_true")
+ args = p.parse_args(argv)
+ repo = os.environ["GITHUB_REPOSITORY"]
+ since = (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=args.days)).strftime("%Y-%m-%d")
+ body = build_report(collect_metrics(fetch_prs(args.days)), cd_durations(args.days), fetch_timings(args.days), since)
+ print(body)
+ if args.dry_run:
+ return 0
+ gh("label", "create", LABEL, "--repo", repo, "--color", "0E8A16", "--description", "Weekly CI speed report", check=False)
+ existing = json.loads(gh("issue", "list", "--repo", repo, "--label", LABEL, "--state", "open", "--limit", "50",
+ "--json", "number,body") or "[]")
+ issue = next((i for i in existing if MARKER in (i.get("body") or "")), None)
+ if issue:
+ gh("issue", "edit", str(issue["number"]), "--repo", repo, "--body", body)
+ print(f"updated #{issue['number']}")
+ else:
+ print(gh("issue", "create", "--repo", repo, "--label", LABEL, "--title", TITLE, "--body", body).strip())
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/flaky/flaky.py b/.github/scripts/flaky/flaky.py
index 803380a05a..a13f0dc485 100755
--- a/.github/scripts/flaky/flaky.py
+++ b/.github/scripts/flaky/flaky.py
@@ -318,6 +318,7 @@ def cmd_run(args: argparse.Namespace) -> int:
write_summary(args.suite, results, flaky, quarantined, real, live)
write_artifact(args.suite, flaky, quarantined)
+ record_timings(args.suite, args.format, paths, lines)
if code == 0:
return 0
@@ -353,6 +354,29 @@ def write_summary(suite: str, results: Results, flaky: Results, quarantined: Res
fh.write("\n".join(out) + "\n")
+def record_timings(suite: str, fmt: str, paths: List[str], lines: List[str]) -> None:
+ """Per-file test times (timings.py). Measurement only: never changes the verdict."""
+ try:
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ import timings
+
+ if fmt == "junit":
+ files = timings.junit_files(paths)
+ elif fmt == "log":
+ files = timings.swift_log_suites(lines)
+ elif fmt == "vitest":
+ files = []
+ for path in paths:
+ side = Path(path + ".timings.json")
+ if side.is_file():
+ files += json.loads(side.read_text())
+ else:
+ return
+ timings.write(suite, files)
+ except Exception as exc: # noqa: BLE001
+ print(f"::warning title=Test timings::{suite}: {exc}")
+
+
def write_artifact(suite: str, flaky: Results, quarantined: Results) -> None:
base = os.environ.get("RUNNER_TEMP")
if not base or not (flaky or quarantined):
diff --git a/.github/scripts/flaky/timings.py b/.github/scripts/flaky/timings.py
new file mode 100644
index 0000000000..6a4c578683
--- /dev/null
+++ b/.github/scripts/flaky/timings.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Per-file test times for CI (CUA-3558, cut #12).
+
+The CI logs did not say which test files are slow, so the next round of cuts
+could not be ranked. This reads a runner's own output and writes, per test
+file (or suite), the seconds it took:
+
+ * JUnit XML: pytest (`--junitxml`) and cargo nextest. A file is the `file`
+ attribute, else the dotted `classname` without its test class.
+ * XCTest logs (`swift test`, `xcodebuild test`): one row per test suite,
+ from "Test Suite 'X' passed ... Executed N tests ... in S (T) seconds".
+ * vitest: the reporter in .github/flaky/vitest-reporter.mjs writes the rows.
+
+Output: a "Slowest test files" table in the job summary and
+`$RUNNER_TEMP/ci-timings/<suite>.json`, which `.github/actions/ci-timings-upload`
+uploads (artifact `timings-<suite>`); `ci_weekly_report.py` ranks them weekly.
+
+ timings.py junit --suite NAME FILE... pytest / nextest JUnit files
+ timings.py swift-log --suite NAME LOGFILE an XCTest console log
+ flaky.py run calls the same functions for the suites it wraps.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, Iterable, List
+
+TOP = 15
+SUITE_RE = re.compile(r"Test Suite '([^']+)' (?:passed|failed) at ")
+EXECUTED_RE = re.compile(r"Executed (\d+) tests?, with \d+ failures?.* in ([0-9.]+) \([0-9.]+\) seconds")
+
+
+def _file_of(case: ET.Element) -> str:
+ f = case.get("file")
+ if f:
+ return f
+ cls = case.get("classname", "") or "(no class)"
+ parts = cls.split(".")
+ # pytest: tests.test_x.TestThing -> tests.test_x
+ while len(parts) > 1 and parts[-1][:1].isupper():
+ parts.pop()
+ return ".".join(parts)
+
+
+def junit_files(paths: Iterable[str]) -> List[dict]:
+ secs: Dict[str, float] = defaultdict(float)
+ tests: Dict[str, int] = defaultdict(int)
+ for path in paths:
+ for case in ET.parse(path).getroot().iter("testcase"):
+ key = _file_of(case)
+ secs[key] += float(case.get("time") or 0)
+ tests[key] += 1
+ return rows(secs, tests)
+
+
+def swift_log_suites(lines: Iterable[str]) -> List[dict]:
+ secs: Dict[str, float] = defaultdict(float)
+ tests: Dict[str, int] = defaultdict(int)
+ suite = None
+ for line in lines:
+ m = SUITE_RE.search(line)
+ if m:
+ suite = m.group(1)
+ continue
+ m = EXECUTED_RE.search(line)
+ if m and suite:
+ # Skip the bundle and selection totals; keep one row per test class.
+ if not suite.endswith((".xctest", "tests")) and suite not in ("All tests", "Selected tests"):
+ secs[suite] += float(m.group(2))
+ tests[suite] += int(m.group(1))
+ suite = None
+ return rows(secs, tests)
+
+
+def rows(secs: Dict[str, float], tests: Dict[str, int]) -> List[dict]:
+ return [{"file": k, "seconds": round(v, 3), "tests": tests.get(k, 0)}
+ for k, v in sorted(secs.items(), key=lambda kv: -kv[1])]
+
+
+def write(suite: str, files: List[dict]) -> None:
+ """Job summary table and the JSON that the weekly report reads."""
+ if not files:
+ return
+ summary = os.environ.get("GITHUB_STEP_SUMMARY")
+ if summary:
+ out = [f"### Slowest test files: {suite}", "", "| File | Seconds | Tests |", "|---|---:|---:|"]
+ out += [f"| `{f['file'].replace('|', '/')}` | {f['seconds']:.1f} | {f['tests']} |" for f in files[:TOP]]
+ total = sum(f["seconds"] for f in files)
+ out += ["", f"{len(files)} files, {total:.0f} s of test time in total."]
+ with open(summary, "a") as fh:
+ fh.write("\n".join(out) + "\n")
+ base = os.environ.get("RUNNER_TEMP")
+ if base:
+ out_dir = Path(base) / "ci-timings"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ meta = {k: os.environ.get(e, "") for k, e in (
+ ("repo", "GITHUB_REPOSITORY"), ("run_id", "GITHUB_RUN_ID"), ("workflow", "GITHUB_WORKFLOW"),
+ ("job", "GITHUB_JOB"), ("sha", "GITHUB_SHA"))}
+ name = re.sub(r"[^A-Za-z0-9_.-]", "_", suite)
+ (out_dir / f"{name}.json").write_text(json.dumps(dict(meta, suite=suite, files=files[:200])))
+
+
+def main(argv: List[str] | None = None) -> int:
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = p.add_subparsers(dest="cmd", required=True)
+ j = sub.add_parser("junit")
+ j.add_argument("--suite", required=True)
+ j.add_argument("files", nargs="+")
+ s = sub.add_parser("swift-log")
+ s.add_argument("--suite", required=True)
+ s.add_argument("log")
+ args = p.parse_args(argv)
+ try:
+ if args.cmd == "junit":
+ write(args.suite, junit_files(args.files))
+ else:
+ write(args.suite, swift_log_suites(Path(args.log).read_text(errors="replace").splitlines()))
+ except (OSError, ValueError, ET.ParseError) as exc:
+ # Timings are measurement only: never fail a job over them.
+ print(f"::warning title=Test timings::could not read results for {args.suite}: {exc}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/tests/test_ci_weekly_report.py b/.github/scripts/tests/test_ci_weekly_report.py
new file mode 100644
index 0000000000..b4891a538f
--- /dev/null
+++ b/.github/scripts/tests/test_ci_weekly_report.py
@@ -0,0 +1,113 @@
+"""ci_weekly_report.py and flaky/timings.py (CUA-3558, cut #12)."""
+
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "flaky"))
+import ci_weekly_report as r # noqa: E402
+import timings # noqa: E402
+
+
+def run(name, start, end, conclusion="success", path=None):
+ return {"name": name, "path": path or f".github/workflows/{name}.yml@refs/pull/1/merge",
+ "started": f"2026-11-02T{start}:00Z", "updated": f"2026-11-02T{end}:00Z", "conclusion": conclusion}
+
+
+def pr(number=1, ref="feature/x", created="2026-11-02T09:00:00Z", merged="2026-11-02T11:00:00Z", signoff=None):
+ ctx = [{"context": "local-ci", "createdAt": signoff}] if signoff else []
+ return {"number": number, "createdAt": created, "mergedAt": merged, "headRefName": ref,
+ "commits": {"nodes": [{"commit": {"oid": "abc", "status": {"contexts": ctx}}}]}}
+
+
+class PrMetricsTests(unittest.TestCase):
+ def test_gate_that_waits_after_the_last_check(self):
+ m = r.pr_metrics(pr(), [run("a", "10:00", "10:30"), run("CI: gate", "10:00", "10:40")])
+ self.assertEqual(m["push_to_green"], 40)
+ self.assertEqual(m["tail"], 10)
+ self.assertFalse(m["tail_signoff"])
+ self.assertEqual(m["merge_wait"], 20)
+
+ def test_tail_is_blamed_on_the_signoff_when_it_lands_inside_it(self):
+ m = r.pr_metrics(pr(signoff="2026-11-02T10:54:00Z"), [run("a", "10:00", "10:30"), run("CI: gate", "10:00", "10:55")])
+ self.assertTrue(m["tail_signoff"])
+
+ def test_a_gate_rerun_after_a_label_is_not_a_tail(self):
+ m = r.pr_metrics(pr(), [run("a", "10:00", "10:30"), run("CI: gate", "10:50", "10:52")])
+ self.assertIsNone(m["tail"])
+ self.assertEqual(m["push_to_green"], 30)
+
+ def test_runs_the_gate_ignores_and_cancelled_runs_are_not_the_last_check(self):
+ runs = [run("a", "10:00", "10:30"), run("CI: gate", "10:00", "10:32"),
+ run("CI: CLA", "10:00", "10:31", path=".github/workflows/cla.yml@x"),
+ run("b", "10:00", "10:31", conclusion="cancelled")]
+ self.assertEqual(r.pr_metrics(pr(), runs)["tail"], 2)
+
+ def test_no_gate_no_metric(self):
+ self.assertIsNone(r.pr_metrics(pr(), [run("a", "10:00", "10:30")]))
+
+
+class ReportTests(unittest.TestCase):
+ def test_report_has_every_section(self):
+ prs = [r.pr_metrics(pr(), [run("a", "10:00", "10:30"), run("CI: gate", "10:00", "10:32")]),
+ r.pr_metrics(pr(number=2, ref="release-please--branches--main"), [run("a", "10:00", "10:20"), run("CI: gate", "10:00", "10:21")])]
+ cd = {"CD: x": {"runs": 2, "fail": 1, "durations": [30.0, 50.0]}}
+ timing = [{"suite": "pytest-x", "files": [{"file": "tests.test_a", "seconds": 90, "tests": 4}]}]
+ text = r.build_report(prs, cd, timing, "2026-10-29")
+ for needle in (r.MARKER, "Push to green", "Release PRs", "CD: x", "tests.test_a", "waiting for the `local-ci` sign-off"):
+ self.assertIn(needle, text)
+
+ def test_empty_data_does_not_crash(self):
+ text = r.build_report([], {}, [], "2026-10-29")
+ self.assertIn("no data", text)
+ self.assertIn("No `timings-*` artifacts", text)
+
+ def test_slowest_files_use_the_median_across_runs(self):
+ reports = [{"suite": "s", "files": [{"file": "f", "seconds": 10}, {"file": "g", "seconds": 1}]},
+ {"suite": "s", "files": [{"file": "f", "seconds": 30}]}]
+ rows = r.slowest_files(reports)
+ self.assertEqual((rows[0]["file"], rows[0]["median"], rows[0]["max"]), ("f", 20, 30))
+
+
+class TimingsTests(unittest.TestCase):
+ def test_junit_groups_by_file_and_drops_the_test_class(self):
+ xml = ('<testsuites><testsuite>'
+ '<testcase classname="tests.test_a.TestOne" name="x" time="2.5"/>'
+ '<testcase classname="tests.test_a" name="y" time="1.5"/>'
+ '<testcase classname="cua-relay::churn" name="z" time="9"/>'
+ '</testsuite></testsuites>')
+ with tempfile.TemporaryDirectory() as d:
+ p = Path(d) / "j.xml"
+ p.write_text(xml)
+ rows = timings.junit_files([str(p)])
+ self.assertEqual([(x["file"], x["seconds"], x["tests"]) for x in rows],
+ [("cua-relay::churn", 9.0, 1), ("tests.test_a", 4.0, 2)])
+
+ def test_swift_log_gives_one_row_per_test_class(self):
+ log = """Test Suite 'FooTests' passed at 2026-11-02 10:00:00.0.
+\t Executed 3 tests, with 0 failures (0 unexpected) in 1.250 (1.260) seconds
+Test Suite 'Pkg.xctest' passed at 2026-11-02 10:00:01.0.
+\t Executed 3 tests, with 0 failures (0 unexpected) in 1.250 (1.260) seconds
+Test Suite 'All tests' passed at 2026-11-02 10:00:01.0.
+\t Executed 3 tests, with 0 failures (0 unexpected) in 1.250 (1.260) seconds
+""".splitlines()
+ rows = timings.swift_log_suites(log)
+ self.assertEqual([(x["file"], x["seconds"], x["tests"]) for x in rows], [("FooTests", 1.25, 3)])
+
+
+class WorkflowCallsTimingsByAbsolutePath(unittest.TestCase):
+ def test_every_workflow_runs_timings_py_from_the_workspace_root(self):
+ """Jobs set `working-directory` (libs/cua-bench, libs/python/cua-sandbox, ...): a relative path is not found there."""
+ import re
+ workflows = Path(__file__).resolve().parents[2] / "workflows"
+ calls = [(f.name, m) for f in workflows.glob("*.yml")
+ for m in re.findall(r"python3\s+(\S*flaky/timings\.py\S*)", f.read_text())]
+ self.assertTrue(calls)
+ for name, path in calls:
+ self.assertEqual(path, '"$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py"', name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/workflows/ci-cua-microvm.yml b/.github/workflows/ci-cua-microvm.yml
index 033286cfdb..06cae28701 100644
--- a/.github/workflows/ci-cua-microvm.yml
+++ b/.github/workflows/ci-cua-microvm.yml
@@ -121,7 +121,17 @@ jobs:
shared-key: libs-cua-vmm
- name: cua-vz Swift tests
working-directory: libs/cua/vz
- run: swift test
+ # Per-file times (timings.py) from the XCTest suite summaries; the exit code is swift test's.
+ run: |
+ rc=0
+ swift test 2>&1 | tee "$RUNNER_TEMP/swift-test-cua-vz.log"
+ rc=${PIPESTATUS[0]}
+ python3 "$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py" swift-log --suite swift-cua-vz "$RUNNER_TEMP/swift-test-cua-vz.log"
+ exit "$rc"
+ - uses: ./.github/actions/ci-timings-upload
+ if: always()
+ with:
+ suite: swift-cua-vz
- name: cua-vz builds and is signed with its entitlement
run: libs/cua/vz/build.sh "$RUNNER_TEMP/vz"
- name: Backend unit tests on a Mac
diff --git a/.github/workflows/ci-cua-sdk.yml b/.github/workflows/ci-cua-sdk.yml
index 8b154fb925..29abb90e7a 100644
--- a/.github/workflows/ci-cua-sdk.yml
+++ b/.github/workflows/ci-cua-sdk.yml
@@ -843,7 +843,16 @@ jobs:
CUA_REQUIRE_FIXTURES: "1"
run: |
../cua/swift/scripts/fresh-abi.sh . debug
- swift test
+ # Per-file times (timings.py) from the XCTest suite summaries; the exit code is swift test's.
+ rc=0
+ swift test 2>&1 | tee "$RUNNER_TEMP/swift-test-spaces-sdk.log"
+ rc=${PIPESTATUS[0]}
+ python3 "$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py" swift-log --suite swift-spaces-sdk "$RUNNER_TEMP/swift-test-spaces-sdk.log"
+ exit "$rc"
+ - uses: ./.github/actions/ci-timings-upload
+ if: always()
+ with:
+ suite: swift-spaces-sdk
- name: XCFramework (static) binary target
run: |
swift/scripts/build-xcframework.sh
diff --git a/.github/workflows/ci-gate.yml b/.github/workflows/ci-gate.yml
index eb1aab576a..fde7e4b4a8 100644
--- a/.github/workflows/ci-gate.yml
+++ b/.github/workflows/ci-gate.yml
@@ -210,7 +210,14 @@ jobs:
break
fi
previous=$current
- sleep "$(poll_interval)"
+ # Everything finished: the next listing only has to confirm the same set, so look again
+ # in 10 s instead of after a full poll interval (up to 60 s). That wait, twice, was most
+ # of the gate's median 1.9 min after the last check (CUA-3558).
+ if [[ -n "$current" ]] && awk -F'\t' '$NF != "completed" { exit 1 }' <<<"$current"; then
+ sleep 10
+ else
+ sleep "$(poll_interval)"
+ fi
done
failed=0
again=0
@@ -261,7 +268,9 @@ jobs:
api() { local out; for _ in 1 2 3 4 5 6; do out=$(gh api "$@") && { printf '%s\n' "$out"; return; }; sleep 20; done; return 1; }
sha=$SHA
tries=1
- [[ "${REQUIRE_LOCAL_CI:-}" == true ]] && tries=75
+ # Polled every 15 s (it was 60 s, so a sign-off that landed just after a poll cost up to a
+ # minute). Still bounded at 75 minutes.
+ [[ "${REQUIRE_LOCAL_CI:-}" == true ]] && tries=300
for i in $(seq 1 "$tries"); do
if [[ "$KIND" == local ]]; then
state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')
@@ -271,7 +280,7 @@ jobs:
fi
if [[ "$state" == success ]]; then echo "sign-off: success"; exit 0; fi
[[ "$state" == failure || "$state" == error ]] && break
- (( i < tries )) && sleep 60
+ (( i < tries )) && sleep 15
done
if [[ "$KIND" == local ]]; then
msg="Run bin/ci --quick and bin/ci --signoff on this head commit. (local-ci on ${sha:0:10}: ${state:-not set})"
diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml
index 14af74d01c..3280a31f9b 100644
--- a/.github/workflows/ci-py-bench.yml
+++ b/.github/workflows/ci-py-bench.yml
@@ -52,7 +52,17 @@ jobs:
# Hermetic: cua-sandbox is replaced by in-memory fakes. Live tests
# (tests/live, CUA_E2E_*) never run here.
- name: Tests (CLI goldens, export snapshots, runner)
- run: uv run --frozen pytest -q cua_bench/tests --ignore=cua_bench/tests/live
+ run: uv run --frozen pytest -q cua_bench/tests --ignore=cua_bench/tests/live --durations=25 --durations-min=1.0 --junitxml="$RUNNER_TEMP/pytest-cua-bench.xml"
+ - name: Per-file test times
+ if: always()
+ run: |
+ if [ -f "$RUNNER_TEMP/pytest-cua-bench.xml" ]; then
+ python3 "$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py" junit --suite pytest-cua-bench "$RUNNER_TEMP/pytest-cua-bench.xml"
+ fi
+ - uses: ./.github/actions/ci-timings-upload
+ if: always()
+ with:
+ suite: pytest-cua-bench
test-rl:
name: cua-bench-rl (workers, dataloader) tests
diff --git a/.github/workflows/ci-test-python.yml b/.github/workflows/ci-test-python.yml
index 0066c9c38e..b5af8cbdf6 100644
--- a/.github/workflows/ci-test-python.yml
+++ b/.github/workflows/ci-test-python.yml
@@ -103,11 +103,23 @@ jobs:
run: |
cd libs/python/${{ matrix.package }}
if [ -d tests ]; then
- python -m pytest tests/ -v --tb=short --cov --cov-report=term --cov-report=xml
+ python -m pytest tests/ -v --tb=short --cov --cov-report=term --cov-report=xml \
+ --durations=25 --durations-min=1.0 --junitxml="$RUNNER_TEMP/pytest-${{ matrix.package }}.xml"
else
echo "No tests directory found, skipping tests"
fi
shell: bash
+ - name: Per-file test times
+ if: always()
+ run: |
+ if [ -f "$RUNNER_TEMP/pytest-${{ matrix.package }}.xml" ]; then
+ python3 "$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py" junit --suite "pytest-${{ matrix.package }}" "$RUNNER_TEMP/pytest-${{ matrix.package }}.xml"
+ fi
+ shell: bash
+ - uses: ./.github/actions/ci-timings-upload
+ if: always()
+ with:
+ suite: pytest-${{ matrix.package }}
env:
CUA_TELEMETRY_ENABLED: "false" # Disable telemetry during tests
@@ -160,11 +172,22 @@ jobs:
CUA_TELEMETRY_ENABLED: "false"
run: >
uv run --frozen pytest tests -p no:cacheprovider --timeout=120 -ra
+ --durations=25 --durations-min=1.0 --junitxml="$RUNNER_TEMP/pytest-cua-sandbox.xml"
--ignore=tests/live
--ignore=tests/test_runtime.py
--ignore=tests/test_snapshots.py
--ignore=tests/test_windows_timing.py
--deselect tests/test_oci.py::TestLiveRegistry
+ - name: Per-file test times
+ if: always()
+ run: |
+ if [ -f "$RUNNER_TEMP/pytest-cua-sandbox.xml" ]; then
+ python3 "$GITHUB_WORKSPACE/.github/scripts/flaky/timings.py" junit --suite pytest-cua-sandbox "$RUNNER_TEMP/pytest-cua-sandbox.xml"
+ fi
+ - uses: ./.github/actions/ci-timings-upload
+ if: always()
+ with:
+ suite: pytest-cua-sandbox
summary:
name: Test Summary
diff --git a/.github/workflows/ci-weekly-report.yml b/.github/workflows/ci-weekly-report.yml
new file mode 100644
index 0000000000..d787784272
--- /dev/null
+++ b/.github/workflows/ci-weekly-report.yml
@@ -0,0 +1,39 @@
+name: "CI weekly speed report"
+
+# Opens or updates ONE issue, "CI speed this week", every Monday: push-to-green, the ci-gate tail,
+# green-to-merge, release-PR wait, CD time and the slowest test files (CUA-3558). Same pattern as the
+# weekly flaky-test issue in flaky-issues.yml. Runs the default branch's copy of the script and never
+# checks out or runs a pull request's code.
+
+on:
+ schedule:
+ - cron: "47 6 * * 1"
+ workflow_dispatch:
+
+permissions:
+ actions: read
+ contents: read
+ issues: write
+ pull-requests: read
+
+concurrency:
+ group: ci-weekly-report
+ cancel-in-progress: false
+
+jobs:
+ report:
+ name: weekly CI speed issue
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+ sparse-checkout: |
+ .github/scripts
+ .github/workflows
+ - name: Open or update the "CI speed this week" issue
+ env:
+ GH_TOKEN: ${{ github.token }}
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ run: python3 .github/scripts/ci_weekly_report.py
diff --git a/docs/contributing/CI.md b/docs/contributing/CI.md
index 1c1b7d8542..ed670000c4 100644
--- a/docs/contributing/CI.md
+++ b/docs/contributing/CI.md
@@ -220,6 +220,22 @@ they pass, the run passes. If they fail again the same way, the gate reports a q
fail. The gate reads the list from the base commit. Every entry has an owner, an issue and an expiry date.
The nightly **Flaky job scan** files a `flaky` issue for each job that failed and then passed on a rerun.
+## Measuring CI speed
+
+- **Per-file test times.** Jobs that run under `flaky.py` (nextest JUnit, vitest, Swift logs) and the pytest and
+ `swift test` steps of `ci-test-python`, `ci-py-bench`, `ci-cua-sdk` and `ci-cua-microvm` write a "Slowest test
+ files" table to the job summary and upload `timings-<suite>` (10 days) through
+ `.github/scripts/flaky/timings.py`. pytest also prints `--durations=25`. A new suite gets this by calling
+ `flaky.sh run` plus the `flaky-upload` action, or `timings.py` plus `ci-timings-upload`.
+- **Weekly report.** `ci-weekly-report.yml` rewrites ONE issue, "CI speed this week", every Monday
+ (`.github/scripts/ci_weekly_report.py`): push to green, green to merged, the `ci-gate` tail split into
+ "waiting for the `local-ci` sign-off" and "other", release-PR wait, CD time per workflow, and the slowest
+ test files. Run it by hand with `gh workflow run ci-weekly-report.yml`.
+- **The gate tail.** After the last other workflow finishes, `ci-gate` needs one confirming listing (10 s once
+ everything is complete) and then the `local-ci` sign-off, polled every 15 s for up to 75 minutes. A long tail is
+ almost always the sign-off queue on the Mac runner or a quarantined-flake rerun, not a poll interval; the
+ weekly report shows which.
+
## Urgent-fix lane
Label a pull request `urgent` for a production fix that cannot wait for the full suite: