Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci: retry flaky tests once, report them, add a quarantine list (CUA-2336) (#5996)

cua-agents[bot]1e6db4557075
diff --git a/.github/actions/flaky-upload/action.yml b/.github/actions/flaky-upload/action.yml
new file mode 100644
index 0000000000..62a59b54f3
--- /dev/null
+++ b/.github/actions/flaky-upload/action.yml
@@ -0,0 +1,15 @@
+name: Upload flaky-test results
+description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml to turn into GitHub issues. Run with if: always()."
+inputs:
+ suite:
+ description: "The suite name given to flaky.py run (unique per job, matrix legs included)."
+ required: true
+runs:
+ using: composite
+ steps:
+ - uses: actions/upload-artifact@v4
+ with:
+ name: flaky-${{ inputs.suite }}
+ path: ${{ runner.temp }}/flaky-results/
+ if-no-files-found: ignore
+ retention-days: 3
diff --git a/.github/actions/nextest/action.yml b/.github/actions/nextest/action.yml
new file mode 100644
index 0000000000..75efc7f4a4
--- /dev/null
+++ b/.github/actions/nextest/action.yml
@@ -0,0 +1,8 @@
+name: Install cargo-nextest
+description: "cargo-nextest, for the CI profile in .github/flaky/nextest.toml (one retry, JUnit output)."
+runs:
+ using: composite
+ steps:
+ - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
+ with:
+ tool: cargo-nextest
diff --git a/.github/flaky/nextest.toml b/.github/flaky/nextest.toml
new file mode 100644
index 0000000000..1ea3bdf226
--- /dev/null
+++ b/.github/flaky/nextest.toml
@@ -0,0 +1,11 @@
+# cargo nextest profile for CI. One retry; a test that passes only on the
+# retry is listed as <flakyFailure> in the JUnit file, which
+# .github/scripts/flaky/flaky.py turns into a CI summary, an annotation and a
+# GitHub issue. Use: cargo nextest run --config-file <this file> --profile ci
+[profile.ci]
+retries = 1
+fail-fast = false
+failure-output = "immediate-final"
+
+[profile.ci.junit]
+path = "junit.xml"
diff --git a/.github/flaky/quarantine.json b/.github/flaky/quarantine.json
new file mode 100644
index 0000000000..03f87f3190
--- /dev/null
+++ b/.github/flaky/quarantine.json
@@ -0,0 +1,4 @@
+{
+ "$comment": "Known-flaky tests that may fail without blocking CI. Add an entry only with an owner, an issue and an expiry date. CI fails when an entry expires (ci-flaky-quarantine). See .github/scripts/flaky/flaky.py.",
+ "entries": []
+}
diff --git a/.github/flaky/vitest-reporter.mjs b/.github/flaky/vitest-reporter.mjs
new file mode 100644
index 0000000000..777cf4a3f7
--- /dev/null
+++ b/.github/flaky/vitest-reporter.mjs
@@ -0,0 +1,31 @@
+// Vitest reporter for CI flaky-test handling (.github/scripts/flaky/flaky.py).
+// Writes one row per test to $FLAKY_RESULTS_OUT: "flaky" when the test passed
+// only on a retry, "failed" when it failed after its retries. Does nothing
+// when FLAKY_RESULTS_OUT is unset, so local runs are unchanged.
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, relative } from "node:path";
+
+export default class FlakyReporter {
+ rows = [];
+
+ onTestCaseResult(testCase) {
+ const result = testCase.result();
+ if (result.state !== "passed" && result.state !== "failed") return;
+ const retried = (testCase.diagnostic()?.retryCount ?? 0) > 0;
+ const status = result.state === "failed" ? "failed" : retried ? "flaky" : "passed";
+ if (status === "passed") return;
+ const file = relative(process.cwd(), testCase.module.moduleId);
+ this.rows.push({
+ id: `${file} > ${testCase.fullName}`,
+ status,
+ message: String(result.errors?.[0]?.message ?? "").slice(0, 1500),
+ });
+ }
+
+ onTestRunEnd() {
+ const out = process.env.FLAKY_RESULTS_OUT;
+ if (!out) return;
+ mkdirSync(dirname(out), { recursive: true });
+ writeFileSync(out, JSON.stringify(this.rows));
+ }
+}
diff --git a/.github/scripts/flaky/flaky.py b/.github/scripts/flaky/flaky.py
new file mode 100755
index 0000000000..b21173a670
--- /dev/null
+++ b/.github/scripts/flaky/flaky.py
@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+"""Flaky-test handling for CI: report, quarantine, and file issues.
+
+The test runners retry once themselves (cargo nextest `retries = 1`, vitest
+`retry`, Playwright `retries`, xcodebuild `-retry-tests-on-failure`). This tool
+wraps the runner and reads its result file:
+
+ run run a test command, then read its results. A test that passed only
+ on the retry is reported (job summary, annotation, JSON artifact),
+ never hidden. A failure that is not in the quarantine list still
+ fails the job. If every failing test is in the quarantine list and
+ not expired, the job passes with a warning.
+ validate check .github/flaky/quarantine.json; exit 1 on an invalid or
+ expired entry.
+ file-issues
+ open or update one GitHub issue per flaky test (label `flaky`)
+ from the JSON artifacts that `run` wrote.
+
+Result formats (--format): junit (cargo nextest), vitest (the reporter in
+.github/flaky/vitest-reporter.mjs), playwright (the JSON reporter), log (swift
+test and xcodebuild console output).
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import fnmatch
+import glob
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+QUARANTINE_FILE = Path(__file__).resolve().parents[2] / "flaky" / "quarantine.json"
+MAX_QUARANTINE_DAYS = 90
+ISSUE_RE = re.compile(r"^(CUA-\d+|#\d+|https://github\.com/[\w.-]+/[\w.-]+/issues/\d+|https://linear\.app/\S+)$")
+RANK = {"skipped": 0, "passed": 1, "flaky": 2, "failed": 3}
+
+Results = Dict[str, dict]
+
+
+def merge(into: Results, test_id: str, status: str, message: str = "") -> None:
+ old = into.get(test_id)
+ if old is None or RANK[status] > RANK[old["status"]]:
+ into[test_id] = {"status": status, "message": message}
+
+
+# --- parsers ---------------------------------------------------------------
+
+
+def parse_junit(paths: Iterable[str]) -> Results:
+ out: Results = {}
+ for path in paths:
+ for case in ET.parse(path).getroot().iter("testcase"):
+ cls, name = case.get("classname", ""), case.get("name", "")
+ test_id = f"{cls}::{name}" if cls and cls != name else name
+ tags = {child.tag: child for child in case}
+ if "failure" in tags or "error" in tags:
+ node = tags.get("failure", tags.get("error"))
+ merge(out, test_id, "failed", (node.get("message") or node.text or "").strip())
+ elif "flakyFailure" in tags or "flakyError" in tags:
+ node = tags.get("flakyFailure", tags.get("flakyError"))
+ merge(out, test_id, "flaky", (node.get("message") or node.text or "").strip())
+ elif "skipped" in tags:
+ merge(out, test_id, "skipped")
+ else:
+ merge(out, test_id, "passed")
+ return out
+
+
+def parse_vitest(paths: Iterable[str]) -> Results:
+ out: Results = {}
+ for path in paths:
+ for row in json.loads(Path(path).read_text()):
+ merge(out, row["id"], row["status"], row.get("message", ""))
+ return out
+
+
+def parse_playwright(paths: Iterable[str]) -> Results:
+ out: Results = {}
+ status_map = {"expected": "passed", "unexpected": "failed", "flaky": "flaky", "skipped": "skipped"}
+
+ def walk(suite: dict, trail: List[str]) -> None:
+ here = trail + ([suite["title"]] if suite.get("title") else [])
+ for spec in suite.get("specs", []):
+ for test in spec.get("tests", []):
+ project = test.get("projectName", "")
+ test_id = " > ".join(here + [spec["title"]]) + (f" [{project}]" if project else "")
+ message = ""
+ for result in test.get("results", []):
+ err = (result.get("error") or {}).get("message")
+ if err:
+ message = re.sub(r"\x1b\[[0-9;]*m", "", err)
+ break
+ merge(out, test_id, status_map.get(test.get("status", ""), "passed"), message)
+ for child in suite.get("suites", []):
+ walk(child, here)
+
+ for path in paths:
+ for suite in json.loads(Path(path).read_text()).get("suites", []):
+ walk(suite, [])
+ return out
+
+
+LOG_PATTERNS = [
+ # Swift Testing: "✘ Test realFail() failed after 0.001 seconds"
+ (re.compile(r"^[✔✘] Test (?!run with )(.+?) (passed|failed) after"), lambda m: m.group(1)),
+ # XCTest: "Test Case '-[LTests.X testY]' failed (0.1 seconds)."
+ (re.compile(r"^Test Case '-\[(\S+) (\S+)\]' (passed|failed)"), lambda m: f"{m.group(1)}/{m.group(2)}"),
+ # xcodebuild: "Test case 'X.y()' failed on 'My Mac - app (123)' (0.1 seconds)"
+ (re.compile(r"^Test case '(.+?)' (passed|failed) on "), lambda m: m.group(1)),
+]
+
+
+def parse_log(lines: Iterable[str]) -> Results:
+ """A test that failed and later passed in the same log is flaky."""
+ seen: Dict[str, List[str]] = {}
+ for line in lines:
+ line = line.strip()
+ for pattern, ident in LOG_PATTERNS:
+ m = pattern.match(line)
+ if m:
+ seen.setdefault(ident(m), []).append(m.group(m.lastindex))
+ break
+ out: Results = {}
+ for test_id, seq in seen.items():
+ if seq[-1] == "failed":
+ status = "failed"
+ elif "failed" in seq:
+ status = "flaky"
+ else:
+ status = "passed"
+ out[test_id] = {"status": status, "message": ""}
+ return out
+
+
+# --- quarantine --------------------------------------------------------------
+
+
+def load_quarantine() -> List[dict]:
+ return json.loads(QUARANTINE_FILE.read_text()).get("entries", [])
+
+
+def check_entries(entries: List[dict], today: dt.date) -> Tuple[List[str], List[str]]:
+ """Return (errors, expired). Both make `validate` fail."""
+ errors: List[str] = []
+ expired: List[str] = []
+ for i, entry in enumerate(entries):
+ label = entry.get("test", f"entry {i}")
+ for key in ("test", "owner", "issue", "expires"):
+ if not str(entry.get(key, "")).strip():
+ errors.append(f"{label}: missing `{key}`")
+ if entry.get("issue") and not ISSUE_RE.match(entry["issue"]):
+ errors.append(f"{label}: `issue` must be CUA-123, #123, or an issue URL, got {entry['issue']!r}")
+ try:
+ day = dt.date.fromisoformat(entry.get("expires", ""))
+ except ValueError:
+ errors.append(f"{label}: `expires` must be YYYY-MM-DD, got {entry.get('expires')!r}")
+ continue
+ if day < today:
+ expired.append(f"{label}: expired on {day} (owner {entry.get('owner')}, issue {entry.get('issue')})")
+ elif (day - today).days > MAX_QUARANTINE_DAYS:
+ errors.append(f"{label}: `expires` is more than {MAX_QUARANTINE_DAYS} days away")
+ return errors, expired
+
+
+def active_entries(entries: List[dict], today: dt.date) -> List[dict]:
+ good = []
+ for entry in entries:
+ try:
+ if dt.date.fromisoformat(entry["expires"]) >= today and entry.get("owner") and entry.get("issue"):
+ good.append(entry)
+ except (KeyError, ValueError):
+ pass
+ return good
+
+
+def is_quarantined(test_id: str, suite: str, entries: List[dict]) -> Optional[dict]:
+ for entry in entries:
+ if not (fnmatch.fnmatchcase(test_id, entry["test"]) or test_id.endswith("::" + entry["test"])):
+ continue
+ if entry.get("suite") and not fnmatch.fnmatchcase(suite, entry["suite"]):
+ continue
+ return entry
+ return None
+
+
+# --- run -----------------------------------------------------------------
+
+
+def stream(cmd: List[str]) -> Tuple[int, List[str]]:
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace")
+ lines: List[str] = []
+ assert proc.stdout is not None
+ for line in proc.stdout:
+ sys.stdout.write(line)
+ lines.append(line)
+ return proc.wait(), lines
+
+
+def run_metadata() -> dict:
+ meta = {
+ "repo": os.environ.get("GITHUB_REPOSITORY", ""),
+ "run_id": os.environ.get("GITHUB_RUN_ID", ""),
+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", ""),
+ "workflow": os.environ.get("GITHUB_WORKFLOW", ""),
+ "job": os.environ.get("GITHUB_JOB", ""),
+ "ref": os.environ.get("GITHUB_REF", ""),
+ "sha": os.environ.get("GITHUB_SHA", ""),
+ "pr": None,
+ }
+ event_path = os.environ.get("GITHUB_EVENT_PATH")
+ if event_path and Path(event_path).is_file():
+ pr = json.loads(Path(event_path).read_text()).get("pull_request")
+ if pr:
+ meta["pr"] = pr.get("number")
+ meta["sha"] = pr.get("head", {}).get("sha", meta["sha"])
+ return meta
+
+
+def cell(text: str) -> str:
+ return text.replace("|", "\\|").replace("\n", " ")[:200]
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+ command = args.command[1:] if args.command[:1] == ["--"] else args.command
+ if not command:
+ print("flaky.py run: no command after --", file=sys.stderr)
+ return 2
+ today = dt.date.today()
+ entries = load_quarantine()
+ bad, expired = check_entries(entries, today)
+ live = active_entries(entries, today)
+ for problem in bad + expired:
+ print(f"::warning title=Quarantine list::{problem}")
+
+ for stale in (p for pattern in args.results for p in glob.glob(pattern, recursive=True)):
+ os.remove(stale) # a leftover file from an earlier run must not be read as this run's
+ code, lines = stream(command)
+ paths = sorted(p for pattern in args.results for p in glob.glob(pattern, recursive=True))
+ try:
+ if args.format == "log":
+ results = parse_log(lines)
+ failed_first = sorted(t for t, r in results.items() if r["status"] == "failed")
+ if code != 0 and failed_first and args.rerun_failed:
+ # swift test has no retry flag: run the failures once more.
+ rerun_cmd = list(command)
+ for test_id in failed_first:
+ rerun_cmd += ["--filter", re.escape(test_id)]
+ print(f"::notice title=Retry::re-running {len(failed_first)} failed test(s) once")
+ code, lines2 = stream(rerun_cmd)
+ second = parse_log(lines2)
+ for test_id in failed_first:
+ after = second.get(test_id, {"status": "failed"})["status"]
+ results[test_id]["status"] = "flaky" if after == "passed" else "failed"
+ elif args.format == "junit":
+ results = parse_junit(paths)
+ elif args.format == "vitest":
+ results = parse_vitest(paths)
+ else:
+ results = parse_playwright(paths)
+ except (OSError, ValueError, KeyError, ET.ParseError) as exc:
+ print(f"::warning title=Flaky report::could not read results ({exc}); exit code {code} stands")
+ return code
+
+ flaky = {t: r for t, r in results.items() if r["status"] == "flaky"}
+ failed = {t: r for t, r in results.items() if r["status"] == "failed"}
+ quarantined = {t: r for t, r in failed.items() if is_quarantined(t, args.suite, live)}
+ real = {t: r for t, r in failed.items() if t not in quarantined}
+
+ for test_id, r in flaky.items():
+ print(f"::warning title=Flaky test ({args.suite})::{test_id} passed only on retry")
+ for test_id in quarantined:
+ entry = is_quarantined(test_id, args.suite, live)
+ print(
+ f"::warning title=Quarantined test failed ({args.suite})::{test_id} "
+ f"(owner {entry['owner']}, {entry['issue']}, expires {entry['expires']})"
+ )
+ for test_id in real:
+ print(f"::error title=Test failed ({args.suite})::{test_id}")
+
+ write_summary(args.suite, results, flaky, quarantined, real, live)
+ write_artifact(args.suite, flaky, quarantined)
+
+ if code == 0:
+ return 0
+ # The runner failed. Pass only if it failed on quarantined tests and nothing else.
+ if quarantined and not real:
+ print(f"::warning title=Quarantine::{len(quarantined)} quarantined test(s) failed; not blocking")
+ return 0
+ return code
+
+
+def write_summary(suite: str, results: Results, flaky: Results, quarantined: Results, real: Results, live: List[dict]) -> None:
+ path = os.environ.get("GITHUB_STEP_SUMMARY")
+ if not path or not (flaky or quarantined or real):
+ return
+ out = [f"### Test results: {suite}", ""]
+ counts = {s: sum(1 for r in results.values() if r["status"] == s) for s in RANK}
+ out.append(
+ f"{counts['passed']} passed, {len(flaky)} flaky (passed only on retry), "
+ f"{len(real)} failed, {len(quarantined)} quarantined failures, {counts['skipped']} skipped."
+ )
+ if flaky:
+ out += ["", "**Flaky: passed only on retry. A `flaky` issue is opened or updated for each.**", "", "| Test | First failure |", "|---|---|"]
+ out += [f"| `{cell(t)}` | {cell(r['message'])} |" for t, r in sorted(flaky.items())]
+ if quarantined:
+ out += ["", "**Quarantined: failed but not blocking.**", "", "| Test | Owner | Issue | Expires |", "|---|---|---|---|"]
+ for t in sorted(quarantined):
+ e = is_quarantined(t, suite, live)
+ out.append(f"| `{cell(t)}` | {e['owner']} | {e['issue']} | {e['expires']} |")
+ if real:
+ out += ["", "**Failed (blocking).**", "", "| Test | Failure |", "|---|---|"]
+ out += [f"| `{cell(t)}` | {cell(r['message'])} |" for t, r in sorted(real.items())]
+ with open(path, "a") as fh:
+ fh.write("\n".join(out) + "\n")
+
+
+def write_artifact(suite: str, flaky: Results, quarantined: Results) -> None:
+ base = os.environ.get("RUNNER_TEMP")
+ if not base or not (flaky or quarantined):
+ return
+ out_dir = Path(base) / "flaky-results"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ payload = dict(run_metadata(), suite=suite, flaky=[dict(id=t, message=r["message"][:1500]) for t, r in flaky.items()],
+ quarantined=sorted(quarantined))
+ (out_dir / f"{re.sub(r'[^A-Za-z0-9_.-]', '_', suite)}.json").write_text(json.dumps(payload, indent=2))
+
+
+# --- validate ----------------------------------------------------------------
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+ today = dt.date.fromisoformat(args.today) if args.today else dt.date.today()
+ try:
+ entries = load_quarantine()
+ except (OSError, ValueError) as exc:
+ print(f"::error::cannot read {QUARANTINE_FILE}: {exc}")
+ return 1
+ errors, expired = check_entries(entries, today)
+ for problem in errors:
+ print(f"::error title=Quarantine entry invalid::{problem}")
+ for problem in expired:
+ print(f"::error title=Quarantine entry expired::{problem}. Fix the test and remove the entry, or renew it with a reason.")
+ print(f"{len(entries)} quarantine entr{'y' if len(entries) == 1 else 'ies'}, {len(errors)} invalid, {len(expired)} expired")
+ return 1 if errors or expired else 0
+
+
+# --- issues ----------------------------------------------------------------
+
+
+def clean(text: str, limit: int) -> str:
+ text = re.sub(r"[\x00-\x08\x0b-\x1f\x7f]", "", text)
+ return text.replace("@", "@\u200b")[:limit]
+
+
+def gh(*args: str, check: bool = True) -> str:
+ done = subprocess.run(["gh", *args], capture_output=True, text=True)
+ if check and done.returncode != 0:
+ raise RuntimeError(f"gh {' '.join(args[:3])} failed: {done.stderr.strip()}")
+ return done.stdout
+
+
+def flaky_key(test_id: str) -> str:
+ return hashlib.sha1(test_id.encode()).hexdigest()[:12]
+
+
+def cmd_file_issues(args: argparse.Namespace) -> int:
+ repo = os.environ["GITHUB_REPOSITORY"]
+ server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+ reports = [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{args.dir}/**/*.json", recursive=True))]
+ items: Dict[str, List[Tuple[dict, dict]]] = {}
+ for report in reports:
+ for row in report.get("flaky", []):
+ items.setdefault(row["id"], []).append((report, row))
+ if not items:
+ print("no flaky tests reported")
+ return 0
+ if args.dry_run:
+ for test_id in items:
+ print(f"would file: Flaky test: {clean(test_id, 120)}")
+ return 0
+ gh("label", "create", "flaky", "--repo", repo, "--color", "FBCA04", "--description", "Test that passed only on retry", check=False)
+ existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "all", "--limit", "1000",
+ "--json", "number,body,state") or "[]")
+ by_key = {}
+ for issue in existing:
+ m = re.search(r"<!-- flaky-key:(\w+) -->", issue.get("body") or "")
+ if m:
+ by_key[m.group(1)] = issue
+ for test_id, occurrences in items.items():
+ key = flaky_key(test_id)
+ report, row = occurrences[0]
+ run_url = f"{server}/{repo}/actions/runs/{report['run_id']}"
+ entry = [
+ f"Passed only on retry in `{clean(report['suite'], 80)}`.",
+ "",
+ f"- Run: {run_url} (attempt {report['run_attempt']}, workflow {clean(report['workflow'], 80)}, job {clean(report['job'], 80)})",
+ f"- Commit: `{report['sha'][:12]}`" + (f", PR #{report['pr']}" if report.get("pr") else f", ref `{clean(report['ref'], 80)}`"),
+ "",
+ "```",
+ clean(row.get("message", "") or "(no message)", 1200).replace("```", "'''"),
+ "```",
+ ]
+ issue = by_key.get(key)
+ if issue is None:
+ body = "\n".join([
+ f"`{clean(test_id, 300)}` passes only on retry in CI. Fix the race, or quarantine it in "
+ "`.github/flaky/quarantine.json` with an owner, this issue and an expiry date.",
+ "",
+ "Failure history is in the comments below, one per CI run.",
+ "",
+ f"<!-- flaky-key:{key} -->",
+ ])
+ url = gh("issue", "create", "--repo", repo, "--label", "flaky",
+ "--title", f"Flaky test: {clean(test_id, 200)}", "--body", body).strip()
+ number = url.rsplit("/", 1)[-1]
+ print(f"opened {url}")
+ else:
+ number = str(issue["number"])
+ have = gh("issue", "view", number, "--repo", repo, "--json", "comments", "--jq", ".comments[].body")
+ if f"/actions/runs/{report['run_id']}" in have:
+ print(f"#{number}: run {report['run_id']} already recorded")
+ continue
+ if issue["state"] == "CLOSED":
+ gh("issue", "reopen", number, "--repo", repo)
+ gh("issue", "comment", number, "--repo", repo, "--body", "\n".join(entry))
+ print(f"#{number}: recorded {run_url}")
+ return 0
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = parser.add_subparsers(dest="cmd", required=True)
+ run = sub.add_parser("run")
+ run.add_argument("--suite", required=True)
+ run.add_argument("--format", choices=["junit", "vitest", "playwright", "log"], required=True)
+ run.add_argument("--results", action="append", default=[], help="result file glob (repeatable)")
+ run.add_argument("--rerun-failed", action="store_true", help="log format: run the failed tests once more with --filter")
+ run.add_argument("command", nargs=argparse.REMAINDER)
+ validate = sub.add_parser("validate")
+ validate.add_argument("--today")
+ issues = sub.add_parser("file-issues")
+ issues.add_argument("--dir", required=True)
+ issues.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args(argv)
+ return {"run": cmd_run, "validate": cmd_validate, "file-issues": cmd_file_issues}[args.cmd](args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/flaky/flaky.sh b/.github/scripts/flaky/flaky.sh
new file mode 100755
index 0000000000..db2adedeb8
--- /dev/null
+++ b/.github/scripts/flaky/flaky.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Runs flaky.py with whichever Python the runner has (python3 on Linux and
+# macOS, python on some Windows images).
+set -euo pipefail
+py=$(command -v python3 || command -v python) || { echo "no python on this runner" >&2; exit 2; }
+exec "$py" "$(dirname "${BASH_SOURCE[0]}")/flaky.py" "$@"
diff --git a/.github/scripts/tests/test_flaky.py b/.github/scripts/tests/test_flaky.py
new file mode 100644
index 0000000000..4fd59afebc
--- /dev/null
+++ b/.github/scripts/tests/test_flaky.py
@@ -0,0 +1,128 @@
+import datetime as dt
+import importlib.util
+import json
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).resolve().parents[1] / "flaky" / "flaky.py"
+spec = importlib.util.spec_from_file_location("flaky", SCRIPT)
+flaky = importlib.util.module_from_spec(spec)
+sys.modules["flaky"] = flaky
+spec.loader.exec_module(flaky)
+
+TODAY = dt.date(2026, 10, 25)
+JUNIT = textwrap.dedent(
+ """\
+ <testsuites><testsuite name="pkg::bin">
+ <testcase classname="pkg::bin" name="ok"/>
+ <testcase classname="pkg::bin" name="racy"><flakyFailure message="timed out"/></testcase>
+ <testcase classname="pkg::bin" name="broken"><failure message="boom"/><rerunFailure message="boom"/></testcase>
+ <testcase classname="pkg::bin" name="skip"><skipped/></testcase>
+ </testsuite></testsuites>
+ """
+)
+
+
+def entry(**kw):
+ base = {"test": "pkg::bin::broken", "owner": "francesco", "issue": "CUA-1", "expires": "2026-11-01"}
+ base.update(kw)
+ return base
+
+
+def env(tmp_path, monkeypatch):
+ quarantine = tmp_path / "quarantine.json"
+ quarantine.write_text(json.dumps({"entries": []}))
+ monkeypatch.setattr(flaky, "QUARANTINE_FILE", quarantine)
+ monkeypatch.setenv("RUNNER_TEMP", str(tmp_path))
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "summary.md"))
+ monkeypatch.setattr(flaky.dt, "date", type("D", (dt.date,), {"today": staticmethod(lambda: TODAY)}))
+ return tmp_path
+
+
+def run(env, *extra, exit_code=1, junit=JUNIT):
+ """Run a fake test command that writes `junit` and exits with `exit_code`."""
+ out = env / "junit.xml"
+ script = f"import pathlib; pathlib.Path({str(out)!r}).write_text({junit!r}); raise SystemExit({exit_code})"
+ return flaky.main(["run", "--suite", "t", "--format", "junit", "--results", str(out), *extra, "--",
+ sys.executable, "-c", script])
+
+
+def test_parse_junit_statuses(env):
+ (env / "junit.xml").write_text(JUNIT)
+ got = flaky.parse_junit([str(env / "junit.xml")])
+ assert {k: v["status"] for k, v in got.items()} == {
+ "pkg::bin::ok": "passed", "pkg::bin::racy": "flaky", "pkg::bin::broken": "failed", "pkg::bin::skip": "skipped"}
+
+
+def test_real_failure_still_fails_and_flaky_is_reported(env):
+ assert run(env) == 1
+ assert "pkg::bin::racy" in (env / "summary.md").read_text()
+ report = json.loads((env / "flaky-results" / "t.json").read_text())
+ assert [row["id"] for row in report["flaky"]] == ["pkg::bin::racy"]
+
+
+def test_flaky_only_passes_with_report(env):
+ assert run(env, exit_code=0, junit=JUNIT.replace('<failure message="boom"/><rerunFailure message="boom"/>', "")) == 0
+ assert (env / "flaky-results" / "t.json").exists()
+
+
+def test_quarantined_failure_does_not_block(env):
+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry()]}))
+ assert run(env) == 0
+
+
+def test_expired_quarantine_blocks(env):
+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(expires="2026-10-01")]}))
+ assert run(env) == 1
+
+
+def test_quarantine_does_not_hide_other_failures(env):
+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(test="pkg::bin::other")]}))
+ assert run(env) == 1
+
+
+def test_failure_without_parsed_tests_fails(env):
+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry()]}))
+ assert run(env, junit="<testsuites/>") == 1
+
+
+def test_validate(env):
+ assert flaky.check_entries([entry()], TODAY) == ([], [])
+ errors, expired = flaky.check_entries([entry(owner="", issue="soon"), entry(expires="2026-10-24")], TODAY)
+ assert len(errors) == 2 and len(expired) == 1
+ assert flaky.check_entries([entry(expires="2027-06-01")], TODAY)[0]
+ assert flaky.main(["validate", "--today", "2026-10-25"]) == 0
+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(expires="2026-10-01")]}))
+ assert flaky.main(["validate", "--today", "2026-10-25"]) == 1
+
+
+def test_parse_log_swift_and_xcodebuild():
+ lines = [
+ "✘ Test racy() failed after 0.001 seconds with 1 issue.",
+ "✔ Test racy() passed after 0.001 seconds.",
+ "✘ Test broken() failed after 0.001 seconds with 1 issue.",
+ "✘ Test run with 3 tests in 1 suite failed after 0.001 seconds with 2 issues.",
+ "Test Case '-[LTests.X testY]' failed (0.1 seconds).",
+ "Test case 'UITests.go()' failed on 'My Mac - App (1)' (1.0 seconds)",
+ "Test case 'UITests.go()' passed on 'My Mac - App (1)' (1.0 seconds)",
+ ]
+ got = {k: v["status"] for k, v in flaky.parse_log(lines).items()}
+ assert got == {"racy()": "flaky", "broken()": "failed", "LTests.X/testY": "failed", "UITests.go()": "flaky"}
+
+
+def test_parse_playwright(tmp_path):
+ flows = [
+ {"title": "one", "tests": [{"projectName": "chromium", "status": "flaky",
+ "results": [{"error": {"message": "\x1b[31mx\x1b[0m"}}]}]},
+ {"title": "two", "tests": [{"projectName": "chromium", "status": "unexpected", "results": []}]},
+ ]
+ report = {"suites": [{"title": "a.spec.ts", "specs": [], "suites": [{"title": "flows", "specs": flows}]}]}
+ path = tmp_path / "pw.json"
+ path.write_text(json.dumps(report))
+ got = flaky.parse_playwright([str(path)])
+ assert got["a.spec.ts > flows > one [chromium]"] == {"status": "flaky", "message": "x"}
+ assert got["a.spec.ts > flows > two [chromium]"]["status"] == "failed"
diff --git a/.github/workflows/ci-cua-spaces-macos.yml b/.github/workflows/ci-cua-spaces-macos.yml
index 58afb55906..b06e8fac8b 100644
--- a/.github/workflows/ci-cua-spaces-macos.yml
+++ b/.github/workflows/ci-cua-spaces-macos.yml
@@ -188,13 +188,19 @@ jobs:
- name: Stage the Cua Spaces app library
working-directory: libs/cua
run: ../spaces-app-swift/scripts/stage-library.sh
+ # swift test has no retry flag, so flaky.sh runs the failed tests once
+ # more. A test that passes only then is reported and gets a `flaky`
+ # issue (.github/flaky/).
- name: spaces-app-swift tests
- run: libs/spaces-app-swift/scripts/test.sh
+ run: |
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-app --format log --rerun-failed -- libs/spaces-app-swift/scripts/test.sh
- name: Notch package tests (the views and the Electron helper's protocol)
- run: libs/spaces-notch-swift/scripts/test.sh
+ run: |
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-notch --format log --rerun-failed -- libs/spaces-notch-swift/scripts/test.sh
- name: Swift tests (view models, parity, snapshots)
working-directory: apps/cua-spaces-macos
- run: scripts/test.sh
+ run: |
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-mac --format log --rerun-failed -- scripts/test.sh
# install-local.sh and uninstall-local.sh against a fake HOME and a fake
# build: a Stable install stays byte-identical, a wrong bundle id is
# refused. Needs no build, certificate or network.
@@ -218,6 +224,10 @@ jobs:
answers="$RUNNER_TEMP/bridge-answers.json"
(cd apps/cua-spaces-macos && CUA_BRIDGE_ANSWERS="$answers" scripts/test.sh --filter BridgeContractTests)
(cd apps/cua-spaces-web && CUA_BRIDGE_ANSWERS="$answers" npx vitest run src/bridge/__tests__/contract.test.ts)
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-swift
- name: Upload snapshot differences
if: failure()
uses: actions/upload-artifact@v4
@@ -343,5 +353,12 @@ jobs:
run: |
brew install xcodegen
xcodegen generate
- xcodebuild test -project CuaSpacesMac.xcodeproj -scheme CuaSpacesMacUITests \
- -destination 'platform=macOS' CODE_SIGNING_ALLOWED=NO
+ # -retry-tests-on-failure with 2 iterations: one retry.
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-xcuitest --format log -- \
+ xcodebuild test -project CuaSpacesMac.xcodeproj -scheme CuaSpacesMacUITests \
+ -destination 'platform=macOS' CODE_SIGNING_ALLOWED=NO \
+ -retry-tests-on-failure -test-iterations 2
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-xcuitest
diff --git a/.github/workflows/ci-cua-spaces-web.yml b/.github/workflows/ci-cua-spaces-web.yml
index 8f2c984b6b..72c08a40c9 100644
--- a/.github/workflows/ci-cua-spaces-web.yml
+++ b/.github/workflows/ci-cua-spaces-web.yml
@@ -22,6 +22,9 @@ on:
- "apps/cua-spaces-macos/Sources/CuaSpacesMacKit/WebHost/**"
- "libs/cua/crates/cua-spaces-app-core/**"
- ".github/workflows/ci-cua-spaces-web.yml"
+ - ".github/flaky/**"
+ - ".github/scripts/flaky/**"
+ - ".github/actions/flaky-upload/**"
push:
branches: [main]
paths:
@@ -77,17 +80,35 @@ jobs:
# Its own pnpm-workspace.yaml (allowBuilds: Electron's download); with
# --ignore-workspace pnpm skips it and fails on electron-winstaller.
(cd apps/cua-spaces-desktop && pnpm install --frozen-lockfile)
- - name: Web UI typecheck and tests (with the bridge contract)
+ - name: Web UI typecheck
working-directory: apps/cua-spaces-web
- run: pnpm check
+ run: pnpm typecheck
+ # One retry; a test that passes only on it is reported and gets a `flaky`
+ # issue (.github/flaky/). The bridge contract tests run here too.
+ - name: Web UI tests (with the bridge contract)
+ working-directory: apps/cua-spaces-web
+ env:
+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/web.json
+ run: |
+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \
+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjs
- name: Web UI build
working-directory: apps/cua-spaces-web
run: pnpm build
- name: Electron shell typecheck, tests (router contract) and build (channel check)
working-directory: apps/cua-spaces-desktop
+ env:
+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/desktop.json
run: |
- pnpm test
+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-desktop-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \
+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjs
pnpm build
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-web-test
parity:
name: parity flows (${{ matrix.host }})
@@ -125,9 +146,11 @@ jobs:
- name: Parity flows in the browser
if: matrix.host == 'browser'
working-directory: apps/cua-spaces-web
+ env:
+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-playwright/parity.json
run: |
npx playwright install --with-deps chromium
- pnpm parity
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-parity-browser --format playwright --results "$FLAKY_RESULTS_OUT" -- pnpm parity
# Layout shift of every route's first load (CUA-1951): fails at 0.05 or more, or a first render over its loose budget.
- name: Layout shift per route
if: matrix.host == 'browser'
@@ -136,9 +159,15 @@ jobs:
- name: Parity flows and the bridge on the Electron shell (demo mode)
if: matrix.host == 'electron'
working-directory: apps/cua-spaces-web
+ env:
+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-playwright/parity.json
run: |
sudo apt-get update && sudo apt-get install -y --no-install-recommends xvfb libnss3 libgbm1 libasound2t64
- xvfb-run -a pnpm parity:electron
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-parity-electron --format playwright --results "$FLAKY_RESULTS_OUT" -- xvfb-run -a pnpm parity:electron
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-web-parity-${{ matrix.host }}
# The parity table (parity-summary) and, for a failure, its traces.
- name: Upload the Playwright results
if: ${{ !cancelled() }}
diff --git a/.github/workflows/ci-cua-spaces.yml b/.github/workflows/ci-cua-spaces.yml
index e34c633b1f..a3dd725645 100644
--- a/.github/workflows/ci-cua-spaces.yml
+++ b/.github/workflows/ci-cua-spaces.yml
@@ -34,6 +34,10 @@ on:
- "apps/cua-spaces/**"
- "libs/cua/**"
- ".github/workflows/ci-cua-spaces.yml"
+ - ".github/flaky/**"
+ - ".github/scripts/flaky/**"
+ - ".github/actions/nextest/**"
+ - ".github/actions/flaky-upload/**"
push:
branches: [main]
paths:
@@ -166,9 +170,22 @@ jobs:
apps/cua-spaces/src-tauri -> target
libs/cua -> target
- run: pnpm install --frozen-lockfile
+ - uses: ./.github/actions/nextest
+ # One retry; a test that passes only on it is reported and gets a `flaky`
+ # issue (.github/flaky/).
- name: Rust command-layer tests (hermetic)
shell: bash
- run: cargo test --manifest-path src-tauri/Cargo.toml --locked --test host_commands --test commands --test installer_commands --test app_core_parity --lib -- --test-threads=4
+ run: >-
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-rust --format junit
+ --results src-tauri/target/nextest/ci/junit.xml --
+ cargo nextest run --manifest-path src-tauri/Cargo.toml --locked
+ --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci
+ --test host_commands --test commands --test installer_commands --test app_core_parity --lib
+ --test-threads 4
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-rust
frontend:
if: ${{ !startsWith(github.head_ref, 'release-please--branches--') }}
@@ -198,7 +215,17 @@ jobs:
cache-dependency-path: apps/cua-spaces/pnpm-lock.yaml
- run: pnpm install --frozen-lockfile
- run: pnpm typecheck
- - run: pnpm test
+ - name: Tests
+ env:
+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/spaces.json
+ run: |
+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \
+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjs
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-vitest
bundle:
name: bundle (${{ matrix.name }})
@@ -266,9 +293,22 @@ jobs:
libx11-dev libxkbcommon-dev libwayland-dev libxcb1-dev libx11-xcb-dev \
libxkbcommon-x11-dev libxrandr-dev libxi-dev libudev-dev
- run: pnpm install --frozen-lockfile
+ - uses: ./.github/actions/nextest
+ # One retry; a test that passes only on it is reported and gets a `flaky`
+ # issue (.github/flaky/).
- name: Rust command-layer tests (hermetic)
shell: bash
- run: cargo test --manifest-path src-tauri/Cargo.toml --locked --test host_commands --test commands --test installer_commands --test app_core_parity --lib -- --test-threads=4
+ run: >-
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spaces-rust-bundle-${{ matrix.name }}" --format junit
+ --results src-tauri/target/nextest/ci/junit.xml --
+ cargo nextest run --manifest-path src-tauri/Cargo.toml --locked
+ --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci
+ --test host_commands --test commands --test installer_commands --test app_core_parity --lib
+ --test-threads 4
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spaces-rust-bundle-${{ matrix.name }}
- name: Stage the cua CLI sidecar
shell: bash
# Not a hang: a cold release build of the CLI took 76-122 minutes on the
diff --git a/.github/workflows/ci-cua-spacesd.yml b/.github/workflows/ci-cua-spacesd.yml
index 0f7c456ff3..28fc2c5f47 100644
--- a/.github/workflows/ci-cua-spacesd.yml
+++ b/.github/workflows/ci-cua-spacesd.yml
@@ -191,8 +191,18 @@ jobs:
libxfixes-dev libxdamage-dev libudev-dev
- name: Format
run: cargo fmt --all --check
+ - uses: ./.github/actions/nextest
+ # One retry; a test that passes only on the retry is reported and gets a
+ # `flaky` issue (see .github/flaky/). Doctests are not run by nextest.
- name: Test
- run: cargo test --workspace --locked
+ run: >-
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-${{ matrix.os }}" --format junit
+ --results target/nextest/ci/junit.xml --
+ cargo nextest run --workspace --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci
+ env:
+ DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift
+ - name: Doctests
+ run: cargo test --workspace --doc --locked
env:
DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift
- name: Clippy
@@ -208,10 +218,18 @@ jobs:
rustfmt --edition 2021 --check account_erase.rs hosted.rs meter.rs org_access.rs plan_source.rs team_api.rs team_sharing.rs teams.rs
- name: Test (cua-relay hosted features)
run: |
- cargo test --locked -p cua-relay --features hosted
- cargo test --locked -p cua-spacesd-server --features relay-hosted --test mesh_metering
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-hosted-${{ matrix.os }}" --format junit \
+ --results target/nextest/ci/junit.xml -- \
+ cargo nextest run --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci -p cua-relay --features hosted
+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-mesh-metering-${{ matrix.os }}" --format junit \
+ --results target/nextest/ci/junit.xml -- \
+ cargo nextest run --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci -p cua-spacesd-server --features relay-hosted --test mesh_metering
env:
DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift
+ - uses: ./.github/actions/flaky-upload
+ if: always()
+ with:
+ suite: spacesd-${{ matrix.os }}
- name: Clippy (cua-relay hosted features)
run: cargo clippy --locked -p cua-relay -p cua-spacesd-server --all-targets --features cua-relay/hosted,cua-spacesd-server/relay-hosted -- -D warnings
env:
diff --git a/.github/workflows/ci-flaky-quarantine.yml b/.github/workflows/ci-flaky-quarantine.yml
new file mode 100644
index 0000000000..c0220c16b8
--- /dev/null
+++ b/.github/workflows/ci-flaky-quarantine.yml
@@ -0,0 +1,30 @@
+name: "CI: flaky quarantine"
+
+# Fails when an entry in .github/flaky/quarantine.json is invalid or past its
+# expiry date, so a quarantined test cannot stay quarantined for good. Runs on
+# every pull request (no path filter: an expired entry blocks every PR until
+# the test is fixed or the entry is renewed with a reason) and daily on main.
+# Also tests the flaky-test tooling (.github/scripts/flaky/flaky.py).
+
+on:
+ pull_request:
+ schedule:
+ - cron: "17 6 * * *"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ quarantine:
+ name: quarantine list and flaky tooling
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - name: Quarantine entries are valid and unexpired
+ run: python3 .github/scripts/flaky/flaky.py validate
+ - name: Flaky tooling tests
+ run: |
+ python3 -m pip install pytest
+ python3 -m pytest -q -p no:cacheprovider .github/scripts/tests/test_flaky.py
diff --git a/.github/workflows/flaky-issues.yml b/.github/workflows/flaky-issues.yml
new file mode 100644
index 0000000000..0b80649791
--- /dev/null
+++ b/.github/workflows/flaky-issues.yml
@@ -0,0 +1,55 @@
+name: "Flaky test issues"
+
+# Opens or updates one GitHub issue per flaky test (label `flaky`): a test that
+# failed and then passed on its automatic retry in one of the CI workflows
+# below. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload);
+# this workflow reads them with issues: write, on the default branch's copy of
+# the script, and never checks out or runs the pull request's code.
+
+on:
+ workflow_run:
+ workflows:
+ - "CI: cua-spacesd"
+ - "CI: Cua Spaces"
+ - "CI: Cua Spaces web UI and Electron shell"
+ - "CI: Cua Spaces for macOS (SwiftUI)"
+ types: [completed]
+
+permissions:
+ actions: read
+ contents: read
+ issues: write
+
+concurrency:
+ group: flaky-issues
+ cancel-in-progress: false
+
+jobs:
+ file:
+ name: file flaky-test issues
+ if: github.event.workflow_run.conclusion != 'cancelled'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ sparse-checkout: .github/scripts/flaky
+ # Most runs have no flaky tests, so no artifact: nothing to download.
+ - name: Download the run's flaky-test results
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ pattern: flaky-*
+ merge-multiple: true
+ path: flaky-results
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+ - name: Open or update the issues
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ if [ -d flaky-results ]; then
+ python3 .github/scripts/flaky/flaky.py file-issues --dir flaky-results
+ else
+ echo "no flaky-test results in this run"
+ fi
diff --git a/apps/cua-spaces-web/e2e/playwright.config.ts b/apps/cua-spaces-web/e2e/playwright.config.ts
index f51d02c402..264eecedd8 100644
--- a/apps/cua-spaces-web/e2e/playwright.config.ts
+++ b/apps/cua-spaces-web/e2e/playwright.config.ts
@@ -21,7 +21,14 @@ export default defineConfig({
// One shell per test; each is a full Electron app.
workers: ELECTRON ? Number(process.env.PARITY_WORKERS ?? 2) : undefined,
timeout: ELECTRON ? 90_000 : undefined,
- reporter: [["list"], ["./summary-reporter.ts"]],
+ // One retry in CI. A test that passes only on it is "flaky" in Playwright's
+ // report; CI lists it and opens an issue (.github/scripts/flaky/flaky.py).
+ retries: process.env.CI ? 1 : 0,
+ reporter: [
+ ["list"],
+ ["./summary-reporter.ts"],
+ ...(process.env.FLAKY_RESULTS_OUT ? [["json", { outputFile: process.env.FLAKY_RESULTS_OUT }] as const] : []),
+ ],
use: {
baseURL: `http://localhost:${PORT}`,
viewport: { width: 1440, height: 900 },