Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci: retry flaky tests once, report them, add a quarantine list (CUA-2336) (#5996)
cua-agents[bot]1e6db4557075
diff --git a/.github/actions/flaky-upload/action.yml b/.github/actions/flaky-upload/action.ymlnew file mode 100644index 0000000000..62a59b54f3--- /dev/null+++ b/.github/actions/flaky-upload/action.yml@@ -0,0 +1,15 @@+name: Upload flaky-test results+description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml to turn into GitHub issues. Run with if: always()."+inputs:+ suite:+ description: "The suite name given to flaky.py run (unique per job, matrix legs included)."+ required: true+runs:+ using: composite+ steps:+ - uses: actions/upload-artifact@v4+ with:+ name: flaky-${{ inputs.suite }}+ path: ${{ runner.temp }}/flaky-results/+ if-no-files-found: ignore+ retention-days: 3diff --git a/.github/actions/nextest/action.yml b/.github/actions/nextest/action.ymlnew file mode 100644index 0000000000..75efc7f4a4--- /dev/null+++ b/.github/actions/nextest/action.yml@@ -0,0 +1,8 @@+name: Install cargo-nextest+description: "cargo-nextest, for the CI profile in .github/flaky/nextest.toml (one retry, JUnit output)."+runs:+ using: composite+ steps:+ - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31+ with:+ tool: cargo-nextestdiff --git a/.github/flaky/nextest.toml b/.github/flaky/nextest.tomlnew file mode 100644index 0000000000..1ea3bdf226--- /dev/null+++ b/.github/flaky/nextest.toml@@ -0,0 +1,11 @@+# cargo nextest profile for CI. One retry; a test that passes only on the+# retry is listed as <flakyFailure> in the JUnit file, which+# .github/scripts/flaky/flaky.py turns into a CI summary, an annotation and a+# GitHub issue. Use: cargo nextest run --config-file <this file> --profile ci+[profile.ci]+retries = 1+fail-fast = false+failure-output = "immediate-final"++[profile.ci.junit]+path = "junit.xml"diff --git a/.github/flaky/quarantine.json b/.github/flaky/quarantine.jsonnew file mode 100644index 0000000000..03f87f3190--- /dev/null+++ b/.github/flaky/quarantine.json@@ -0,0 +1,4 @@+{+ "$comment": "Known-flaky tests that may fail without blocking CI. Add an entry only with an owner, an issue and an expiry date. CI fails when an entry expires (ci-flaky-quarantine). See .github/scripts/flaky/flaky.py.",+ "entries": []+}diff --git a/.github/flaky/vitest-reporter.mjs b/.github/flaky/vitest-reporter.mjsnew file mode 100644index 0000000000..777cf4a3f7--- /dev/null+++ b/.github/flaky/vitest-reporter.mjs@@ -0,0 +1,31 @@+// Vitest reporter for CI flaky-test handling (.github/scripts/flaky/flaky.py).+// Writes one row per test to $FLAKY_RESULTS_OUT: "flaky" when the test passed+// only on a retry, "failed" when it failed after its retries. Does nothing+// when FLAKY_RESULTS_OUT is unset, so local runs are unchanged.+import { mkdirSync, writeFileSync } from "node:fs";+import { dirname, relative } from "node:path";++export default class FlakyReporter {+ rows = [];++ onTestCaseResult(testCase) {+ const result = testCase.result();+ if (result.state !== "passed" && result.state !== "failed") return;+ const retried = (testCase.diagnostic()?.retryCount ?? 0) > 0;+ const status = result.state === "failed" ? "failed" : retried ? "flaky" : "passed";+ if (status === "passed") return;+ const file = relative(process.cwd(), testCase.module.moduleId);+ this.rows.push({+ id: `${file} > ${testCase.fullName}`,+ status,+ message: String(result.errors?.[0]?.message ?? "").slice(0, 1500),+ });+ }++ onTestRunEnd() {+ const out = process.env.FLAKY_RESULTS_OUT;+ if (!out) return;+ mkdirSync(dirname(out), { recursive: true });+ writeFileSync(out, JSON.stringify(this.rows));+ }+}diff --git a/.github/scripts/flaky/flaky.py b/.github/scripts/flaky/flaky.pynew file mode 100755index 0000000000..b21173a670--- /dev/null+++ b/.github/scripts/flaky/flaky.py@@ -0,0 +1,457 @@+#!/usr/bin/env python3+"""Flaky-test handling for CI: report, quarantine, and file issues.++The test runners retry once themselves (cargo nextest `retries = 1`, vitest+`retry`, Playwright `retries`, xcodebuild `-retry-tests-on-failure`). This tool+wraps the runner and reads its result file:++ run run a test command, then read its results. A test that passed only+ on the retry is reported (job summary, annotation, JSON artifact),+ never hidden. A failure that is not in the quarantine list still+ fails the job. If every failing test is in the quarantine list and+ not expired, the job passes with a warning.+ validate check .github/flaky/quarantine.json; exit 1 on an invalid or+ expired entry.+ file-issues+ open or update one GitHub issue per flaky test (label `flaky`)+ from the JSON artifacts that `run` wrote.++Result formats (--format): junit (cargo nextest), vitest (the reporter in+.github/flaky/vitest-reporter.mjs), playwright (the JSON reporter), log (swift+test and xcodebuild console output).+"""++from __future__ import annotations++import argparse+import datetime as dt+import fnmatch+import glob+import hashlib+import json+import os+import re+import subprocess+import sys+import xml.etree.ElementTree as ET+from pathlib import Path+from typing import Dict, Iterable, List, Optional, Tuple++QUARANTINE_FILE = Path(__file__).resolve().parents[2] / "flaky" / "quarantine.json"+MAX_QUARANTINE_DAYS = 90+ISSUE_RE = re.compile(r"^(CUA-\d+|#\d+|https://github\.com/[\w.-]+/[\w.-]+/issues/\d+|https://linear\.app/\S+)$")+RANK = {"skipped": 0, "passed": 1, "flaky": 2, "failed": 3}++Results = Dict[str, dict]+++def merge(into: Results, test_id: str, status: str, message: str = "") -> None:+ old = into.get(test_id)+ if old is None or RANK[status] > RANK[old["status"]]:+ into[test_id] = {"status": status, "message": message}+++# --- parsers ---------------------------------------------------------------+++def parse_junit(paths: Iterable[str]) -> Results:+ out: Results = {}+ for path in paths:+ for case in ET.parse(path).getroot().iter("testcase"):+ cls, name = case.get("classname", ""), case.get("name", "")+ test_id = f"{cls}::{name}" if cls and cls != name else name+ tags = {child.tag: child for child in case}+ if "failure" in tags or "error" in tags:+ node = tags.get("failure", tags.get("error"))+ merge(out, test_id, "failed", (node.get("message") or node.text or "").strip())+ elif "flakyFailure" in tags or "flakyError" in tags:+ node = tags.get("flakyFailure", tags.get("flakyError"))+ merge(out, test_id, "flaky", (node.get("message") or node.text or "").strip())+ elif "skipped" in tags:+ merge(out, test_id, "skipped")+ else:+ merge(out, test_id, "passed")+ return out+++def parse_vitest(paths: Iterable[str]) -> Results:+ out: Results = {}+ for path in paths:+ for row in json.loads(Path(path).read_text()):+ merge(out, row["id"], row["status"], row.get("message", ""))+ return out+++def parse_playwright(paths: Iterable[str]) -> Results:+ out: Results = {}+ status_map = {"expected": "passed", "unexpected": "failed", "flaky": "flaky", "skipped": "skipped"}++ def walk(suite: dict, trail: List[str]) -> None:+ here = trail + ([suite["title"]] if suite.get("title") else [])+ for spec in suite.get("specs", []):+ for test in spec.get("tests", []):+ project = test.get("projectName", "")+ test_id = " > ".join(here + [spec["title"]]) + (f" [{project}]" if project else "")+ message = ""+ for result in test.get("results", []):+ err = (result.get("error") or {}).get("message")+ if err:+ message = re.sub(r"\x1b\[[0-9;]*m", "", err)+ break+ merge(out, test_id, status_map.get(test.get("status", ""), "passed"), message)+ for child in suite.get("suites", []):+ walk(child, here)++ for path in paths:+ for suite in json.loads(Path(path).read_text()).get("suites", []):+ walk(suite, [])+ return out+++LOG_PATTERNS = [+ # Swift Testing: "✘ Test realFail() failed after 0.001 seconds"+ (re.compile(r"^[✔✘] Test (?!run with )(.+?) (passed|failed) after"), lambda m: m.group(1)),+ # XCTest: "Test Case '-[LTests.X testY]' failed (0.1 seconds)."+ (re.compile(r"^Test Case '-\[(\S+) (\S+)\]' (passed|failed)"), lambda m: f"{m.group(1)}/{m.group(2)}"),+ # xcodebuild: "Test case 'X.y()' failed on 'My Mac - app (123)' (0.1 seconds)"+ (re.compile(r"^Test case '(.+?)' (passed|failed) on "), lambda m: m.group(1)),+]+++def parse_log(lines: Iterable[str]) -> Results:+ """A test that failed and later passed in the same log is flaky."""+ seen: Dict[str, List[str]] = {}+ for line in lines:+ line = line.strip()+ for pattern, ident in LOG_PATTERNS:+ m = pattern.match(line)+ if m:+ seen.setdefault(ident(m), []).append(m.group(m.lastindex))+ break+ out: Results = {}+ for test_id, seq in seen.items():+ if seq[-1] == "failed":+ status = "failed"+ elif "failed" in seq:+ status = "flaky"+ else:+ status = "passed"+ out[test_id] = {"status": status, "message": ""}+ return out+++# --- quarantine --------------------------------------------------------------+++def load_quarantine() -> List[dict]:+ return json.loads(QUARANTINE_FILE.read_text()).get("entries", [])+++def check_entries(entries: List[dict], today: dt.date) -> Tuple[List[str], List[str]]:+ """Return (errors, expired). Both make `validate` fail."""+ errors: List[str] = []+ expired: List[str] = []+ for i, entry in enumerate(entries):+ label = entry.get("test", f"entry {i}")+ for key in ("test", "owner", "issue", "expires"):+ if not str(entry.get(key, "")).strip():+ errors.append(f"{label}: missing `{key}`")+ if entry.get("issue") and not ISSUE_RE.match(entry["issue"]):+ errors.append(f"{label}: `issue` must be CUA-123, #123, or an issue URL, got {entry['issue']!r}")+ try:+ day = dt.date.fromisoformat(entry.get("expires", ""))+ except ValueError:+ errors.append(f"{label}: `expires` must be YYYY-MM-DD, got {entry.get('expires')!r}")+ continue+ if day < today:+ expired.append(f"{label}: expired on {day} (owner {entry.get('owner')}, issue {entry.get('issue')})")+ elif (day - today).days > MAX_QUARANTINE_DAYS:+ errors.append(f"{label}: `expires` is more than {MAX_QUARANTINE_DAYS} days away")+ return errors, expired+++def active_entries(entries: List[dict], today: dt.date) -> List[dict]:+ good = []+ for entry in entries:+ try:+ if dt.date.fromisoformat(entry["expires"]) >= today and entry.get("owner") and entry.get("issue"):+ good.append(entry)+ except (KeyError, ValueError):+ pass+ return good+++def is_quarantined(test_id: str, suite: str, entries: List[dict]) -> Optional[dict]:+ for entry in entries:+ if not (fnmatch.fnmatchcase(test_id, entry["test"]) or test_id.endswith("::" + entry["test"])):+ continue+ if entry.get("suite") and not fnmatch.fnmatchcase(suite, entry["suite"]):+ continue+ return entry+ return None+++# --- run -----------------------------------------------------------------+++def stream(cmd: List[str]) -> Tuple[int, List[str]]:+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace")+ lines: List[str] = []+ assert proc.stdout is not None+ for line in proc.stdout:+ sys.stdout.write(line)+ lines.append(line)+ return proc.wait(), lines+++def run_metadata() -> dict:+ meta = {+ "repo": os.environ.get("GITHUB_REPOSITORY", ""),+ "run_id": os.environ.get("GITHUB_RUN_ID", ""),+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", ""),+ "workflow": os.environ.get("GITHUB_WORKFLOW", ""),+ "job": os.environ.get("GITHUB_JOB", ""),+ "ref": os.environ.get("GITHUB_REF", ""),+ "sha": os.environ.get("GITHUB_SHA", ""),+ "pr": None,+ }+ event_path = os.environ.get("GITHUB_EVENT_PATH")+ if event_path and Path(event_path).is_file():+ pr = json.loads(Path(event_path).read_text()).get("pull_request")+ if pr:+ meta["pr"] = pr.get("number")+ meta["sha"] = pr.get("head", {}).get("sha", meta["sha"])+ return meta+++def cell(text: str) -> str:+ return text.replace("|", "\\|").replace("\n", " ")[:200]+++def cmd_run(args: argparse.Namespace) -> int:+ command = args.command[1:] if args.command[:1] == ["--"] else args.command+ if not command:+ print("flaky.py run: no command after --", file=sys.stderr)+ return 2+ today = dt.date.today()+ entries = load_quarantine()+ bad, expired = check_entries(entries, today)+ live = active_entries(entries, today)+ for problem in bad + expired:+ print(f"::warning title=Quarantine list::{problem}")++ for stale in (p for pattern in args.results for p in glob.glob(pattern, recursive=True)):+ os.remove(stale) # a leftover file from an earlier run must not be read as this run's+ code, lines = stream(command)+ paths = sorted(p for pattern in args.results for p in glob.glob(pattern, recursive=True))+ try:+ if args.format == "log":+ results = parse_log(lines)+ failed_first = sorted(t for t, r in results.items() if r["status"] == "failed")+ if code != 0 and failed_first and args.rerun_failed:+ # swift test has no retry flag: run the failures once more.+ rerun_cmd = list(command)+ for test_id in failed_first:+ rerun_cmd += ["--filter", re.escape(test_id)]+ print(f"::notice title=Retry::re-running {len(failed_first)} failed test(s) once")+ code, lines2 = stream(rerun_cmd)+ second = parse_log(lines2)+ for test_id in failed_first:+ after = second.get(test_id, {"status": "failed"})["status"]+ results[test_id]["status"] = "flaky" if after == "passed" else "failed"+ elif args.format == "junit":+ results = parse_junit(paths)+ elif args.format == "vitest":+ results = parse_vitest(paths)+ else:+ results = parse_playwright(paths)+ except (OSError, ValueError, KeyError, ET.ParseError) as exc:+ print(f"::warning title=Flaky report::could not read results ({exc}); exit code {code} stands")+ return code++ flaky = {t: r for t, r in results.items() if r["status"] == "flaky"}+ failed = {t: r for t, r in results.items() if r["status"] == "failed"}+ quarantined = {t: r for t, r in failed.items() if is_quarantined(t, args.suite, live)}+ real = {t: r for t, r in failed.items() if t not in quarantined}++ for test_id, r in flaky.items():+ print(f"::warning title=Flaky test ({args.suite})::{test_id} passed only on retry")+ for test_id in quarantined:+ entry = is_quarantined(test_id, args.suite, live)+ print(+ f"::warning title=Quarantined test failed ({args.suite})::{test_id} "+ f"(owner {entry['owner']}, {entry['issue']}, expires {entry['expires']})"+ )+ for test_id in real:+ print(f"::error title=Test failed ({args.suite})::{test_id}")++ write_summary(args.suite, results, flaky, quarantined, real, live)+ write_artifact(args.suite, flaky, quarantined)++ if code == 0:+ return 0+ # The runner failed. Pass only if it failed on quarantined tests and nothing else.+ if quarantined and not real:+ print(f"::warning title=Quarantine::{len(quarantined)} quarantined test(s) failed; not blocking")+ return 0+ return code+++def write_summary(suite: str, results: Results, flaky: Results, quarantined: Results, real: Results, live: List[dict]) -> None:+ path = os.environ.get("GITHUB_STEP_SUMMARY")+ if not path or not (flaky or quarantined or real):+ return+ out = [f"### Test results: {suite}", ""]+ counts = {s: sum(1 for r in results.values() if r["status"] == s) for s in RANK}+ out.append(+ f"{counts['passed']} passed, {len(flaky)} flaky (passed only on retry), "+ f"{len(real)} failed, {len(quarantined)} quarantined failures, {counts['skipped']} skipped."+ )+ if flaky:+ out += ["", "**Flaky: passed only on retry. A `flaky` issue is opened or updated for each.**", "", "| Test | First failure |", "|---|---|"]+ out += [f"| `{cell(t)}` | {cell(r['message'])} |" for t, r in sorted(flaky.items())]+ if quarantined:+ out += ["", "**Quarantined: failed but not blocking.**", "", "| Test | Owner | Issue | Expires |", "|---|---|---|---|"]+ for t in sorted(quarantined):+ e = is_quarantined(t, suite, live)+ out.append(f"| `{cell(t)}` | {e['owner']} | {e['issue']} | {e['expires']} |")+ if real:+ out += ["", "**Failed (blocking).**", "", "| Test | Failure |", "|---|---|"]+ out += [f"| `{cell(t)}` | {cell(r['message'])} |" for t, r in sorted(real.items())]+ with open(path, "a") as fh:+ fh.write("\n".join(out) + "\n")+++def write_artifact(suite: str, flaky: Results, quarantined: Results) -> None:+ base = os.environ.get("RUNNER_TEMP")+ if not base or not (flaky or quarantined):+ return+ out_dir = Path(base) / "flaky-results"+ out_dir.mkdir(parents=True, exist_ok=True)+ payload = dict(run_metadata(), suite=suite, flaky=[dict(id=t, message=r["message"][:1500]) for t, r in flaky.items()],+ quarantined=sorted(quarantined))+ (out_dir / f"{re.sub(r'[^A-Za-z0-9_.-]', '_', suite)}.json").write_text(json.dumps(payload, indent=2))+++# --- validate ----------------------------------------------------------------+++def cmd_validate(args: argparse.Namespace) -> int:+ today = dt.date.fromisoformat(args.today) if args.today else dt.date.today()+ try:+ entries = load_quarantine()+ except (OSError, ValueError) as exc:+ print(f"::error::cannot read {QUARANTINE_FILE}: {exc}")+ return 1+ errors, expired = check_entries(entries, today)+ for problem in errors:+ print(f"::error title=Quarantine entry invalid::{problem}")+ for problem in expired:+ print(f"::error title=Quarantine entry expired::{problem}. Fix the test and remove the entry, or renew it with a reason.")+ print(f"{len(entries)} quarantine entr{'y' if len(entries) == 1 else 'ies'}, {len(errors)} invalid, {len(expired)} expired")+ return 1 if errors or expired else 0+++# --- issues ----------------------------------------------------------------+++def clean(text: str, limit: int) -> str:+ text = re.sub(r"[\x00-\x08\x0b-\x1f\x7f]", "", text)+ return text.replace("@", "@\u200b")[:limit]+++def gh(*args: str, check: bool = True) -> str:+ done = subprocess.run(["gh", *args], capture_output=True, text=True)+ if check and done.returncode != 0:+ raise RuntimeError(f"gh {' '.join(args[:3])} failed: {done.stderr.strip()}")+ return done.stdout+++def flaky_key(test_id: str) -> str:+ return hashlib.sha1(test_id.encode()).hexdigest()[:12]+++def cmd_file_issues(args: argparse.Namespace) -> int:+ repo = os.environ["GITHUB_REPOSITORY"]+ server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")+ reports = [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{args.dir}/**/*.json", recursive=True))]+ items: Dict[str, List[Tuple[dict, dict]]] = {}+ for report in reports:+ for row in report.get("flaky", []):+ items.setdefault(row["id"], []).append((report, row))+ if not items:+ print("no flaky tests reported")+ return 0+ if args.dry_run:+ for test_id in items:+ print(f"would file: Flaky test: {clean(test_id, 120)}")+ return 0+ gh("label", "create", "flaky", "--repo", repo, "--color", "FBCA04", "--description", "Test that passed only on retry", check=False)+ existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "all", "--limit", "1000",+ "--json", "number,body,state") or "[]")+ by_key = {}+ for issue in existing:+ m = re.search(r"<!-- flaky-key:(\w+) -->", issue.get("body") or "")+ if m:+ by_key[m.group(1)] = issue+ for test_id, occurrences in items.items():+ key = flaky_key(test_id)+ report, row = occurrences[0]+ run_url = f"{server}/{repo}/actions/runs/{report['run_id']}"+ entry = [+ f"Passed only on retry in `{clean(report['suite'], 80)}`.",+ "",+ f"- Run: {run_url} (attempt {report['run_attempt']}, workflow {clean(report['workflow'], 80)}, job {clean(report['job'], 80)})",+ f"- Commit: `{report['sha'][:12]}`" + (f", PR #{report['pr']}" if report.get("pr") else f", ref `{clean(report['ref'], 80)}`"),+ "",+ "```",+ clean(row.get("message", "") or "(no message)", 1200).replace("```", "'''"),+ "```",+ ]+ issue = by_key.get(key)+ if issue is None:+ body = "\n".join([+ f"`{clean(test_id, 300)}` passes only on retry in CI. Fix the race, or quarantine it in "+ "`.github/flaky/quarantine.json` with an owner, this issue and an expiry date.",+ "",+ "Failure history is in the comments below, one per CI run.",+ "",+ f"<!-- flaky-key:{key} -->",+ ])+ url = gh("issue", "create", "--repo", repo, "--label", "flaky",+ "--title", f"Flaky test: {clean(test_id, 200)}", "--body", body).strip()+ number = url.rsplit("/", 1)[-1]+ print(f"opened {url}")+ else:+ number = str(issue["number"])+ have = gh("issue", "view", number, "--repo", repo, "--json", "comments", "--jq", ".comments[].body")+ if f"/actions/runs/{report['run_id']}" in have:+ print(f"#{number}: run {report['run_id']} already recorded")+ continue+ if issue["state"] == "CLOSED":+ gh("issue", "reopen", number, "--repo", repo)+ gh("issue", "comment", number, "--repo", repo, "--body", "\n".join(entry))+ print(f"#{number}: recorded {run_url}")+ return 0+++def main(argv: Optional[List[str]] = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ sub = parser.add_subparsers(dest="cmd", required=True)+ run = sub.add_parser("run")+ run.add_argument("--suite", required=True)+ run.add_argument("--format", choices=["junit", "vitest", "playwright", "log"], required=True)+ run.add_argument("--results", action="append", default=[], help="result file glob (repeatable)")+ run.add_argument("--rerun-failed", action="store_true", help="log format: run the failed tests once more with --filter")+ run.add_argument("command", nargs=argparse.REMAINDER)+ validate = sub.add_parser("validate")+ validate.add_argument("--today")+ issues = sub.add_parser("file-issues")+ issues.add_argument("--dir", required=True)+ issues.add_argument("--dry-run", action="store_true")+ args = parser.parse_args(argv)+ return {"run": cmd_run, "validate": cmd_validate, "file-issues": cmd_file_issues}[args.cmd](args)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/.github/scripts/flaky/flaky.sh b/.github/scripts/flaky/flaky.shnew file mode 100755index 0000000000..db2adedeb8--- /dev/null+++ b/.github/scripts/flaky/flaky.sh@@ -0,0 +1,6 @@+#!/usr/bin/env bash+# Runs flaky.py with whichever Python the runner has (python3 on Linux and+# macOS, python on some Windows images).+set -euo pipefail+py=$(command -v python3 || command -v python) || { echo "no python on this runner" >&2; exit 2; }+exec "$py" "$(dirname "${BASH_SOURCE[0]}")/flaky.py" "$@"diff --git a/.github/scripts/tests/test_flaky.py b/.github/scripts/tests/test_flaky.pynew file mode 100644index 0000000000..4fd59afebc--- /dev/null+++ b/.github/scripts/tests/test_flaky.py@@ -0,0 +1,128 @@+import datetime as dt+import importlib.util+import json+import sys+import textwrap+from pathlib import Path++import pytest++SCRIPT = Path(__file__).resolve().parents[1] / "flaky" / "flaky.py"+spec = importlib.util.spec_from_file_location("flaky", SCRIPT)+flaky = importlib.util.module_from_spec(spec)+sys.modules["flaky"] = flaky+spec.loader.exec_module(flaky)++TODAY = dt.date(2026, 10, 25)+JUNIT = textwrap.dedent(+ """\+ <testsuites><testsuite name="pkg::bin">+ <testcase classname="pkg::bin" name="ok"/>+ <testcase classname="pkg::bin" name="racy"><flakyFailure message="timed out"/></testcase>+ <testcase classname="pkg::bin" name="broken"><failure message="boom"/><rerunFailure message="boom"/></testcase>+ <testcase classname="pkg::bin" name="skip"><skipped/></testcase>+ </testsuite></testsuites>+ """+)+++def entry(**kw):+ base = {"test": "pkg::bin::broken", "owner": "francesco", "issue": "CUA-1", "expires": "2026-11-01"}+ base.update(kw)+ return base+++def env(tmp_path, monkeypatch):+ quarantine = tmp_path / "quarantine.json"+ quarantine.write_text(json.dumps({"entries": []}))+ monkeypatch.setattr(flaky, "QUARANTINE_FILE", quarantine)+ monkeypatch.setenv("RUNNER_TEMP", str(tmp_path))+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "summary.md"))+ monkeypatch.setattr(flaky.dt, "date", type("D", (dt.date,), {"today": staticmethod(lambda: TODAY)}))+ return tmp_path+++def run(env, *extra, exit_code=1, junit=JUNIT):+ """Run a fake test command that writes `junit` and exits with `exit_code`."""+ out = env / "junit.xml"+ script = f"import pathlib; pathlib.Path({str(out)!r}).write_text({junit!r}); raise SystemExit({exit_code})"+ return flaky.main(["run", "--suite", "t", "--format", "junit", "--results", str(out), *extra, "--",+ sys.executable, "-c", script])+++def test_parse_junit_statuses(env):+ (env / "junit.xml").write_text(JUNIT)+ got = flaky.parse_junit([str(env / "junit.xml")])+ assert {k: v["status"] for k, v in got.items()} == {+ "pkg::bin::ok": "passed", "pkg::bin::racy": "flaky", "pkg::bin::broken": "failed", "pkg::bin::skip": "skipped"}+++def test_real_failure_still_fails_and_flaky_is_reported(env):+ assert run(env) == 1+ assert "pkg::bin::racy" in (env / "summary.md").read_text()+ report = json.loads((env / "flaky-results" / "t.json").read_text())+ assert [row["id"] for row in report["flaky"]] == ["pkg::bin::racy"]+++def test_flaky_only_passes_with_report(env):+ assert run(env, exit_code=0, junit=JUNIT.replace('<failure message="boom"/><rerunFailure message="boom"/>', "")) == 0+ assert (env / "flaky-results" / "t.json").exists()+++def test_quarantined_failure_does_not_block(env):+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry()]}))+ assert run(env) == 0+++def test_expired_quarantine_blocks(env):+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(expires="2026-10-01")]}))+ assert run(env) == 1+++def test_quarantine_does_not_hide_other_failures(env):+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(test="pkg::bin::other")]}))+ assert run(env) == 1+++def test_failure_without_parsed_tests_fails(env):+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry()]}))+ assert run(env, junit="<testsuites/>") == 1+++def test_validate(env):+ assert flaky.check_entries([entry()], TODAY) == ([], [])+ errors, expired = flaky.check_entries([entry(owner="", issue="soon"), entry(expires="2026-10-24")], TODAY)+ assert len(errors) == 2 and len(expired) == 1+ assert flaky.check_entries([entry(expires="2027-06-01")], TODAY)[0]+ assert flaky.main(["validate", "--today", "2026-10-25"]) == 0+ flaky.QUARANTINE_FILE.write_text(json.dumps({"entries": [entry(expires="2026-10-01")]}))+ assert flaky.main(["validate", "--today", "2026-10-25"]) == 1+++def test_parse_log_swift_and_xcodebuild():+ lines = [+ "✘ Test racy() failed after 0.001 seconds with 1 issue.",+ "✔ Test racy() passed after 0.001 seconds.",+ "✘ Test broken() failed after 0.001 seconds with 1 issue.",+ "✘ Test run with 3 tests in 1 suite failed after 0.001 seconds with 2 issues.",+ "Test Case '-[LTests.X testY]' failed (0.1 seconds).",+ "Test case 'UITests.go()' failed on 'My Mac - App (1)' (1.0 seconds)",+ "Test case 'UITests.go()' passed on 'My Mac - App (1)' (1.0 seconds)",+ ]+ got = {k: v["status"] for k, v in flaky.parse_log(lines).items()}+ assert got == {"racy()": "flaky", "broken()": "failed", "LTests.X/testY": "failed", "UITests.go()": "flaky"}+++def test_parse_playwright(tmp_path):+ flows = [+ {"title": "one", "tests": [{"projectName": "chromium", "status": "flaky",+ "results": [{"error": {"message": "\x1b[31mx\x1b[0m"}}]}]},+ {"title": "two", "tests": [{"projectName": "chromium", "status": "unexpected", "results": []}]},+ ]+ report = {"suites": [{"title": "a.spec.ts", "specs": [], "suites": [{"title": "flows", "specs": flows}]}]}+ path = tmp_path / "pw.json"+ path.write_text(json.dumps(report))+ got = flaky.parse_playwright([str(path)])+ assert got["a.spec.ts > flows > one [chromium]"] == {"status": "flaky", "message": "x"}+ assert got["a.spec.ts > flows > two [chromium]"]["status"] == "failed"diff --git a/.github/workflows/ci-cua-spaces-macos.yml b/.github/workflows/ci-cua-spaces-macos.ymlindex 58afb55906..b06e8fac8b 100644--- a/.github/workflows/ci-cua-spaces-macos.yml+++ b/.github/workflows/ci-cua-spaces-macos.yml@@ -188,13 +188,19 @@ jobs:- name: Stage the Cua Spaces app libraryworking-directory: libs/cuarun: ../spaces-app-swift/scripts/stage-library.sh+ # swift test has no retry flag, so flaky.sh runs the failed tests once+ # more. A test that passes only then is reported and gets a `flaky`+ # issue (.github/flaky/).- name: spaces-app-swift tests- run: libs/spaces-app-swift/scripts/test.sh+ run: |+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-app --format log --rerun-failed -- libs/spaces-app-swift/scripts/test.sh- name: Notch package tests (the views and the Electron helper's protocol)- run: libs/spaces-notch-swift/scripts/test.sh+ run: |+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-notch --format log --rerun-failed -- libs/spaces-notch-swift/scripts/test.sh- name: Swift tests (view models, parity, snapshots)working-directory: apps/cua-spaces-macos- run: scripts/test.sh+ run: |+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-swift-mac --format log --rerun-failed -- scripts/test.sh# install-local.sh and uninstall-local.sh against a fake HOME and a fake# build: a Stable install stays byte-identical, a wrong bundle id is# refused. Needs no build, certificate or network.@@ -218,6 +224,10 @@ jobs:answers="$RUNNER_TEMP/bridge-answers.json"(cd apps/cua-spaces-macos && CUA_BRIDGE_ANSWERS="$answers" scripts/test.sh --filter BridgeContractTests)(cd apps/cua-spaces-web && CUA_BRIDGE_ANSWERS="$answers" npx vitest run src/bridge/__tests__/contract.test.ts)+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-swift- name: Upload snapshot differencesif: failure()uses: actions/upload-artifact@v4@@ -343,5 +353,12 @@ jobs:run: |brew install xcodegenxcodegen generate- xcodebuild test -project CuaSpacesMac.xcodeproj -scheme CuaSpacesMacUITests \- -destination 'platform=macOS' CODE_SIGNING_ALLOWED=NO+ # -retry-tests-on-failure with 2 iterations: one retry.+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-xcuitest --format log -- \+ xcodebuild test -project CuaSpacesMac.xcodeproj -scheme CuaSpacesMacUITests \+ -destination 'platform=macOS' CODE_SIGNING_ALLOWED=NO \+ -retry-tests-on-failure -test-iterations 2+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-xcuitestdiff --git a/.github/workflows/ci-cua-spaces-web.yml b/.github/workflows/ci-cua-spaces-web.ymlindex 8f2c984b6b..72c08a40c9 100644--- a/.github/workflows/ci-cua-spaces-web.yml+++ b/.github/workflows/ci-cua-spaces-web.yml@@ -22,6 +22,9 @@ on:- "apps/cua-spaces-macos/Sources/CuaSpacesMacKit/WebHost/**"- "libs/cua/crates/cua-spaces-app-core/**"- ".github/workflows/ci-cua-spaces-web.yml"+ - ".github/flaky/**"+ - ".github/scripts/flaky/**"+ - ".github/actions/flaky-upload/**"push:branches: [main]paths:@@ -77,17 +80,35 @@ jobs:# Its own pnpm-workspace.yaml (allowBuilds: Electron's download); with# --ignore-workspace pnpm skips it and fails on electron-winstaller.(cd apps/cua-spaces-desktop && pnpm install --frozen-lockfile)- - name: Web UI typecheck and tests (with the bridge contract)+ - name: Web UI typecheckworking-directory: apps/cua-spaces-web- run: pnpm check+ run: pnpm typecheck+ # One retry; a test that passes only on it is reported and gets a `flaky`+ # issue (.github/flaky/). The bridge contract tests run here too.+ - name: Web UI tests (with the bridge contract)+ working-directory: apps/cua-spaces-web+ env:+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/web.json+ run: |+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjs- name: Web UI buildworking-directory: apps/cua-spaces-webrun: pnpm build- name: Electron shell typecheck, tests (router contract) and build (channel check)working-directory: apps/cua-spaces-desktop+ env:+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/desktop.jsonrun: |- pnpm test+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-desktop-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjspnpm build+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-web-testparity:name: parity flows (${{ matrix.host }})@@ -125,9 +146,11 @@ jobs:- name: Parity flows in the browserif: matrix.host == 'browser'working-directory: apps/cua-spaces-web+ env:+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-playwright/parity.jsonrun: |npx playwright install --with-deps chromium- pnpm parity+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-parity-browser --format playwright --results "$FLAKY_RESULTS_OUT" -- pnpm parity# Layout shift of every route's first load (CUA-1951): fails at 0.05 or more, or a first render over its loose budget.- name: Layout shift per routeif: matrix.host == 'browser'@@ -136,9 +159,15 @@ jobs:- name: Parity flows and the bridge on the Electron shell (demo mode)if: matrix.host == 'electron'working-directory: apps/cua-spaces-web+ env:+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-playwright/parity.jsonrun: |sudo apt-get update && sudo apt-get install -y --no-install-recommends xvfb libnss3 libgbm1 libasound2t64- xvfb-run -a pnpm parity:electron+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-web-parity-electron --format playwright --results "$FLAKY_RESULTS_OUT" -- xvfb-run -a pnpm parity:electron+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-web-parity-${{ matrix.host }}# The parity table (parity-summary) and, for a failure, its traces.- name: Upload the Playwright resultsif: ${{ !cancelled() }}diff --git a/.github/workflows/ci-cua-spaces.yml b/.github/workflows/ci-cua-spaces.ymlindex e34c633b1f..a3dd725645 100644--- a/.github/workflows/ci-cua-spaces.yml+++ b/.github/workflows/ci-cua-spaces.yml@@ -34,6 +34,10 @@ on:- "apps/cua-spaces/**"- "libs/cua/**"- ".github/workflows/ci-cua-spaces.yml"+ - ".github/flaky/**"+ - ".github/scripts/flaky/**"+ - ".github/actions/nextest/**"+ - ".github/actions/flaky-upload/**"push:branches: [main]paths:@@ -166,9 +170,22 @@ jobs:apps/cua-spaces/src-tauri -> targetlibs/cua -> target- run: pnpm install --frozen-lockfile+ - uses: ./.github/actions/nextest+ # One retry; a test that passes only on it is reported and gets a `flaky`+ # issue (.github/flaky/).- name: Rust command-layer tests (hermetic)shell: bash- run: cargo test --manifest-path src-tauri/Cargo.toml --locked --test host_commands --test commands --test installer_commands --test app_core_parity --lib -- --test-threads=4+ run: >-+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-rust --format junit+ --results src-tauri/target/nextest/ci/junit.xml --+ cargo nextest run --manifest-path src-tauri/Cargo.toml --locked+ --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci+ --test host_commands --test commands --test installer_commands --test app_core_parity --lib+ --test-threads 4+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-rustfrontend:if: ${{ !startsWith(github.head_ref, 'release-please--branches--') }}@@ -198,7 +215,17 @@ jobs:cache-dependency-path: apps/cua-spaces/pnpm-lock.yaml- run: pnpm install --frozen-lockfile- run: pnpm typecheck- - run: pnpm test+ - name: Tests+ env:+ FLAKY_RESULTS_OUT: ${{ runner.temp }}/flaky-vitest/spaces.json+ run: |+ cp "$GITHUB_WORKSPACE/.github/flaky/vitest-reporter.mjs" ./.flaky-vitest-reporter.mjs+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite spaces-vitest --format vitest --results "$FLAKY_RESULTS_OUT" -- \+ pnpm test -- --retry=1 --reporter=default --reporter=./.flaky-vitest-reporter.mjs+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-vitestbundle:name: bundle (${{ matrix.name }})@@ -266,9 +293,22 @@ jobs:libx11-dev libxkbcommon-dev libwayland-dev libxcb1-dev libx11-xcb-dev \libxkbcommon-x11-dev libxrandr-dev libxi-dev libudev-dev- run: pnpm install --frozen-lockfile+ - uses: ./.github/actions/nextest+ # One retry; a test that passes only on it is reported and gets a `flaky`+ # issue (.github/flaky/).- name: Rust command-layer tests (hermetic)shell: bash- run: cargo test --manifest-path src-tauri/Cargo.toml --locked --test host_commands --test commands --test installer_commands --test app_core_parity --lib -- --test-threads=4+ run: >-+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spaces-rust-bundle-${{ matrix.name }}" --format junit+ --results src-tauri/target/nextest/ci/junit.xml --+ cargo nextest run --manifest-path src-tauri/Cargo.toml --locked+ --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci+ --test host_commands --test commands --test installer_commands --test app_core_parity --lib+ --test-threads 4+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spaces-rust-bundle-${{ matrix.name }}- name: Stage the cua CLI sidecarshell: bash# Not a hang: a cold release build of the CLI took 76-122 minutes on thediff --git a/.github/workflows/ci-cua-spacesd.yml b/.github/workflows/ci-cua-spacesd.ymlindex 0f7c456ff3..28fc2c5f47 100644--- a/.github/workflows/ci-cua-spacesd.yml+++ b/.github/workflows/ci-cua-spacesd.yml@@ -191,8 +191,18 @@ jobs:libxfixes-dev libxdamage-dev libudev-dev- name: Formatrun: cargo fmt --all --check+ - uses: ./.github/actions/nextest+ # One retry; a test that passes only on the retry is reported and gets a+ # `flaky` issue (see .github/flaky/). Doctests are not run by nextest.- name: Test- run: cargo test --workspace --locked+ run: >-+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-${{ matrix.os }}" --format junit+ --results target/nextest/ci/junit.xml --+ cargo nextest run --workspace --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci+ env:+ DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift+ - name: Doctests+ run: cargo test --workspace --doc --lockedenv:DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift- name: Clippy@@ -208,10 +218,18 @@ jobs:rustfmt --edition 2021 --check account_erase.rs hosted.rs meter.rs org_access.rs plan_source.rs team_api.rs team_sharing.rs teams.rs- name: Test (cua-relay hosted features)run: |- cargo test --locked -p cua-relay --features hosted- cargo test --locked -p cua-spacesd-server --features relay-hosted --test mesh_metering+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-hosted-${{ matrix.os }}" --format junit \+ --results target/nextest/ci/junit.xml -- \+ cargo nextest run --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci -p cua-relay --features hosted+ "$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "spacesd-mesh-metering-${{ matrix.os }}" --format junit \+ --results target/nextest/ci/junit.xml -- \+ cargo nextest run --locked --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci -p cua-spacesd-server --features relay-hosted --test mesh_meteringenv:DYLD_FALLBACK_LIBRARY_PATH: /usr/lib/swift+ - uses: ./.github/actions/flaky-upload+ if: always()+ with:+ suite: spacesd-${{ matrix.os }}- name: Clippy (cua-relay hosted features)run: cargo clippy --locked -p cua-relay -p cua-spacesd-server --all-targets --features cua-relay/hosted,cua-spacesd-server/relay-hosted -- -D warningsenv:diff --git a/.github/workflows/ci-flaky-quarantine.yml b/.github/workflows/ci-flaky-quarantine.ymlnew file mode 100644index 0000000000..c0220c16b8--- /dev/null+++ b/.github/workflows/ci-flaky-quarantine.yml@@ -0,0 +1,30 @@+name: "CI: flaky quarantine"++# Fails when an entry in .github/flaky/quarantine.json is invalid or past its+# expiry date, so a quarantined test cannot stay quarantined for good. Runs on+# every pull request (no path filter: an expired entry blocks every PR until+# the test is fixed or the entry is renewed with a reason) and daily on main.+# Also tests the flaky-test tooling (.github/scripts/flaky/flaky.py).++on:+ pull_request:+ schedule:+ - cron: "17 6 * * *"+ workflow_dispatch:++permissions:+ contents: read++jobs:+ quarantine:+ name: quarantine list and flaky tooling+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - name: Quarantine entries are valid and unexpired+ run: python3 .github/scripts/flaky/flaky.py validate+ - name: Flaky tooling tests+ run: |+ python3 -m pip install pytest+ python3 -m pytest -q -p no:cacheprovider .github/scripts/tests/test_flaky.pydiff --git a/.github/workflows/flaky-issues.yml b/.github/workflows/flaky-issues.ymlnew file mode 100644index 0000000000..0b80649791--- /dev/null+++ b/.github/workflows/flaky-issues.yml@@ -0,0 +1,55 @@+name: "Flaky test issues"++# Opens or updates one GitHub issue per flaky test (label `flaky`): a test that+# failed and then passed on its automatic retry in one of the CI workflows+# below. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload);+# this workflow reads them with issues: write, on the default branch's copy of+# the script, and never checks out or runs the pull request's code.++on:+ workflow_run:+ workflows:+ - "CI: cua-spacesd"+ - "CI: Cua Spaces"+ - "CI: Cua Spaces web UI and Electron shell"+ - "CI: Cua Spaces for macOS (SwiftUI)"+ types: [completed]++permissions:+ actions: read+ contents: read+ issues: write++concurrency:+ group: flaky-issues+ cancel-in-progress: false++jobs:+ file:+ name: file flaky-test issues+ if: github.event.workflow_run.conclusion != 'cancelled'+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ with:+ sparse-checkout: .github/scripts/flaky+ # Most runs have no flaky tests, so no artifact: nothing to download.+ - name: Download the run's flaky-test results+ continue-on-error: true+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ pattern: flaky-*+ merge-multiple: true+ path: flaky-results+ run-id: ${{ github.event.workflow_run.id }}+ github-token: ${{ github.token }}+ - name: Open or update the issues+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ if [ -d flaky-results ]; then+ python3 .github/scripts/flaky/flaky.py file-issues --dir flaky-results+ else+ echo "no flaky-test results in this run"+ fidiff --git a/apps/cua-spaces-web/e2e/playwright.config.ts b/apps/cua-spaces-web/e2e/playwright.config.tsindex f51d02c402..264eecedd8 100644--- a/apps/cua-spaces-web/e2e/playwright.config.ts+++ b/apps/cua-spaces-web/e2e/playwright.config.ts@@ -21,7 +21,14 @@ export default defineConfig({// One shell per test; each is a full Electron app.workers: ELECTRON ? Number(process.env.PARITY_WORKERS ?? 2) : undefined,timeout: ELECTRON ? 90_000 : undefined,- reporter: [["list"], ["./summary-reporter.ts"]],+ // One retry in CI. A test that passes only on it is "flaky" in Playwright's+ // report; CI lists it and opens an issue (.github/scripts/flaky/flaky.py).+ retries: process.env.CI ? 1 : 0,+ reporter: [+ ["list"],+ ["./summary-reporter.ts"],+ ...(process.env.FLAKY_RESULTS_OUT ? [["json", { outputFile: process.env.FLAKY_RESULTS_OUT }] as const] : []),+ ],use: {baseURL: `http://localhost:${PORT}`,viewport: { width: 1440, height: 900 },