Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
CI: cut bot noise (Codecov, CLA pass comments, per-run flaky comments) (#6586)
cua-agents[bot]810bc90f2fac
diff --git a/.github/actions/flaky-upload/action.yml b/.github/actions/flaky-upload/action.ymlindex 62a59b54f3..047898e14b 100644--- a/.github/actions/flaky-upload/action.yml+++ b/.github/actions/flaky-upload/action.yml@@ -1,5 +1,5 @@name: Upload flaky-test results-description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml to turn into GitHub issues. Run with if: always()."+description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml (job summary, and the weekly \"Flaky tests this week\" issue). Run with if: always()."inputs:suite:description: "The suite name given to flaky.py run (unique per job, matrix legs included)."@@ -12,4 +12,4 @@ runs:name: flaky-${{ inputs.suite }}path: ${{ runner.temp }}/flaky-results/if-no-files-found: ignore- retention-days: 3+ retention-days: 10diff --git a/.github/scripts/flaky/flaky.py b/.github/scripts/flaky/flaky.pyindex e4f5103736..803380a05a 100755--- a/.github/scripts/flaky/flaky.py+++ b/.github/scripts/flaky/flaky.py@@ -12,9 +12,11 @@ wraps the runner and reads its result file:not expired, the job passes with a warning.validate check .github/flaky/quarantine.json; exit 1 on an invalid orexpired entry.- file-issues- open or update one GitHub issue per flaky test (label `flaky`)- from the JSON artifacts that `run` (or `detect`) wrote.+ record write the flaky results of one run to the job summary. It never+ comments or opens issues.+ digest weekly: open or update ONE issue, "Flaky tests this week", with a+ retry count per test or job from the last 7 days of `flaky-*`+ artifacts.gate-checkfor ci-gate: read the failed jobs of one workflow run on stdin("<job id>\t<job name>" lines) and print `all` when every one@@ -22,7 +24,7 @@ wraps the runner and reads its result file:ci-gate then reruns the failed jobs once; a quarantined job thatfails again is reported and does not fail the gate.detect for the nightly flake scan: find jobs that failed and then passed- on a rerun of the same commit, and write `file-issues` reports.+ on a rerun of the same commit, and write reports for `record` and `digest`.Quarantine entries (.github/flaky/quarantine.json) name either a `test` (atest id, matched inside a job by `run`) or a `job` (a check name glob, with an@@ -41,13 +43,14 @@ import argparseimport datetime as dtimport fnmatchimport glob-import hashlib+import ioimport jsonimport osimport reimport subprocessimport sysimport xml.etree.ElementTree as ET+import zipfilefrom pathlib import Pathfrom typing import Dict, Iterable, List, Optional, Tuple@@ -395,71 +398,99 @@ def gh(*args: str, check: bool = True) -> str:return done.stdout-def flaky_key(test_id: str) -> str:- return hashlib.sha1(test_id.encode()).hexdigest()[:12]+def load_reports(directory: str) -> List[dict]:+ return [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{directory}/**/*.json", recursive=True))]-def cmd_file_issues(args: argparse.Namespace) -> int:- repo = os.environ["GITHUB_REPOSITORY"]- server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")- reports = [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{args.dir}/**/*.json", recursive=True))]- items: Dict[str, List[Tuple[dict, dict]]] = {}- for report in reports:- for row in report.get("flaky", []):- items.setdefault(row["id"], []).append((report, row))- if not items:+def cmd_record(args: argparse.Namespace) -> int:+ """Write the flaky results of one run to the job summary. Never comments or opens issues:+ `digest` turns the artifacts into one weekly issue."""+ rows = [(r, row) for r in load_reports(args.dir) for row in r.get("flaky", [])]+ if not rows:print("no flaky tests reported")return 0+ lines = ["### Passed only on retry", "", "| Test or job | Suite | Run |", "|---|---|---|"]+ for report, row in rows:+ lines.append(f"| `{cell(clean(row['id'], 200))}` | {cell(clean(report['suite'], 80))} | {report['run_id']} (attempt {report['run_attempt']}) |")+ text = "\n".join(lines) + "\n"+ print(text)+ summary = os.environ.get("GITHUB_STEP_SUMMARY")+ if summary:+ with open(summary, "a") as fh:+ fh.write(text)+ return 0+++DIGEST_TITLE = "Flaky tests this week"+DIGEST_MARKER = "<!-- flaky-digest -->"+++def build_digest(reports: List[dict], since: str) -> str:+ """Retry count per test or job: one per distinct (run, attempt) it passed only on retry in."""+ seen: Dict[str, set] = {}+ for report in reports:+ for row in report.get("flaky", []):+ seen.setdefault(row["id"], set()).add((report.get("run_id"), report.get("run_attempt")))+ ranked = sorted(seen.items(), key=lambda kv: (-len(kv[1]), kv[0]))+ lines = [DIGEST_MARKER, f"Tests and jobs that passed only on retry since {since} (UTC). "+ "Updated weekly. Fix the cause, or quarantine in `.github/flaky/quarantine.json` "+ "with an owner, an issue and an expiry date.", ""]+ if not ranked:+ lines.append("None this week.")+ else:+ lines += ["| Retries | Test or job |", "|---|---|"]+ lines += [f"| {len(runs)} | `{cell(clean(test_id, 200))}` |" for test_id, runs in ranked]+ return "\n".join(lines) + "\n"+++def list_flaky_artifacts(repo: str, since: dt.datetime) -> List[dict]:+ found, page = [], 1+ while page <= 30:+ data = json.loads(gh("api", f"repos/{repo}/actions/artifacts?per_page=100&page={page}"))+ batch = data.get("artifacts", [])+ for art in batch:+ created = dt.datetime.fromisoformat(art["created_at"].replace("Z", "+00:00"))+ if created >= since and not art.get("expired") and art["name"].startswith("flaky-"):+ found.append(art)+ if len(batch) < 100 or (batch and dt.datetime.fromisoformat(batch[-1]["created_at"].replace("Z", "+00:00")) < since):+ break+ page += 1+ return found+++def fetch_artifact_reports(repo: str, art: dict) -> List[dict]:+ done = subprocess.run(["gh", "api", f"repos/{repo}/actions/artifacts/{art['id']}/zip"], capture_output=True)+ if done.returncode != 0:+ return []+ out = []+ with zipfile.ZipFile(io.BytesIO(done.stdout)) as z:+ for name in z.namelist():+ if name.endswith(".json"):+ try:+ out.append(json.loads(z.read(name)))+ except ValueError:+ pass+ return out+++def cmd_digest(args: argparse.Namespace, list_artifacts=list_flaky_artifacts, fetch=fetch_artifact_reports) -> int:+ """Open or update ONE issue, "Flaky tests this week", from the last 7 days of flaky-* artifacts."""+ repo = os.environ["GITHUB_REPOSITORY"]+ since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=args.days)+ reports = [r for art in list_artifacts(repo, since) for r in fetch(repo, art)]+ body = build_digest(reports, since.strftime("%Y-%m-%d"))+ print(body)if args.dry_run:- for test_id in items:- print(f"would file: Flaky test: {clean(test_id, 120)}")return 0gh("label", "create", "flaky", "--repo", repo, "--color", "FBCA04", "--description", "Test that passed only on retry", check=False)- existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "all", "--limit", "1000",- "--json", "number,body,state") or "[]")- by_key = {}- for issue in existing:- m = re.search(r"<!-- flaky-key:(\w+) -->", issue.get("body") or "")- if m:- by_key[m.group(1)] = issue- for test_id, occurrences in items.items():- key = flaky_key(test_id)- report, row = occurrences[0]- run_url = f"{server}/{repo}/actions/runs/{report['run_id']}"- entry = [- f"Passed only on retry in `{clean(report['suite'], 80)}`.",- "",- f"- Run: {run_url} (attempt {report['run_attempt']}, workflow {clean(report['workflow'], 80)}, job {clean(report['job'], 80)})",- f"- Commit: `{report['sha'][:12]}`" + (f", PR #{report['pr']}" if report.get("pr") else f", ref `{clean(report['ref'], 80)}`"),- "",- "```",- clean(row.get("message", "") or "(no message)", 1200).replace("```", "'''"),- "```",- ]- issue = by_key.get(key)- if issue is None:- body = "\n".join([- f"`{clean(test_id, 300)}` passes only on retry in CI. Fix the cause, or quarantine it in "- "`.github/flaky/quarantine.json` with an owner, this issue and an expiry date.",- "",- "Failure history is in the comments below, one per CI run.",- "",- f"<!-- flaky-key:{key} -->",- ])- url = gh("issue", "create", "--repo", repo, "--label", "flaky",- "--title", f"Flaky {'job' if report.get('kind') == 'job' else 'test'}: {clean(test_id, 200)}", "--body", body).strip()- number = url.rsplit("/", 1)[-1]- print(f"opened {url}")- else:- number = str(issue["number"])- have = gh("issue", "view", number, "--repo", repo, "--json", "comments", "--jq", ".comments[].body")- if f"/actions/runs/{report['run_id']}" in have:- print(f"#{number}: run {report['run_id']} already recorded")- continue- if issue["state"] == "CLOSED":- gh("issue", "reopen", number, "--repo", repo)- gh("issue", "comment", number, "--repo", repo, "--body", "\n".join(entry))- print(f"#{number}: recorded {run_url}")+ existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "open", "--limit", "200",+ "--json", "number,title,body") or "[]")+ issue = next((i for i in existing if DIGEST_MARKER in (i.get("body") or "")), None)+ if issue:+ gh("issue", "edit", str(issue["number"]), "--repo", repo, "--body", body)+ print(f"updated #{issue['number']}")+ else:+ print(gh("issue", "create", "--repo", repo, "--label", "flaky", "--title", DIGEST_TITLE, "--body", body).strip())return 0@@ -580,9 +611,11 @@ def main(argv: Optional[List[str]] = None) -> int:run.add_argument("command", nargs=argparse.REMAINDER)validate = sub.add_parser("validate")validate.add_argument("--today")- issues = sub.add_parser("file-issues")- issues.add_argument("--dir", required=True)- issues.add_argument("--dry-run", action="store_true")+ record = sub.add_parser("record")+ record.add_argument("--dir", required=True)+ digest = sub.add_parser("digest")+ digest.add_argument("--days", type=int, default=7)+ digest.add_argument("--dry-run", action="store_true")gate = sub.add_parser("gate-check")gate.add_argument("--workflow", required=True)gate.add_argument("--quarantine", help="quarantine file (ci-gate passes the base commit's copy)")@@ -592,7 +625,7 @@ def main(argv: Optional[List[str]] = None) -> int:detect.add_argument("--since-hours", type=int, default=26)detect.add_argument("--max-pages", type=int, default=200)args = parser.parse_args(argv)- return {"run": cmd_run, "validate": cmd_validate, "file-issues": cmd_file_issues,+ return {"run": cmd_run, "validate": cmd_validate, "record": cmd_record, "digest": cmd_digest,"gate-check": cmd_gate_check, "detect": cmd_detect}[args.cmd](args)diff --git a/.github/scripts/tests/test_flaky.py b/.github/scripts/tests/test_flaky.pyindex b10379b64f..9708f9164e 100644--- a/.github/scripts/tests/test_flaky.py+++ b/.github/scripts/tests/test_flaky.py@@ -203,3 +203,25 @@ def test_detect_reports_jobs_that_failed_then_passed_on_a_rerun(env, monkeypatchassert report["kind"] == "job" and report["pr"] == 5assert [r["id"] for r in report["flaky"]] == ["CI: jev-use / macOS mock E2E"]assert not (env / "out" / "run-8.json").exists()+++def test_digest_counts_retries_per_test_and_sorts(env, monkeypatch):+ monkeypatch.setenv("GITHUB_REPOSITORY", "o/r")+ reports = [+ {"run_id": "1", "run_attempt": "1", "flaky": [{"id": "a::b"}, {"id": "c::d"}]},+ {"run_id": "2", "run_attempt": "1", "flaky": [{"id": "c::d"}]},+ {"run_id": "2", "run_attempt": "1", "flaky": [{"id": "c::d"}]}, # same run twice counts once+ ]+ body = flaky.build_digest(reports, "2026-11-01")+ assert body.index("c::d") < body.index("a::b")+ assert "| 2 | `c::d` |" in body and "| 1 | `a::b` |" in body+ args = flaky.argparse.Namespace(days=7, dry_run=True)+ assert flaky.cmd_digest(args, list_artifacts=lambda r, s: [{"id": 1}], fetch=lambda r, a: reports) == 0+++def test_record_writes_summary_only(env, capsys):+ d = env / "res"+ d.mkdir()+ (d / "x.json").write_text(json.dumps({"suite": "s", "run_id": "9", "run_attempt": "2", "flaky": [{"id": "t::x"}]}))+ assert flaky.main(["record", "--dir", str(d)]) == 0+ assert "t::x" in (env / "summary.md").read_text()diff --git a/.github/workflows/ci-test-python.yml b/.github/workflows/ci-test-python.ymlindex bffb1a8f22..0066c9c38e 100644--- a/.github/workflows/ci-test-python.yml+++ b/.github/workflows/ci-test-python.yml@@ -111,16 +111,6 @@ jobs:env:CUA_TELEMETRY_ENABLED: "false" # Disable telemetry during tests- - name: Upload coverage to Codecov- uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4- if: always()- with:- file: ./libs/python/${{ matrix.package }}/coverage.xml- flags: ${{ matrix.package }}- name: codecov-${{ matrix.package }}- fail_ci_if_error: false- continue-on-error: true-cua-sandbox:needs: quickif: ${{ !failure() && !cancelled() && (!startsWith(github.head_ref, 'release-please--branches--')) }}diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.ymlindex d0cde87cd9..136c3db8d3 100644--- a/.github/workflows/cla.yml+++ b/.github/workflows/cla.yml@@ -51,7 +51,28 @@ jobs:gh api "repos/$REPO/git/refs" -f ref=refs/heads/cla-signatures -f "sha=$commit" --silentecho "Created cla-signatures at $commit"+ # The action comments "All contributors have signed the CLA" on every pull+ # request. Org members and bots get a silent pass instead (the job still+ # succeeds, so the `cla` check stays green); only community authors reach+ # the action, which keeps the real "please sign" message for them.+ - name: Classify the pull request author+ id: who+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ REPO: ${{ github.repository }}+ NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}+ run: |+ set -euo pipefail+ read -r assoc type < <(gh api "repos/$REPO/pulls/$NUMBER" --jq '[.author_association, .user.type] | @tsv')+ if [ "$assoc" = MEMBER ] || [ "$assoc" = OWNER ] || [ "$type" = Bot ]; then+ echo "community=false" >> "$GITHUB_OUTPUT"+ echo "Author is an org member or a bot ($assoc/$type): silent pass, no comment."+ else+ echo "community=true" >> "$GITHUB_OUTPUT"+ fi+- name: Check CLA signatures+ if: steps.who.outputs.community == 'true'uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1env:GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}diff --git a/.github/workflows/flaky-detect.yml b/.github/workflows/flaky-detect.ymlindex b1e3841fce..ab68a32bdf 100644--- a/.github/workflows/flaky-detect.yml+++ b/.github/workflows/flaky-detect.yml@@ -1,11 +1,10 @@name: "Flaky job scan"# Nightly: finds jobs that failed and then passed when the same commit was rerun-# (any workflow, any event, the last 26 hours) and opens or updates one GitHub-# issue per job (label `flaky`) through .github/scripts/flaky/flaky.py. The issue-# says how to quarantine the job in .github/flaky/quarantine.json (an owner, the-# issue, a failure-log signature and a 7-day expiry) while the cause is fixed.-# Test-level flakes inside a job are filed by "Flaky test issues".+# (any workflow, any event, the last 26 hours) and records them (job summary + a `flaky-jobscan` artifact) through+# .github/scripts/flaky/flaky.py. "Flaky test issues" turns them into the weekly+# "Flaky tests this week" issue. To quarantine a job see .github/flaky/quarantine.json+# (an owner, an issue, a failure-log signature and a 7-day expiry).on:schedule:@@ -15,7 +14,6 @@ on:permissions:actions: readcontents: read- issues: writeconcurrency:group: flaky-detect@@ -35,7 +33,11 @@ jobs:env:GH_TOKEN: ${{ github.token }}run: python3 .github/scripts/flaky/flaky.py detect --out "$RUNNER_TEMP/flaky-results" --since-hours 26- - name: Open or update the issues- env:- GH_TOKEN: ${{ github.token }}- run: python3 .github/scripts/flaky/flaky.py file-issues --dir "$RUNNER_TEMP/flaky-results"+ - name: Write the job summary+ run: python3 .github/scripts/flaky/flaky.py record --dir "$RUNNER_TEMP/flaky-results"+ - uses: actions/upload-artifact@v4+ with:+ name: flaky-jobscan+ path: ${{ runner.temp }}/flaky-results/+ if-no-files-found: ignore+ retention-days: 10diff --git a/.github/workflows/flaky-issues.yml b/.github/workflows/flaky-issues.ymlindex 0b80649791..8d58f8dd98 100644--- a/.github/workflows/flaky-issues.yml+++ b/.github/workflows/flaky-issues.yml@@ -1,10 +1,12 @@name: "Flaky test issues"-# Opens or updates one GitHub issue per flaky test (label `flaky`): a test that-# failed and then passed on its automatic retry in one of the CI workflows-# below. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload);-# this workflow reads them with issues: write, on the default branch's copy of-# the script, and never checks out or runs the pull request's code.+# A test that failed and then passed on its automatic retry is recorded, never+# commented. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload,+# kept 10 days). Two jobs read them with the default branch's copy of the script+# and never check out or run the pull request's code:+# - file: after each CI run, writes the results to a job summary (no comments).+# - weekly: Mondays, opens or updates ONE issue, "Flaky tests this week", with+# a retry count per test or job, sorted (includes the nightly "Flaky job scan").on:workflow_run:@@ -14,6 +16,9 @@ on:- "CI: Cua Spaces web UI and Electron shell"- "CI: Cua Spaces for macOS (SwiftUI)"types: [completed]+ schedule:+ - cron: "17 6 * * 1"+ workflow_dispatch:permissions:actions: read@@ -27,7 +32,7 @@ concurrency:jobs:file:name: file flaky-test issues- if: github.event.workflow_run.conclusion != 'cancelled'+ if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'cancelled'runs-on: ubuntu-latesttimeout-minutes: 10steps:@@ -44,12 +49,28 @@ jobs:path: flaky-resultsrun-id: ${{ github.event.workflow_run.id }}github-token: ${{ github.token }}- - name: Open or update the issues+ - name: Write the job summaryenv:GH_TOKEN: ${{ github.token }}run: |if [ -d flaky-results ]; then- python3 .github/scripts/flaky/flaky.py file-issues --dir flaky-results+ python3 .github/scripts/flaky/flaky.py record --dir flaky-resultselseecho "no flaky-test results in this run"fi++ weekly:+ name: weekly flaky-test issue+ if: github.event_name != 'workflow_run'+ runs-on: ubuntu-latest+ timeout-minutes: 15+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ with:+ persist-credentials: false+ sparse-checkout: .github/scripts/flaky+ - name: Open or update the "Flaky tests this week" issue+ env:+ GH_TOKEN: ${{ github.token }}+ GITHUB_REPOSITORY: ${{ github.repository }}+ run: python3 .github/scripts/flaky/flaky.py digest