Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

CI: cut bot noise (Codecov, CLA pass comments, per-run flaky comments) (#6586)

cua-agents[bot]810bc90f2fac
diff --git a/.github/actions/flaky-upload/action.yml b/.github/actions/flaky-upload/action.yml
index 62a59b54f3..047898e14b 100644
--- a/.github/actions/flaky-upload/action.yml
+++ b/.github/actions/flaky-upload/action.yml
@@ -1,5 +1,5 @@
name: Upload flaky-test results
-description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml to turn into GitHub issues. Run with if: always()."
+description: "Uploads what .github/scripts/flaky/flaky.py found, for flaky-issues.yml (job summary, and the weekly \"Flaky tests this week\" issue). Run with if: always()."
inputs:
suite:
description: "The suite name given to flaky.py run (unique per job, matrix legs included)."
@@ -12,4 +12,4 @@ runs:
name: flaky-${{ inputs.suite }}
path: ${{ runner.temp }}/flaky-results/
if-no-files-found: ignore
- retention-days: 3
+ retention-days: 10
diff --git a/.github/scripts/flaky/flaky.py b/.github/scripts/flaky/flaky.py
index e4f5103736..803380a05a 100755
--- a/.github/scripts/flaky/flaky.py
+++ b/.github/scripts/flaky/flaky.py
@@ -12,9 +12,11 @@ wraps the runner and reads its result file:
not expired, the job passes with a warning.
validate check .github/flaky/quarantine.json; exit 1 on an invalid or
expired entry.
- file-issues
- open or update one GitHub issue per flaky test (label `flaky`)
- from the JSON artifacts that `run` (or `detect`) wrote.
+ record write the flaky results of one run to the job summary. It never
+ comments or opens issues.
+ digest weekly: open or update ONE issue, "Flaky tests this week", with a
+ retry count per test or job from the last 7 days of `flaky-*`
+ artifacts.
gate-check
for ci-gate: read the failed jobs of one workflow run on stdin
("<job id>\t<job name>" lines) and print `all` when every one
@@ -22,7 +24,7 @@ wraps the runner and reads its result file:
ci-gate then reruns the failed jobs once; a quarantined job that
fails again is reported and does not fail the gate.
detect for the nightly flake scan: find jobs that failed and then passed
- on a rerun of the same commit, and write `file-issues` reports.
+ on a rerun of the same commit, and write reports for `record` and `digest`.
Quarantine entries (.github/flaky/quarantine.json) name either a `test` (a
test id, matched inside a job by `run`) or a `job` (a check name glob, with an
@@ -41,13 +43,14 @@ import argparse
import datetime as dt
import fnmatch
import glob
-import hashlib
+import io
import json
import os
import re
import subprocess
import sys
import xml.etree.ElementTree as ET
+import zipfile
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
@@ -395,71 +398,99 @@ def gh(*args: str, check: bool = True) -> str:
return done.stdout
-def flaky_key(test_id: str) -> str:
- return hashlib.sha1(test_id.encode()).hexdigest()[:12]
+def load_reports(directory: str) -> List[dict]:
+ return [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{directory}/**/*.json", recursive=True))]
-def cmd_file_issues(args: argparse.Namespace) -> int:
- repo = os.environ["GITHUB_REPOSITORY"]
- server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
- reports = [json.loads(Path(p).read_text()) for p in sorted(glob.glob(f"{args.dir}/**/*.json", recursive=True))]
- items: Dict[str, List[Tuple[dict, dict]]] = {}
- for report in reports:
- for row in report.get("flaky", []):
- items.setdefault(row["id"], []).append((report, row))
- if not items:
+def cmd_record(args: argparse.Namespace) -> int:
+ """Write the flaky results of one run to the job summary. Never comments or opens issues:
+ `digest` turns the artifacts into one weekly issue."""
+ rows = [(r, row) for r in load_reports(args.dir) for row in r.get("flaky", [])]
+ if not rows:
print("no flaky tests reported")
return 0
+ lines = ["### Passed only on retry", "", "| Test or job | Suite | Run |", "|---|---|---|"]
+ for report, row in rows:
+ lines.append(f"| `{cell(clean(row['id'], 200))}` | {cell(clean(report['suite'], 80))} | {report['run_id']} (attempt {report['run_attempt']}) |")
+ text = "\n".join(lines) + "\n"
+ print(text)
+ summary = os.environ.get("GITHUB_STEP_SUMMARY")
+ if summary:
+ with open(summary, "a") as fh:
+ fh.write(text)
+ return 0
+
+
+DIGEST_TITLE = "Flaky tests this week"
+DIGEST_MARKER = "<!-- flaky-digest -->"
+
+
+def build_digest(reports: List[dict], since: str) -> str:
+ """Retry count per test or job: one per distinct (run, attempt) it passed only on retry in."""
+ seen: Dict[str, set] = {}
+ for report in reports:
+ for row in report.get("flaky", []):
+ seen.setdefault(row["id"], set()).add((report.get("run_id"), report.get("run_attempt")))
+ ranked = sorted(seen.items(), key=lambda kv: (-len(kv[1]), kv[0]))
+ lines = [DIGEST_MARKER, f"Tests and jobs that passed only on retry since {since} (UTC). "
+ "Updated weekly. Fix the cause, or quarantine in `.github/flaky/quarantine.json` "
+ "with an owner, an issue and an expiry date.", ""]
+ if not ranked:
+ lines.append("None this week.")
+ else:
+ lines += ["| Retries | Test or job |", "|---|---|"]
+ lines += [f"| {len(runs)} | `{cell(clean(test_id, 200))}` |" for test_id, runs in ranked]
+ return "\n".join(lines) + "\n"
+
+
+def list_flaky_artifacts(repo: str, since: dt.datetime) -> List[dict]:
+ found, page = [], 1
+ while page <= 30:
+ data = json.loads(gh("api", f"repos/{repo}/actions/artifacts?per_page=100&page={page}"))
+ batch = data.get("artifacts", [])
+ for art in batch:
+ created = dt.datetime.fromisoformat(art["created_at"].replace("Z", "+00:00"))
+ if created >= since and not art.get("expired") and art["name"].startswith("flaky-"):
+ found.append(art)
+ if len(batch) < 100 or (batch and dt.datetime.fromisoformat(batch[-1]["created_at"].replace("Z", "+00:00")) < since):
+ break
+ page += 1
+ return found
+
+
+def fetch_artifact_reports(repo: str, art: dict) -> List[dict]:
+ done = subprocess.run(["gh", "api", f"repos/{repo}/actions/artifacts/{art['id']}/zip"], capture_output=True)
+ if done.returncode != 0:
+ return []
+ out = []
+ with zipfile.ZipFile(io.BytesIO(done.stdout)) as z:
+ for name in z.namelist():
+ if name.endswith(".json"):
+ try:
+ out.append(json.loads(z.read(name)))
+ except ValueError:
+ pass
+ return out
+
+
+def cmd_digest(args: argparse.Namespace, list_artifacts=list_flaky_artifacts, fetch=fetch_artifact_reports) -> int:
+ """Open or update ONE issue, "Flaky tests this week", from the last 7 days of flaky-* artifacts."""
+ repo = os.environ["GITHUB_REPOSITORY"]
+ since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=args.days)
+ reports = [r for art in list_artifacts(repo, since) for r in fetch(repo, art)]
+ body = build_digest(reports, since.strftime("%Y-%m-%d"))
+ print(body)
if args.dry_run:
- for test_id in items:
- print(f"would file: Flaky test: {clean(test_id, 120)}")
return 0
gh("label", "create", "flaky", "--repo", repo, "--color", "FBCA04", "--description", "Test that passed only on retry", check=False)
- existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "all", "--limit", "1000",
- "--json", "number,body,state") or "[]")
- by_key = {}
- for issue in existing:
- m = re.search(r"<!-- flaky-key:(\w+) -->", issue.get("body") or "")
- if m:
- by_key[m.group(1)] = issue
- for test_id, occurrences in items.items():
- key = flaky_key(test_id)
- report, row = occurrences[0]
- run_url = f"{server}/{repo}/actions/runs/{report['run_id']}"
- entry = [
- f"Passed only on retry in `{clean(report['suite'], 80)}`.",
- "",
- f"- Run: {run_url} (attempt {report['run_attempt']}, workflow {clean(report['workflow'], 80)}, job {clean(report['job'], 80)})",
- f"- Commit: `{report['sha'][:12]}`" + (f", PR #{report['pr']}" if report.get("pr") else f", ref `{clean(report['ref'], 80)}`"),
- "",
- "```",
- clean(row.get("message", "") or "(no message)", 1200).replace("```", "'''"),
- "```",
- ]
- issue = by_key.get(key)
- if issue is None:
- body = "\n".join([
- f"`{clean(test_id, 300)}` passes only on retry in CI. Fix the cause, or quarantine it in "
- "`.github/flaky/quarantine.json` with an owner, this issue and an expiry date.",
- "",
- "Failure history is in the comments below, one per CI run.",
- "",
- f"<!-- flaky-key:{key} -->",
- ])
- url = gh("issue", "create", "--repo", repo, "--label", "flaky",
- "--title", f"Flaky {'job' if report.get('kind') == 'job' else 'test'}: {clean(test_id, 200)}", "--body", body).strip()
- number = url.rsplit("/", 1)[-1]
- print(f"opened {url}")
- else:
- number = str(issue["number"])
- have = gh("issue", "view", number, "--repo", repo, "--json", "comments", "--jq", ".comments[].body")
- if f"/actions/runs/{report['run_id']}" in have:
- print(f"#{number}: run {report['run_id']} already recorded")
- continue
- if issue["state"] == "CLOSED":
- gh("issue", "reopen", number, "--repo", repo)
- gh("issue", "comment", number, "--repo", repo, "--body", "\n".join(entry))
- print(f"#{number}: recorded {run_url}")
+ existing = json.loads(gh("issue", "list", "--repo", repo, "--label", "flaky", "--state", "open", "--limit", "200",
+ "--json", "number,title,body") or "[]")
+ issue = next((i for i in existing if DIGEST_MARKER in (i.get("body") or "")), None)
+ if issue:
+ gh("issue", "edit", str(issue["number"]), "--repo", repo, "--body", body)
+ print(f"updated #{issue['number']}")
+ else:
+ print(gh("issue", "create", "--repo", repo, "--label", "flaky", "--title", DIGEST_TITLE, "--body", body).strip())
return 0
@@ -580,9 +611,11 @@ def main(argv: Optional[List[str]] = None) -> int:
run.add_argument("command", nargs=argparse.REMAINDER)
validate = sub.add_parser("validate")
validate.add_argument("--today")
- issues = sub.add_parser("file-issues")
- issues.add_argument("--dir", required=True)
- issues.add_argument("--dry-run", action="store_true")
+ record = sub.add_parser("record")
+ record.add_argument("--dir", required=True)
+ digest = sub.add_parser("digest")
+ digest.add_argument("--days", type=int, default=7)
+ digest.add_argument("--dry-run", action="store_true")
gate = sub.add_parser("gate-check")
gate.add_argument("--workflow", required=True)
gate.add_argument("--quarantine", help="quarantine file (ci-gate passes the base commit's copy)")
@@ -592,7 +625,7 @@ def main(argv: Optional[List[str]] = None) -> int:
detect.add_argument("--since-hours", type=int, default=26)
detect.add_argument("--max-pages", type=int, default=200)
args = parser.parse_args(argv)
- return {"run": cmd_run, "validate": cmd_validate, "file-issues": cmd_file_issues,
+ return {"run": cmd_run, "validate": cmd_validate, "record": cmd_record, "digest": cmd_digest,
"gate-check": cmd_gate_check, "detect": cmd_detect}[args.cmd](args)
diff --git a/.github/scripts/tests/test_flaky.py b/.github/scripts/tests/test_flaky.py
index b10379b64f..9708f9164e 100644
--- a/.github/scripts/tests/test_flaky.py
+++ b/.github/scripts/tests/test_flaky.py
@@ -203,3 +203,25 @@ def test_detect_reports_jobs_that_failed_then_passed_on_a_rerun(env, monkeypatch
assert report["kind"] == "job" and report["pr"] == 5
assert [r["id"] for r in report["flaky"]] == ["CI: jev-use / macOS mock E2E"]
assert not (env / "out" / "run-8.json").exists()
+
+
+def test_digest_counts_retries_per_test_and_sorts(env, monkeypatch):
+ monkeypatch.setenv("GITHUB_REPOSITORY", "o/r")
+ reports = [
+ {"run_id": "1", "run_attempt": "1", "flaky": [{"id": "a::b"}, {"id": "c::d"}]},
+ {"run_id": "2", "run_attempt": "1", "flaky": [{"id": "c::d"}]},
+ {"run_id": "2", "run_attempt": "1", "flaky": [{"id": "c::d"}]}, # same run twice counts once
+ ]
+ body = flaky.build_digest(reports, "2026-11-01")
+ assert body.index("c::d") < body.index("a::b")
+ assert "| 2 | `c::d` |" in body and "| 1 | `a::b` |" in body
+ args = flaky.argparse.Namespace(days=7, dry_run=True)
+ assert flaky.cmd_digest(args, list_artifacts=lambda r, s: [{"id": 1}], fetch=lambda r, a: reports) == 0
+
+
+def test_record_writes_summary_only(env, capsys):
+ d = env / "res"
+ d.mkdir()
+ (d / "x.json").write_text(json.dumps({"suite": "s", "run_id": "9", "run_attempt": "2", "flaky": [{"id": "t::x"}]}))
+ assert flaky.main(["record", "--dir", str(d)]) == 0
+ assert "t::x" in (env / "summary.md").read_text()
diff --git a/.github/workflows/ci-test-python.yml b/.github/workflows/ci-test-python.yml
index bffb1a8f22..0066c9c38e 100644
--- a/.github/workflows/ci-test-python.yml
+++ b/.github/workflows/ci-test-python.yml
@@ -111,16 +111,6 @@ jobs:
env:
CUA_TELEMETRY_ENABLED: "false" # Disable telemetry during tests
- - name: Upload coverage to Codecov
- uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
- if: always()
- with:
- file: ./libs/python/${{ matrix.package }}/coverage.xml
- flags: ${{ matrix.package }}
- name: codecov-${{ matrix.package }}
- fail_ci_if_error: false
- continue-on-error: true
-
cua-sandbox:
needs: quick
if: ${{ !failure() && !cancelled() && (!startsWith(github.head_ref, 'release-please--branches--')) }}
diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml
index d0cde87cd9..136c3db8d3 100644
--- a/.github/workflows/cla.yml
+++ b/.github/workflows/cla.yml
@@ -51,7 +51,28 @@ jobs:
gh api "repos/$REPO/git/refs" -f ref=refs/heads/cla-signatures -f "sha=$commit" --silent
echo "Created cla-signatures at $commit"
+ # The action comments "All contributors have signed the CLA" on every pull
+ # request. Org members and bots get a silent pass instead (the job still
+ # succeeds, so the `cla` check stays green); only community authors reach
+ # the action, which keeps the real "please sign" message for them.
+ - name: Classify the pull request author
+ id: who
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ REPO: ${{ github.repository }}
+ NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
+ run: |
+ set -euo pipefail
+ read -r assoc type < <(gh api "repos/$REPO/pulls/$NUMBER" --jq '[.author_association, .user.type] | @tsv')
+ if [ "$assoc" = MEMBER ] || [ "$assoc" = OWNER ] || [ "$type" = Bot ]; then
+ echo "community=false" >> "$GITHUB_OUTPUT"
+ echo "Author is an org member or a bot ($assoc/$type): silent pass, no comment."
+ else
+ echo "community=true" >> "$GITHUB_OUTPUT"
+ fi
+
- name: Check CLA signatures
+ if: steps.who.outputs.community == 'true'
uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/flaky-detect.yml b/.github/workflows/flaky-detect.yml
index b1e3841fce..ab68a32bdf 100644
--- a/.github/workflows/flaky-detect.yml
+++ b/.github/workflows/flaky-detect.yml
@@ -1,11 +1,10 @@
name: "Flaky job scan"
# Nightly: finds jobs that failed and then passed when the same commit was rerun
-# (any workflow, any event, the last 26 hours) and opens or updates one GitHub
-# issue per job (label `flaky`) through .github/scripts/flaky/flaky.py. The issue
-# says how to quarantine the job in .github/flaky/quarantine.json (an owner, the
-# issue, a failure-log signature and a 7-day expiry) while the cause is fixed.
-# Test-level flakes inside a job are filed by "Flaky test issues".
+# (any workflow, any event, the last 26 hours) and records them (job summary + a `flaky-jobscan` artifact) through
+# .github/scripts/flaky/flaky.py. "Flaky test issues" turns them into the weekly
+# "Flaky tests this week" issue. To quarantine a job see .github/flaky/quarantine.json
+# (an owner, an issue, a failure-log signature and a 7-day expiry).
on:
schedule:
@@ -15,7 +14,6 @@ on:
permissions:
actions: read
contents: read
- issues: write
concurrency:
group: flaky-detect
@@ -35,7 +33,11 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: python3 .github/scripts/flaky/flaky.py detect --out "$RUNNER_TEMP/flaky-results" --since-hours 26
- - name: Open or update the issues
- env:
- GH_TOKEN: ${{ github.token }}
- run: python3 .github/scripts/flaky/flaky.py file-issues --dir "$RUNNER_TEMP/flaky-results"
+ - name: Write the job summary
+ run: python3 .github/scripts/flaky/flaky.py record --dir "$RUNNER_TEMP/flaky-results"
+ - uses: actions/upload-artifact@v4
+ with:
+ name: flaky-jobscan
+ path: ${{ runner.temp }}/flaky-results/
+ if-no-files-found: ignore
+ retention-days: 10
diff --git a/.github/workflows/flaky-issues.yml b/.github/workflows/flaky-issues.yml
index 0b80649791..8d58f8dd98 100644
--- a/.github/workflows/flaky-issues.yml
+++ b/.github/workflows/flaky-issues.yml
@@ -1,10 +1,12 @@
name: "Flaky test issues"
-# Opens or updates one GitHub issue per flaky test (label `flaky`): a test that
-# failed and then passed on its automatic retry in one of the CI workflows
-# below. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload);
-# this workflow reads them with issues: write, on the default branch's copy of
-# the script, and never checks out or runs the pull request's code.
+# A test that failed and then passed on its automatic retry is recorded, never
+# commented. Each CI job uploads a `flaky-*` artifact (.github/actions/flaky-upload,
+# kept 10 days). Two jobs read them with the default branch's copy of the script
+# and never check out or run the pull request's code:
+# - file: after each CI run, writes the results to a job summary (no comments).
+# - weekly: Mondays, opens or updates ONE issue, "Flaky tests this week", with
+# a retry count per test or job, sorted (includes the nightly "Flaky job scan").
on:
workflow_run:
@@ -14,6 +16,9 @@ on:
- "CI: Cua Spaces web UI and Electron shell"
- "CI: Cua Spaces for macOS (SwiftUI)"
types: [completed]
+ schedule:
+ - cron: "17 6 * * 1"
+ workflow_dispatch:
permissions:
actions: read
@@ -27,7 +32,7 @@ concurrency:
jobs:
file:
name: file flaky-test issues
- if: github.event.workflow_run.conclusion != 'cancelled'
+ if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'cancelled'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -44,12 +49,28 @@ jobs:
path: flaky-results
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ github.token }}
- - name: Open or update the issues
+ - name: Write the job summary
env:
GH_TOKEN: ${{ github.token }}
run: |
if [ -d flaky-results ]; then
- python3 .github/scripts/flaky/flaky.py file-issues --dir flaky-results
+ python3 .github/scripts/flaky/flaky.py record --dir flaky-results
else
echo "no flaky-test results in this run"
fi
+
+ weekly:
+ name: weekly flaky-test issue
+ if: github.event_name != 'workflow_run'
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+ sparse-checkout: .github/scripts/flaky
+ - name: Open or update the "Flaky tests this week" issue
+ env:
+ GH_TOKEN: ${{ github.token }}
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ run: python3 .github/scripts/flaky/flaky.py digest