Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci: fail workflows that use paid GitHub larger runners (CUA-3351) (#6471)

Francesco Bonacciecaf10b1973f
diff --git a/.github/larger-runners-allowed.txt b/.github/larger-runners-allowed.txt
new file mode 100644
index 0000000000..10ddf473e6
--- /dev/null
+++ b/.github/larger-runners-allowed.txt
@@ -0,0 +1,10 @@
+# Opt-out list for scripts/ci/check-no-larger-runners.py (CI: Block larger runners).
+#
+# GitHub bills larger runners (macOS xlarge/large, Linux or Windows N-core,
+# custom larger-runner groups) even on public repositories. Standard runners on
+# trycua/cua are free. October 2026: $3.4k on this repo for larger runners.
+#
+# To allow one workflow to use one larger-runner label, add a line:
+# <workflow file name> <label> # why, who approved, tracking issue
+# Every entry needs a comment with the reason and Francesco's approval.
+# This list starts empty on purpose.
diff --git a/.github/scripts/check_no_larger_runners.py b/.github/scripts/check_no_larger_runners.py
new file mode 100644
index 0000000000..435cb96201
--- /dev/null
+++ b/.github/scripts/check_no_larger_runners.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Fail when a workflow asks for a paid GitHub larger runner.
+
+Standard GitHub-hosted runners are free on public repositories; larger runners
+are billed there too (macOS xlarge, Linux 16-core and Windows 8-core cost $3.4k
+on trycua/cua in October 2026). This check reads every `runs-on` in
+.github/workflows, resolves `matrix.*` and `inputs.*` references to their
+values, and fails on:
+
+ * labels ending in -large / -xlarge (macos-26-xlarge, macos-latest-large)
+ * labels naming a core count (ubuntu-latest-8-cores, cua-linux-16core)
+ * `group:` runners (custom larger-runner groups)
+
+Self-hosted labels that do not look like those pass. Opt-outs live in
+.github/larger-runners-allowed.txt (starts empty).
+
+Usage: check_no_larger_runners.py [--list] [repo_root]
+ --list print every workflow that uses a larger runner, even if allowed
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+LARGER = re.compile(r"(?:^|-)(?:x{0,2}large)$|\d+-?cores?(?:$|-)", re.IGNORECASE)
+LITERAL = re.compile(r"'([^']*)'|\"([^\"]*)\"")
+REF = re.compile(r"\b(matrix|inputs|vars)\.([A-Za-z0-9_-]+)")
+
+
+def is_larger(label: str) -> bool:
+ return bool(LARGER.search(label.strip()))
+
+
+def load_allowed(root: Path) -> set[tuple[str, str]]:
+ path = root / ".github" / "larger-runners-allowed.txt"
+ allowed: set[tuple[str, str]] = set()
+ if not path.exists():
+ return allowed
+ for raw in path.read_text().splitlines():
+ line = raw.split("#", 1)[0].strip()
+ if not line:
+ continue
+ parts = line.split()
+ if len(parts) != 2:
+ raise SystemExit(f"{path}: expected '<workflow> <label>', got: {raw!r}")
+ allowed.add((parts[0], parts[1]))
+ return allowed
+
+
+def _strings(value) -> list[str]:
+ """Every string reachable inside a YAML value."""
+ if isinstance(value, str):
+ return [value]
+ if isinstance(value, (int, float, bool)) or value is None:
+ return []
+ if isinstance(value, dict):
+ return [s for v in value.values() for s in _strings(v)]
+ if isinstance(value, list):
+ return [s for v in value for s in _strings(v)]
+ return []
+
+
+def _matrix_values(matrix, key: str) -> list[str]:
+ if not isinstance(matrix, dict):
+ return []
+ out: list[str] = []
+ if key in matrix:
+ out += _strings(matrix[key])
+ for entry in matrix.get("include") or []:
+ if isinstance(entry, dict) and key in entry:
+ out += _strings(entry[key])
+ if isinstance(matrix.get("include"), str): # fromJSON(...) literal
+ out += [s for s in LITERAL_ALL(matrix["include"]) if s]
+ return out
+
+
+def LITERAL_ALL(text: str) -> list[str]:
+ """Strings inside an expression, including JSON embedded in fromJSON('...')."""
+ found = [a or b for a, b in LITERAL.findall(text)]
+ extra: list[str] = []
+ for chunk in found:
+ extra += re.findall(r'"(?:os|runner|runs-on|platform)"\s*:\s*"([^"]+)"', chunk)
+ return found + extra
+
+
+def labels_for(workflow: dict, job: dict) -> list[str]:
+ """Candidate runner labels of a job: literals plus referenced matrix/input values."""
+ runs_on = job.get("runs-on")
+ if runs_on is None:
+ return []
+ labels: list[str] = []
+ if isinstance(runs_on, dict):
+ if "group" in runs_on:
+ labels.append(f"group:{runs_on['group']}")
+ runs_on = runs_on.get("labels", [])
+ texts = _strings(runs_on)
+ matrix = (job.get("strategy") or {}).get("matrix")
+ # `on` parses as boolean True in YAML 1.1.
+ triggers = workflow.get("on", workflow.get(True)) or {}
+ inputs: dict = {}
+ if isinstance(triggers, dict):
+ for kind in ("workflow_call", "workflow_dispatch"):
+ body = triggers.get(kind)
+ if isinstance(body, dict) and isinstance(body.get("inputs"), dict):
+ inputs.update(body["inputs"])
+ for text in texts:
+ if "${{" not in text:
+ labels.append(text)
+ continue
+ labels += [s for s in LITERAL_ALL(text) if s]
+ for scope, name in REF.findall(text):
+ if scope == "matrix":
+ labels += _matrix_values(matrix, name)
+ elif scope == "inputs":
+ spec = inputs.get(name)
+ if isinstance(spec, dict):
+ labels += _strings(spec.get("default")) + _strings(spec.get("options"))
+ return labels
+
+
+def find_uses(root: Path) -> list[tuple[str, str, str]]:
+ """(workflow file name, job id, label) for every larger-runner label."""
+ uses: list[tuple[str, str, str]] = []
+ for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
+ try:
+ workflow = yaml.safe_load(path.read_text())
+ except yaml.YAMLError as err:
+ raise SystemExit(f"{path}: cannot parse: {err}")
+ if not isinstance(workflow, dict):
+ continue
+ for job_id, job in (workflow.get("jobs") or {}).items():
+ if not isinstance(job, dict):
+ continue
+ for label in labels_for(workflow, job):
+ if label.startswith("group:") or is_larger(label):
+ uses.append((path.name, job_id, label))
+ return uses
+
+
+def main(argv: list[str]) -> int:
+ list_only = "--list" in argv
+ args = [a for a in argv if not a.startswith("--")]
+ root = Path(args[0]) if args else Path(__file__).resolve().parents[2]
+ allowed = load_allowed(root)
+ uses = find_uses(root)
+ if list_only:
+ for wf, job, label in uses:
+ print(f"{wf}\t{job}\t{label}")
+ return 0
+ bad = [u for u in uses if (u[0], u[2]) not in allowed]
+ for wf, job, label in bad:
+ print(f"::error file=.github/workflows/{wf}::job '{job}' uses larger runner '{label}'")
+ print(f"FAIL {wf} job {job}: larger runner '{label}' is billed even on public repos; "
+ "use ubuntu-latest, macos-26, windows-latest (see .github/larger-runners-allowed.txt)")
+ if bad:
+ return 1
+ print(f"ok: no larger runners ({len(uses)} allowed)")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/.github/scripts/tests/test_check_no_larger_runners.py b/.github/scripts/tests/test_check_no_larger_runners.py
new file mode 100644
index 0000000000..9df732e5b6
--- /dev/null
+++ b/.github/scripts/tests/test_check_no_larger_runners.py
@@ -0,0 +1,85 @@
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import check_no_larger_runners as chk # noqa: E402
+
+REPO = Path(__file__).resolve().parents[3]
+
+
+def make_repo(workflows: dict[str, str], allowed: str = "") -> Path:
+ root = Path(tempfile.mkdtemp())
+ (root / ".github" / "workflows").mkdir(parents=True)
+ for name, body in workflows.items():
+ (root / ".github" / "workflows" / name).write_text(textwrap.dedent(body))
+ (root / ".github" / "larger-runners-allowed.txt").write_text(allowed)
+ return root
+
+
+class LargerRunnerCheck(unittest.TestCase):
+ def test_label_classifier(self):
+ for bad in ("macos-26-xlarge", "macos-latest-large", "macos-15-xlarge",
+ "cua-linux-16core", "cua-windows-8core", "ubuntu-latest-8-cores",
+ "windows-latest-16-cores"):
+ self.assertTrue(chk.is_larger(bad), bad)
+ for ok in ("ubuntu-latest", "ubuntu-24.04-arm", "macos-26", "macos-15-intel",
+ "windows-latest", "windows-11-arm", "self-hosted", "lume"):
+ self.assertFalse(chk.is_larger(ok), ok)
+
+ def test_literal_and_list(self):
+ root = make_repo({"a.yml": """
+ on: push
+ jobs:
+ one: {runs-on: macos-26-xlarge, steps: []}
+ two: {runs-on: [self-hosted, cua-linux-16core], steps: []}
+ three: {runs-on: ubuntu-latest, steps: []}
+ four: {runs-on: {group: big-runners}, steps: []}
+ """})
+ got = {(j, l) for _, j, l in chk.find_uses(root)}
+ self.assertEqual(got, {("one", "macos-26-xlarge"), ("two", "cua-linux-16core"),
+ ("four", "group:big-runners")})
+
+ def test_matrix_expression_and_inputs(self):
+ root = make_repo({"m.yml": """
+ on:
+ workflow_call:
+ inputs:
+ runner: {type: string, default: macos-latest-large}
+ jobs:
+ j:
+ strategy:
+ matrix:
+ os: [ubuntu-latest, cua-windows-8core]
+ runs-on: ${{ matrix.os == 'ubuntu-latest' && 'ubuntu-latest' || matrix.os }}
+ steps: []
+ k:
+ runs-on: ${{ inputs.runner }}
+ steps: []
+ l:
+ strategy:
+ matrix:
+ include:
+ - {runner: macos-26-xlarge}
+ runs-on: ${{ matrix.runner }}
+ steps: []
+ """})
+ got = {(j, l) for _, j, l in chk.find_uses(root)}
+ self.assertEqual(got, {("j", "cua-windows-8core"), ("k", "macos-latest-large"),
+ ("l", "macos-26-xlarge")})
+
+ def test_allowlist_and_exit_code(self):
+ wf = {"a.yml": "on: push\njobs:\n j:\n runs-on: macos-26-xlarge\n steps: []\n"}
+ self.assertEqual(chk.main([str(make_repo(wf))]), 1)
+ allowed = "# reason\na.yml macos-26-xlarge # approved\n"
+ self.assertEqual(chk.main([str(make_repo(wf, allowed))]), 0)
+
+ def test_repository_has_no_larger_runners_and_empty_allowlist(self):
+ self.assertEqual(chk.find_uses(REPO), [])
+ self.assertEqual(chk.load_allowed(REPO), set())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/workflows/ci-block-larger-runners.yml b/.github/workflows/ci-block-larger-runners.yml
new file mode 100644
index 0000000000..f6c9a60140
--- /dev/null
+++ b/.github/workflows/ci-block-larger-runners.yml
@@ -0,0 +1,37 @@
+name: "CI: Block larger runners"
+
+# GitHub bills larger runners (macOS xlarge, Linux 16-core, Windows 8-core,
+# custom groups) even on public repositories; standard runners are free here.
+# October 2026: $3.4k. This fails any workflow `runs-on` that names one.
+# Opt-outs go in .github/larger-runners-allowed.txt (starts empty).
+# Own fast job until the quick stage (#6465) exists; then move it there.
+
+on:
+ pull_request:
+ paths:
+ - ".github/workflows/**"
+ - ".github/larger-runners-allowed.txt"
+ - ".github/scripts/check_no_larger_runners.py"
+ push:
+ branches: [main]
+ paths:
+ - ".github/workflows/**"
+ - ".github/larger-runners-allowed.txt"
+ merge_group:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ block-larger-runners:
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v4
+ - name: Fail on larger runners
+ run: python3 .github/scripts/check_no_larger_runners.py
diff --git a/bin/ci b/bin/ci
index c6d46ca5b3..d516fc783a 100755
--- a/bin/ci
+++ b/bin/ci
@@ -628,6 +628,12 @@ def static_checks(changes: Changes, ctx: "Context") -> List[Check]:
tools=("actionlint",), hint="install: brew install actionlint (CI pins v1.7.12)",
ci=(("ci-lint-workflows.yml", "./actionlint -shellcheck= -pyflakes="),)))
+ if changes.touches(r"^\.github/(workflows/|larger-runners-allowed\.txt|scripts/check_no_larger_runners\.py)"):
+ add(Check("larger-runners", "no paid larger runners in workflows",
+ "python3 .github/scripts/check_no_larger_runners.py",
+ hint="use ubuntu-latest, macos-26 or windows-latest; opt-outs: .github/larger-runners-allowed.txt",
+ ci=(("ci-block-larger-runners.yml", "check_no_larger_runners.py"),)))
+
# Repo-wide gate (no path filter in CI): every image ref comes from the catalog.
if changes.files or changes.full:
add(Check("image-refs", "image refs come from the catalog",