Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci-gate: every pull request needs a sign-off, forks get a GitHub quick run (#6584)
cua-agents[bot]8563c3b0c514
diff --git a/.github/scripts/tests/test_ci_gate_workflow.py b/.github/scripts/tests/test_ci_gate_workflow.pyindex e875fca4ee..68eeab128c 100644--- a/.github/scripts/tests/test_ci_gate_workflow.py+++ b/.github/scripts/tests/test_ci_gate_workflow.py@@ -324,70 +324,89 @@ def test_quick_stage_is_wired_into_heavy_workflows():assert '!failure() && !cancelled()' in str(job['if']), f'{path.name}: {name}'-LOCAL_CI_FAST = STEPS['Check the local sign-off on agent pull requests (fail fast)']['run']-LOCAL_CI_WAIT = STEPS['Require local sign-off on agent pull requests']['run']-GH_STUB = '''+LOCAL_CI_FAST = STEPS['Check the sign-off on the pull request (fail fast)']['run']+LOCAL_CI_WAIT = STEPS['Require the sign-off on the pull request']['run']+# The stub answers every sign-off lookup with STATE: a commit status for a branch of this repository,+# the newest CI: Quick run for a fork. SOURCE records which one the gate asked.+GH_STUB = """sleep() { :; }gh() {case "$*" in- *'/pulls/'*) printf '%s\\t%s\\t%s\\t%s\\n' "$AUTHOR" "deadbeefcafe" "$REF" "$HEAD_REPO" ;;- *'/labels'*) printf '%s\\n' "$LABELS" ;;- *'/statuses'*) printf '%s\\n' "$STATE" ;;+ *'/pulls/'*) printf '%s\\t%s\\n' "deadbeefcafe" "$HEAD_REPO" ;;+ *'/statuses'*) echo statuses >> "$SOURCE"; printf '%s\\n' "$STATE" ;;+ *'/workflows/ci-quick.yml/runs'*) echo quick-run >> "$SOURCE"; printf '%s\\n' "$STATE" ;;esac}-'''+"""-def run_local_ci(author='someone', labels='', ref='fix/x', head_repo='fork/repo', state='', require='true'):- """Run the two sign-off steps in order, as the job does: the fast check, then (for an- agent pull request) the wait that is the last step. Returns the first failure, or the last result."""+def run_local_ci(head_repo='test/repo', state='', require='true'):+ """Run the two sign-off steps in order, as the job does: the fast check, then the wait that is+ the last step. Returns the first failure, or the last result; `.source` is what was asked."""import tempfile- with tempfile.NamedTemporaryFile('r+') as out:+ with tempfile.NamedTemporaryFile('r+') as out, tempfile.NamedTemporaryFile('r+') as source:env = {**os.environ, 'PR_NUMBER': '7', 'QUEUE_REF': '', 'GITHUB_REPOSITORY': 'test/repo',- 'AUTHOR': author, 'LABELS': labels, 'REF': ref, 'HEAD_REPO': head_repo, 'STATE': state,+ 'HEAD_REPO': head_repo, 'STATE': state, 'SOURCE': source.name,'REQUIRE_LOCAL_CI': require, 'GITHUB_OUTPUT': out.name}fast = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_FAST], env=env,capture_output=True, text=True, timeout=20)outputs = dict(line.split('=', 1) for line in out.read().splitlines())- if fast.returncode != 0 or outputs.get('agent') != 'true':- return fast- wait = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_WAIT], env={**env, 'SHA': outputs['sha']},- capture_output=True, text=True, timeout=20)- wait.stdout = fast.stdout + wait.stdout+ if fast.returncode != 0:+ fast.source = source.read().split()+ return fast+ wait = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_WAIT],+ env={**env, 'SHA': outputs['sha'], 'KIND': outputs['kind']},+ capture_output=True, text=True, timeout=20)+ wait.stdout = fast.stdout + wait.stdout+ wait.source = source.read().split()return wait[email protected]('kwargs', [- {'author': 'cua-agents[bot]'},- {'labels': 'bug,agent'},- {'ref': 'agent/cua-1-x', 'head_repo': 'test/repo'},-])-def test_local_ci_is_required_for_agent_pull_requests(kwargs):- missing = run_local_ci(**kwargs)+def test_every_pull_request_needs_a_sign_off_whoever_opens_it():+ # The author, the labels and the commit trailers are not read: a branch of this repository+ # always needs the local-ci status, a fork always needs the CI: Quick run.+ assert 'author' not in LOCAL_CI_FAST.lower().replace('author:', '')+ for word in ('labels', 'trailer', 'agent/'):+ assert word not in LOCAL_CI_FAST+ assert word not in LOCAL_CI_WAIT+++def test_branch_of_this_repo_needs_a_local_ci_status():+ missing = run_local_ci()assert missing.returncode == 1- assert 'Run bin/ci and bin/ci --signoff on this head commit.' in missing.stdout- assert run_local_ci(**kwargs, state='failure').returncode == 1- assert run_local_ci(**kwargs, state='success').returncode == 0+ assert 'Run bin/ci --quick and bin/ci --signoff on this head commit.' in missing.stdout+ assert set(missing.source) == {'statuses'}+ assert run_local_ci(state='failure').returncode == 1+ assert run_local_ci(state='success').returncode == 0[email protected]('kwargs', [- {},- {'ref': 'agent/x', 'head_repo': 'fork/repo'}, # a fork cannot opt in or out by branch name- {'labels': 'agents,bug'},-])-def test_local_ci_is_not_required_for_community_pull_requests(kwargs):- done = run_local_ci(**kwargs)- assert done.returncode == 0, done.stdout- assert 'Not an agent pull request' in done.stdout---def test_local_ci_is_report_only_until_the_switch_is_on():- for state in ('', 'failure'):- done = run_local_ci(author='cua-agents[bot]', state=state, require='')- assert done.returncode == 0, done.stdout- assert '::warning' in done.stdout- assert 'Run bin/ci and bin/ci --signoff on this head commit.' in done.stdout- assert run_local_ci(author='cua-agents[bot]', state='success', require='').returncode == 0+def test_fork_needs_a_successful_quick_run_in_github_actions():+ missing = run_local_ci(head_repo='someone/fork')+ assert missing.returncode == 1+ assert 'no successful run on this head commit' in missing.stdout+ assert set(missing.source) == {'quick-run'} # never the local-ci status, which only our Macs set+ assert run_local_ci(head_repo='someone/fork', state='failure').returncode == 1+ assert run_local_ci(head_repo='someone/fork', state='success').returncode == 0+++def test_sign_off_is_report_only_until_the_switch_is_on():+ for head_repo in ('test/repo', 'someone/fork'):+ for state in ('', 'failure'):+ done = run_local_ci(head_repo=head_repo, state=state, require='')+ assert done.returncode == 0, done.stdout+ assert '::warning' in done.stdout+ assert run_local_ci(head_repo=head_repo, state='success', require='').returncode == 0+++def test_the_quick_workflow_a_fork_relies_on_is_safe_to_run_on_its_code():+ wf = yaml.safe_load((WORKFLOWS / 'ci-quick.yml').read_text())+ triggers = wf.get(True) or wf.get('on')+ assert 'pull_request' in triggers # runs on fork PRs, GitHub-hosted+ assert 'pull_request_target' not in triggers+ assert wf['permissions'] == {'contents': 'read'}+ text = (WORKFLOWS / 'ci-quick.yml').read_text()+ assert 'secrets.' not in text+ assert 'self-hosted' not in textdef test_quick_stage_runs_once_per_commit():diff --git a/.github/workflows/ci-gate.yml b/.github/workflows/ci-gate.ymlindex e55651820d..36440599a8 100644--- a/.github/workflows/ci-gate.yml+++ b/.github/workflows/ci-gate.yml@@ -78,16 +78,20 @@ jobs:# The files API stops at 3000 entries; a PR that large waits on everything.if (( $(wc -l < "$RUNNER_TEMP/files.txt") >= 3000 )); then echo ".github/too-many-files" > "$RUNNER_TEMP/files.txt"; fipython3 base/.github/scripts/ci_gate_scope.py --labels "$labels" --body-file "$RUNNER_TEMP/pr-body.txt" < "$RUNNER_TEMP/files.txt"- # Pull requests from our agents must carry a `local-ci` commit status on the head commit, set by- # `bin/ci --signoff` (or our Mac CI runner) after a green local `bin/ci`. An agent PR is one- # opened by the Cua Agents GitHub App (`cua-agents[bot]`), labelled `agent`, or opened from an- # `agent/` branch of this repository. Community PRs are not asked for it.+ # Every pull request needs a sign-off on its head commit. Which one depends on where the code lives:+ # - A branch of this repository (members and agents, any author): a `local-ci` commit status, set+ # by `bin/ci --signoff` (or our Mac CI runner, which `ci-remote-sweep` starts for each such PR)+ # after a green quick run.+ # - A fork (community): untrusted code, which never runs on our Macs. Its sign-off is a successful+ # `CI: Quick` run (ci-quick.yml, `bin/ci --quick`) on the head commit in GitHub Actions: a+ # pull_request job with a read-only token and no secrets.+ # Neither the author, the labels nor the commit trailers decide anything here.# The requirement is report-only (a warning) until the repository variable REQUIRE_LOCAL_CI is- # set to "true"; that variable is the one switch that makes a missing status block the merge.+ # set to "true"; that variable is the one switch that makes a missing sign-off block the merge.# A sign-off can take 20-40 minutes (a queued run on the Mac runner), so it is checked twice:- # here, once and without waiting, so a `failure` fails the gate at once; and after the other- # workflows have finished (the last step), where a still-missing status is waited for.- - name: Check the local sign-off on agent pull requests (fail fast)+ # here, once and without waiting, so a failure fails the gate at once; and after the other+ # workflows have finished (the last step), where a still-missing sign-off is waited for.+ - name: Check the sign-off on the pull request (fail fast)id: signoffenv:GH_TOKEN: ${{ github.token }}@@ -102,27 +106,29 @@ jobs:[[ "$QUEUE_REF" =~ /pr-([0-9]+) ]] || { echo "::error::cannot read the pull request from $QUEUE_REF"; exit 1; }pr=${BASH_REMATCH[1]}fi- info=$(api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '[.user.login, .head.sha, .head.ref, (.head.repo.full_name // "")] | @tsv')- IFS=$'\t' read -r author sha ref head_repo <<<"$info"- labels=$(api "repos/$GITHUB_REPOSITORY/issues/$pr/labels" --jq '[.[].name] | join(",")')- why=- if [[ "$author" == "cua-agents[bot]" || "$author" == "cua-agents" ]]; then why="opened by $author"- elif [[ ",$labels," == *,agent,* ]]; then why="labelled agent"- elif [[ "$ref" == agent/* && "$head_repo" == "$GITHUB_REPOSITORY" ]]; then why="opened from the $ref branch"+ info=$(api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '[.head.sha, (.head.repo.full_name // "")] | @tsv')+ IFS=$'\t' read -r sha head_repo <<<"$info"+ if [[ "$head_repo" == "$GITHUB_REPOSITORY" ]]; then+ kind=local+ what="a successful local-ci status (bin/ci --quick, then bin/ci --signoff)"+ else+ kind=fork+ what="a successful CI: Quick run (bin/ci --quick in GitHub Actions)"fi- if [[ -z "$why" ]]; then- echo "Not an agent pull request: no local-ci status required."- echo "agent=false" >> "$GITHUB_OUTPUT"- exit 0+ echo "Requiring $what on ${sha:0:10}."+ { echo "kind=$kind"; echo "sha=$sha"; } >> "$GITHUB_OUTPUT"+ if [[ "$kind" == local ]]; then+ state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')+ else+ # The newest run only: an older one may have been cancelled by a newer push.+ state=$(api "repos/$GITHUB_REPOSITORY/actions/workflows/ci-quick.yml/runs?head_sha=$sha&event=pull_request&per_page=30" \+ --jq '[.workflow_runs[]] | sort_by(.created_at) | last | if . == null or .status != "completed" then "" elif .conclusion == "success" then "success" elif .conclusion == "cancelled" then "" else "failure" end')fi- echo "Agent pull request ($why): requiring a successful local-ci status on ${sha:0:10}."- { echo "agent=true"; echo "sha=$sha"; } >> "$GITHUB_OUTPUT"- state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')- echo "local-ci on ${sha:0:10}: ${state:-not set yet}"+ echo "sign-off on ${sha:0:10}: ${state:-not set yet}"if [[ "$state" == failure || "$state" == error ]]; then- msg="local-ci reported $state on ${sha:0:10}. Fix the failing checks, push, and rerun bin/ci."- if [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then echo "::error title=local-ci failed::$msg"; exit 1; fi- echo "::warning title=local-ci failed (report-only)::$msg"+ msg="The sign-off reported $state on ${sha:0:10}. Fix the failing checks and push."+ if [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then echo "::error title=sign-off failed::$msg"; exit 1; fi+ echo "::warning title=sign-off failed (report-only)::$msg"fi- name: Wait for every other workflow on the pull request head@@ -235,13 +241,13 @@ jobs:exit $failed# Last step: the other workflows are done, so only the sign-off can still be missing. Wait for- # it (bounded, 75 minutes: a queued 20-40 minute run on the Mac runner plus slack). A `failure`- # or `error` fails at once. Report-only mode looks once and only warns.- - name: Require local sign-off on agent pull requests- if: steps.signoff.outputs.agent == 'true'+ # it (bounded, 75 minutes: a queued 20-40 minute run on the Mac runner plus slack). A failure+ # fails at once. Report-only mode looks once and only warns.+ - name: Require the sign-off on the pull requestenv:GH_TOKEN: ${{ github.token }}SHA: ${{ steps.signoff.outputs.sha }}+ KIND: ${{ steps.signoff.outputs.kind }}REQUIRE_LOCAL_CI: ${{ vars.REQUIRE_LOCAL_CI }}run: |set -euo pipefail@@ -250,15 +256,24 @@ jobs:tries=1[[ "${REQUIRE_LOCAL_CI:-}" == true ]] && tries=75for i in $(seq 1 "$tries"); do- state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')- if [[ "$state" == success ]]; then echo "local-ci: success"; exit 0; fi+ if [[ "$KIND" == local ]]; then+ state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')+ else+ state=$(api "repos/$GITHUB_REPOSITORY/actions/workflows/ci-quick.yml/runs?head_sha=$sha&event=pull_request&per_page=30" \+ --jq '[.workflow_runs[]] | sort_by(.created_at) | last | if . == null or .status != "completed" then "" elif .conclusion == "success" then "success" elif .conclusion == "cancelled" then "" else "failure" end')+ fi+ if [[ "$state" == success ]]; then echo "sign-off: success"; exit 0; fi[[ "$state" == failure || "$state" == error ]] && break(( i < tries )) && sleep 60done- msg="Run bin/ci and bin/ci --signoff on this head commit. (local-ci on ${sha:0:10}: ${state:-not set})"+ if [[ "$KIND" == local ]]; then+ msg="Run bin/ci --quick and bin/ci --signoff on this head commit. (local-ci on ${sha:0:10}: ${state:-not set})"+ else+ msg="The CI: Quick workflow has no successful run on this head commit. (${sha:0:10}: ${state:-not run yet}; a first-time contributor's run needs a maintainer to approve it)"+ fiif [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then- echo "::error title=local-ci missing::$msg"+ echo "::error title=sign-off missing::$msg"exit 1fi- echo "::warning title=local-ci missing (report-only)::$msg"- echo "local-ci is report-only until the repository variable REQUIRE_LOCAL_CI is true; not failing the gate."+ echo "::warning title=sign-off missing (report-only)::$msg"+ echo "The sign-off is report-only until the repository variable REQUIRE_LOCAL_CI is true; not failing the gate."diff --git a/docs/contributing/CI.md b/docs/contributing/CI.mdindex 1ec75984f2..b090f3a256 100644--- a/docs/contributing/CI.md+++ b/docs/contributing/CI.md@@ -76,10 +76,18 @@ After a green run on a clean tree, `bin/ci --signoff` sets the `local-ci` commit`gh api`. If the commit is not on GitHub yet, push and run it again: it reuses the green run. It refuses tosign off a run that failed or skipped a check. Sign off again after every new commit.-`ci-gate` requires a successful `local-ci` status on the head commit for **agent pull requests**: those-opened by the Cua Agents GitHub App (`cua-agents[bot]`), labelled `agent`, or opened from an `agent/` branch-of this repository. Community pull requests are not asked for it. When it is missing the gate says "Run-bin/ci and bin/ci --signoff on this head commit."+`ci-gate` requires a sign-off on the head commit of **every pull request**, whoever opens it. The author,+labels and commit trailers are not read.++- **A branch of this repository** (members and agents): a successful `local-ci` status. Our Mac runner sets+ it after `bin/ci --quick` (`ci-remote-sweep` starts that run for each such PR), or you set it with+ `bin/ci --signoff`. When it is missing the gate says "Run bin/ci --quick and bin/ci --signoff on this head+ commit."+- **A fork** (community): fork code never runs on our Macs. The sign-off is a successful `CI: Quick` run+ (`ci-quick.yml`, which runs `bin/ci --quick`) on the head commit in GitHub Actions: a `pull_request` job+ with a read-only token and no secrets. A first-time contributor's run needs a maintainer to approve it.++The `agent` label is only for filtering; it does not change the gate.The requirement is **report-only** at first: the gate prints that line as a warning and passes. Setting therepository variable `REQUIRE_LOCAL_CI` to `true` makes it blocking. It is the only switch. When blocking, the