Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci-gate: every pull request needs a sign-off, forks get a GitHub quick run (#6584)

cua-agents[bot]8563c3b0c514
diff --git a/.github/scripts/tests/test_ci_gate_workflow.py b/.github/scripts/tests/test_ci_gate_workflow.py
index e875fca4ee..68eeab128c 100644
--- a/.github/scripts/tests/test_ci_gate_workflow.py
+++ b/.github/scripts/tests/test_ci_gate_workflow.py
@@ -324,70 +324,89 @@ def test_quick_stage_is_wired_into_heavy_workflows():
assert '!failure() && !cancelled()' in str(job['if']), f'{path.name}: {name}'
-LOCAL_CI_FAST = STEPS['Check the local sign-off on agent pull requests (fail fast)']['run']
-LOCAL_CI_WAIT = STEPS['Require local sign-off on agent pull requests']['run']
-GH_STUB = '''
+LOCAL_CI_FAST = STEPS['Check the sign-off on the pull request (fail fast)']['run']
+LOCAL_CI_WAIT = STEPS['Require the sign-off on the pull request']['run']
+# The stub answers every sign-off lookup with STATE: a commit status for a branch of this repository,
+# the newest CI: Quick run for a fork. SOURCE records which one the gate asked.
+GH_STUB = """
sleep() { :; }
gh() {
case "$*" in
- *'/pulls/'*) printf '%s\\t%s\\t%s\\t%s\\n' "$AUTHOR" "deadbeefcafe" "$REF" "$HEAD_REPO" ;;
- *'/labels'*) printf '%s\\n' "$LABELS" ;;
- *'/statuses'*) printf '%s\\n' "$STATE" ;;
+ *'/pulls/'*) printf '%s\\t%s\\n' "deadbeefcafe" "$HEAD_REPO" ;;
+ *'/statuses'*) echo statuses >> "$SOURCE"; printf '%s\\n' "$STATE" ;;
+ *'/workflows/ci-quick.yml/runs'*) echo quick-run >> "$SOURCE"; printf '%s\\n' "$STATE" ;;
esac
}
-'''
+"""
-def run_local_ci(author='someone', labels='', ref='fix/x', head_repo='fork/repo', state='', require='true'):
- """Run the two sign-off steps in order, as the job does: the fast check, then (for an
- agent pull request) the wait that is the last step. Returns the first failure, or the last result."""
+def run_local_ci(head_repo='test/repo', state='', require='true'):
+ """Run the two sign-off steps in order, as the job does: the fast check, then the wait that is
+ the last step. Returns the first failure, or the last result; `.source` is what was asked."""
import tempfile
- with tempfile.NamedTemporaryFile('r+') as out:
+ with tempfile.NamedTemporaryFile('r+') as out, tempfile.NamedTemporaryFile('r+') as source:
env = {**os.environ, 'PR_NUMBER': '7', 'QUEUE_REF': '', 'GITHUB_REPOSITORY': 'test/repo',
- 'AUTHOR': author, 'LABELS': labels, 'REF': ref, 'HEAD_REPO': head_repo, 'STATE': state,
+ 'HEAD_REPO': head_repo, 'STATE': state, 'SOURCE': source.name,
'REQUIRE_LOCAL_CI': require, 'GITHUB_OUTPUT': out.name}
fast = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_FAST], env=env,
capture_output=True, text=True, timeout=20)
outputs = dict(line.split('=', 1) for line in out.read().splitlines())
- if fast.returncode != 0 or outputs.get('agent') != 'true':
- return fast
- wait = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_WAIT], env={**env, 'SHA': outputs['sha']},
- capture_output=True, text=True, timeout=20)
- wait.stdout = fast.stdout + wait.stdout
+ if fast.returncode != 0:
+ fast.source = source.read().split()
+ return fast
+ wait = subprocess.run(['bash', '-e', '-c', GH_STUB + LOCAL_CI_WAIT],
+ env={**env, 'SHA': outputs['sha'], 'KIND': outputs['kind']},
+ capture_output=True, text=True, timeout=20)
+ wait.stdout = fast.stdout + wait.stdout
+ wait.source = source.read().split()
return wait
[email protected]('kwargs', [
- {'author': 'cua-agents[bot]'},
- {'labels': 'bug,agent'},
- {'ref': 'agent/cua-1-x', 'head_repo': 'test/repo'},
-])
-def test_local_ci_is_required_for_agent_pull_requests(kwargs):
- missing = run_local_ci(**kwargs)
+def test_every_pull_request_needs_a_sign_off_whoever_opens_it():
+ # The author, the labels and the commit trailers are not read: a branch of this repository
+ # always needs the local-ci status, a fork always needs the CI: Quick run.
+ assert 'author' not in LOCAL_CI_FAST.lower().replace('author:', '')
+ for word in ('labels', 'trailer', 'agent/'):
+ assert word not in LOCAL_CI_FAST
+ assert word not in LOCAL_CI_WAIT
+
+
+def test_branch_of_this_repo_needs_a_local_ci_status():
+ missing = run_local_ci()
assert missing.returncode == 1
- assert 'Run bin/ci and bin/ci --signoff on this head commit.' in missing.stdout
- assert run_local_ci(**kwargs, state='failure').returncode == 1
- assert run_local_ci(**kwargs, state='success').returncode == 0
+ assert 'Run bin/ci --quick and bin/ci --signoff on this head commit.' in missing.stdout
+ assert set(missing.source) == {'statuses'}
+ assert run_local_ci(state='failure').returncode == 1
+ assert run_local_ci(state='success').returncode == 0
[email protected]('kwargs', [
- {},
- {'ref': 'agent/x', 'head_repo': 'fork/repo'}, # a fork cannot opt in or out by branch name
- {'labels': 'agents,bug'},
-])
-def test_local_ci_is_not_required_for_community_pull_requests(kwargs):
- done = run_local_ci(**kwargs)
- assert done.returncode == 0, done.stdout
- assert 'Not an agent pull request' in done.stdout
-
-
-def test_local_ci_is_report_only_until_the_switch_is_on():
- for state in ('', 'failure'):
- done = run_local_ci(author='cua-agents[bot]', state=state, require='')
- assert done.returncode == 0, done.stdout
- assert '::warning' in done.stdout
- assert 'Run bin/ci and bin/ci --signoff on this head commit.' in done.stdout
- assert run_local_ci(author='cua-agents[bot]', state='success', require='').returncode == 0
+def test_fork_needs_a_successful_quick_run_in_github_actions():
+ missing = run_local_ci(head_repo='someone/fork')
+ assert missing.returncode == 1
+ assert 'no successful run on this head commit' in missing.stdout
+ assert set(missing.source) == {'quick-run'} # never the local-ci status, which only our Macs set
+ assert run_local_ci(head_repo='someone/fork', state='failure').returncode == 1
+ assert run_local_ci(head_repo='someone/fork', state='success').returncode == 0
+
+
+def test_sign_off_is_report_only_until_the_switch_is_on():
+ for head_repo in ('test/repo', 'someone/fork'):
+ for state in ('', 'failure'):
+ done = run_local_ci(head_repo=head_repo, state=state, require='')
+ assert done.returncode == 0, done.stdout
+ assert '::warning' in done.stdout
+ assert run_local_ci(head_repo=head_repo, state='success', require='').returncode == 0
+
+
+def test_the_quick_workflow_a_fork_relies_on_is_safe_to_run_on_its_code():
+ wf = yaml.safe_load((WORKFLOWS / 'ci-quick.yml').read_text())
+ triggers = wf.get(True) or wf.get('on')
+ assert 'pull_request' in triggers # runs on fork PRs, GitHub-hosted
+ assert 'pull_request_target' not in triggers
+ assert wf['permissions'] == {'contents': 'read'}
+ text = (WORKFLOWS / 'ci-quick.yml').read_text()
+ assert 'secrets.' not in text
+ assert 'self-hosted' not in text
def test_quick_stage_runs_once_per_commit():
diff --git a/.github/workflows/ci-gate.yml b/.github/workflows/ci-gate.yml
index e55651820d..36440599a8 100644
--- a/.github/workflows/ci-gate.yml
+++ b/.github/workflows/ci-gate.yml
@@ -78,16 +78,20 @@ jobs:
# The files API stops at 3000 entries; a PR that large waits on everything.
if (( $(wc -l < "$RUNNER_TEMP/files.txt") >= 3000 )); then echo ".github/too-many-files" > "$RUNNER_TEMP/files.txt"; fi
python3 base/.github/scripts/ci_gate_scope.py --labels "$labels" --body-file "$RUNNER_TEMP/pr-body.txt" < "$RUNNER_TEMP/files.txt"
- # Pull requests from our agents must carry a `local-ci` commit status on the head commit, set by
- # `bin/ci --signoff` (or our Mac CI runner) after a green local `bin/ci`. An agent PR is one
- # opened by the Cua Agents GitHub App (`cua-agents[bot]`), labelled `agent`, or opened from an
- # `agent/` branch of this repository. Community PRs are not asked for it.
+ # Every pull request needs a sign-off on its head commit. Which one depends on where the code lives:
+ # - A branch of this repository (members and agents, any author): a `local-ci` commit status, set
+ # by `bin/ci --signoff` (or our Mac CI runner, which `ci-remote-sweep` starts for each such PR)
+ # after a green quick run.
+ # - A fork (community): untrusted code, which never runs on our Macs. Its sign-off is a successful
+ # `CI: Quick` run (ci-quick.yml, `bin/ci --quick`) on the head commit in GitHub Actions: a
+ # pull_request job with a read-only token and no secrets.
+ # Neither the author, the labels nor the commit trailers decide anything here.
# The requirement is report-only (a warning) until the repository variable REQUIRE_LOCAL_CI is
- # set to "true"; that variable is the one switch that makes a missing status block the merge.
+ # set to "true"; that variable is the one switch that makes a missing sign-off block the merge.
# A sign-off can take 20-40 minutes (a queued run on the Mac runner), so it is checked twice:
- # here, once and without waiting, so a `failure` fails the gate at once; and after the other
- # workflows have finished (the last step), where a still-missing status is waited for.
- - name: Check the local sign-off on agent pull requests (fail fast)
+ # here, once and without waiting, so a failure fails the gate at once; and after the other
+ # workflows have finished (the last step), where a still-missing sign-off is waited for.
+ - name: Check the sign-off on the pull request (fail fast)
id: signoff
env:
GH_TOKEN: ${{ github.token }}
@@ -102,27 +106,29 @@ jobs:
[[ "$QUEUE_REF" =~ /pr-([0-9]+) ]] || { echo "::error::cannot read the pull request from $QUEUE_REF"; exit 1; }
pr=${BASH_REMATCH[1]}
fi
- info=$(api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '[.user.login, .head.sha, .head.ref, (.head.repo.full_name // "")] | @tsv')
- IFS=$'\t' read -r author sha ref head_repo <<<"$info"
- labels=$(api "repos/$GITHUB_REPOSITORY/issues/$pr/labels" --jq '[.[].name] | join(",")')
- why=
- if [[ "$author" == "cua-agents[bot]" || "$author" == "cua-agents" ]]; then why="opened by $author"
- elif [[ ",$labels," == *,agent,* ]]; then why="labelled agent"
- elif [[ "$ref" == agent/* && "$head_repo" == "$GITHUB_REPOSITORY" ]]; then why="opened from the $ref branch"
+ info=$(api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '[.head.sha, (.head.repo.full_name // "")] | @tsv')
+ IFS=$'\t' read -r sha head_repo <<<"$info"
+ if [[ "$head_repo" == "$GITHUB_REPOSITORY" ]]; then
+ kind=local
+ what="a successful local-ci status (bin/ci --quick, then bin/ci --signoff)"
+ else
+ kind=fork
+ what="a successful CI: Quick run (bin/ci --quick in GitHub Actions)"
fi
- if [[ -z "$why" ]]; then
- echo "Not an agent pull request: no local-ci status required."
- echo "agent=false" >> "$GITHUB_OUTPUT"
- exit 0
+ echo "Requiring $what on ${sha:0:10}."
+ { echo "kind=$kind"; echo "sha=$sha"; } >> "$GITHUB_OUTPUT"
+ if [[ "$kind" == local ]]; then
+ state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')
+ else
+ # The newest run only: an older one may have been cancelled by a newer push.
+ state=$(api "repos/$GITHUB_REPOSITORY/actions/workflows/ci-quick.yml/runs?head_sha=$sha&event=pull_request&per_page=30" \
+ --jq '[.workflow_runs[]] | sort_by(.created_at) | last | if . == null or .status != "completed" then "" elif .conclusion == "success" then "success" elif .conclusion == "cancelled" then "" else "failure" end')
fi
- echo "Agent pull request ($why): requiring a successful local-ci status on ${sha:0:10}."
- { echo "agent=true"; echo "sha=$sha"; } >> "$GITHUB_OUTPUT"
- state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')
- echo "local-ci on ${sha:0:10}: ${state:-not set yet}"
+ echo "sign-off on ${sha:0:10}: ${state:-not set yet}"
if [[ "$state" == failure || "$state" == error ]]; then
- msg="local-ci reported $state on ${sha:0:10}. Fix the failing checks, push, and rerun bin/ci."
- if [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then echo "::error title=local-ci failed::$msg"; exit 1; fi
- echo "::warning title=local-ci failed (report-only)::$msg"
+ msg="The sign-off reported $state on ${sha:0:10}. Fix the failing checks and push."
+ if [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then echo "::error title=sign-off failed::$msg"; exit 1; fi
+ echo "::warning title=sign-off failed (report-only)::$msg"
fi
- name: Wait for every other workflow on the pull request head
@@ -235,13 +241,13 @@ jobs:
exit $failed
# Last step: the other workflows are done, so only the sign-off can still be missing. Wait for
- # it (bounded, 75 minutes: a queued 20-40 minute run on the Mac runner plus slack). A `failure`
- # or `error` fails at once. Report-only mode looks once and only warns.
- - name: Require local sign-off on agent pull requests
- if: steps.signoff.outputs.agent == 'true'
+ # it (bounded, 75 minutes: a queued 20-40 minute run on the Mac runner plus slack). A failure
+ # fails at once. Report-only mode looks once and only warns.
+ - name: Require the sign-off on the pull request
env:
GH_TOKEN: ${{ github.token }}
SHA: ${{ steps.signoff.outputs.sha }}
+ KIND: ${{ steps.signoff.outputs.kind }}
REQUIRE_LOCAL_CI: ${{ vars.REQUIRE_LOCAL_CI }}
run: |
set -euo pipefail
@@ -250,15 +256,24 @@ jobs:
tries=1
[[ "${REQUIRE_LOCAL_CI:-}" == true ]] && tries=75
for i in $(seq 1 "$tries"); do
- state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')
- if [[ "$state" == success ]]; then echo "local-ci: success"; exit 0; fi
+ if [[ "$KIND" == local ]]; then
+ state=$(api "repos/$GITHUB_REPOSITORY/commits/$sha/statuses?per_page=100" --jq '[.[] | select(.context == "local-ci")][0].state // ""')
+ else
+ state=$(api "repos/$GITHUB_REPOSITORY/actions/workflows/ci-quick.yml/runs?head_sha=$sha&event=pull_request&per_page=30" \
+ --jq '[.workflow_runs[]] | sort_by(.created_at) | last | if . == null or .status != "completed" then "" elif .conclusion == "success" then "success" elif .conclusion == "cancelled" then "" else "failure" end')
+ fi
+ if [[ "$state" == success ]]; then echo "sign-off: success"; exit 0; fi
[[ "$state" == failure || "$state" == error ]] && break
(( i < tries )) && sleep 60
done
- msg="Run bin/ci and bin/ci --signoff on this head commit. (local-ci on ${sha:0:10}: ${state:-not set})"
+ if [[ "$KIND" == local ]]; then
+ msg="Run bin/ci --quick and bin/ci --signoff on this head commit. (local-ci on ${sha:0:10}: ${state:-not set})"
+ else
+ msg="The CI: Quick workflow has no successful run on this head commit. (${sha:0:10}: ${state:-not run yet}; a first-time contributor's run needs a maintainer to approve it)"
+ fi
if [[ "${REQUIRE_LOCAL_CI:-}" == true ]]; then
- echo "::error title=local-ci missing::$msg"
+ echo "::error title=sign-off missing::$msg"
exit 1
fi
- echo "::warning title=local-ci missing (report-only)::$msg"
- echo "local-ci is report-only until the repository variable REQUIRE_LOCAL_CI is true; not failing the gate."
+ echo "::warning title=sign-off missing (report-only)::$msg"
+ echo "The sign-off is report-only until the repository variable REQUIRE_LOCAL_CI is true; not failing the gate."
diff --git a/docs/contributing/CI.md b/docs/contributing/CI.md
index 1ec75984f2..b090f3a256 100644
--- a/docs/contributing/CI.md
+++ b/docs/contributing/CI.md
@@ -76,10 +76,18 @@ After a green run on a clean tree, `bin/ci --signoff` sets the `local-ci` commit
`gh api`. If the commit is not on GitHub yet, push and run it again: it reuses the green run. It refuses to
sign off a run that failed or skipped a check. Sign off again after every new commit.
-`ci-gate` requires a successful `local-ci` status on the head commit for **agent pull requests**: those
-opened by the Cua Agents GitHub App (`cua-agents[bot]`), labelled `agent`, or opened from an `agent/` branch
-of this repository. Community pull requests are not asked for it. When it is missing the gate says "Run
-bin/ci and bin/ci --signoff on this head commit."
+`ci-gate` requires a sign-off on the head commit of **every pull request**, whoever opens it. The author,
+labels and commit trailers are not read.
+
+- **A branch of this repository** (members and agents): a successful `local-ci` status. Our Mac runner sets
+ it after `bin/ci --quick` (`ci-remote-sweep` starts that run for each such PR), or you set it with
+ `bin/ci --signoff`. When it is missing the gate says "Run bin/ci --quick and bin/ci --signoff on this head
+ commit."
+- **A fork** (community): fork code never runs on our Macs. The sign-off is a successful `CI: Quick` run
+ (`ci-quick.yml`, which runs `bin/ci --quick`) on the head commit in GitHub Actions: a `pull_request` job
+ with a read-only token and no secrets. A first-time contributor's run needs a maintainer to approve it.
+
+The `agent` label is only for filtering; it does not change the gate.
The requirement is **report-only** at first: the gate prints that line as a warning and passes. Setting the
repository variable `REQUIRE_LOCAL_CI` to `true` makes it blocking. It is the only switch. When blocking, the