Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Align PR review workflows with OpenHands repos (#51)

Graham Neubigfb03842d1bb0
diff --git a/.github/workflows/pr-review-by-openhands.yml b/.github/workflows/pr-review-by-openhands.yml
new file mode 100644
index 0000000000..4e5a437313
--- /dev/null
+++ b/.github/workflows/pr-review-by-openhands.yml
@@ -0,0 +1,63 @@
+name: PR Review by OpenHands
+
+on:
+ # Use pull_request for same-repo PRs so workflow changes can self-verify in PRs.
+ pull_request:
+ types: [opened, ready_for_review, labeled, review_requested]
+ # Use pull_request_target for fork PRs.
+ # The bot token used here is intentionally scoped to PR review operations,
+ # so the remaining blast radius is bounded even though PR content is untrusted.
+ pull_request_target:
+ types: [opened, ready_for_review, labeled, review_requested]
+
+permissions:
+ contents: read
+ pull-requests: write
+ issues: write
+
+jobs:
+ pr-review:
+ # Run on same-repo PRs via pull_request and on fork PRs via pull_request_target.
+ # Trigger when one of the following conditions is met:
+ # 1. A new non-draft PR is opened by a non-first-time contributor, OR
+ # 2. A draft PR is converted to ready for review by a non-first-time contributor, OR
+ # 3. The 'review-this' label is added, OR
+ # 4. openhands-agent or all-hands-bot is requested as a reviewer
+ # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request.
+ if: |
+ (
+ (
+ github.event_name == 'pull_request' &&
+ github.event.pull_request.head.repo.full_name == github.repository
+ ) ||
+ (
+ github.event_name == 'pull_request_target' &&
+ github.event.pull_request.head.repo.full_name != github.repository
+ )
+ ) &&
+ (
+ (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
+ (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
+ (github.event.action == 'labeled' && github.event.label.name == 'review-this') ||
+ (
+ github.event.action == 'review_requested' &&
+ (
+ github.event.requested_reviewer.login == 'openhands-agent' ||
+ github.event.requested_reviewer.login == 'all-hands-bot'
+ )
+ )
+ )
+ concurrency:
+ group: pr-review-${{ github.event.pull_request.number }}
+ cancel-in-progress: true
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Run PR Review
+ uses: OpenHands/extensions/plugins/pr-review@main
+ with:
+ llm-model: litellm_proxy/claude-sonnet-4-5-20250929
+ llm-base-url: https://llm-proxy.app.all-hands.dev
+ use-sub-agents: "false"
+ llm-api-key: ${{ secrets.LLM_API_KEY }}
+ github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC }}
+ lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }}
diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
new file mode 100644
index 0000000000..b163ce9ac6
--- /dev/null
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -0,0 +1,82 @@
+name: PR Review Evaluation
+
+# This workflow evaluates how well PR review comments were addressed.
+# It runs when a PR is closed to assess review effectiveness.
+#
+# Security note: pull_request_target is safe here because:
+# 1. Only triggers on PR close (not on code changes)
+# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs
+# 3. Runs evaluation scripts from the extensions repo, not from the PR
+
+on:
+ pull_request_target:
+ types: [closed]
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ evaluate:
+ runs-on: ubuntu-24.04
+ env:
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ REPO_NAME: ${{ github.repository }}
+ PR_MERGED: ${{ github.event.pull_request.merged }}
+
+ steps:
+ - name: Download review trace artifact
+ id: download-trace
+ uses: dawidd6/action-download-artifact@v20
+ continue-on-error: true
+ with:
+ workflow: pr-review-by-openhands.yml
+ name: pr-review-trace-${{ github.event.pull_request.number }}
+ path: trace-info
+ search_artifacts: true
+ if_no_artifact_found: warn
+
+ - name: Check if trace file exists
+ id: check-trace
+ run: |
+ if [ -f "trace-info/laminar_trace_info.json" ]; then
+ echo "trace_exists=true" >> $GITHUB_OUTPUT
+ echo "Found trace file for PR #$PR_NUMBER"
+ else
+ echo "trace_exists=false" >> $GITHUB_OUTPUT
+ echo "No trace file found for PR #$PR_NUMBER - skipping evaluation"
+ fi
+
+ - name: Checkout extensions repository
+ if: steps.check-trace.outputs.trace_exists == 'true'
+ uses: actions/checkout@v6
+ with:
+ repository: OpenHands/extensions
+ path: extensions
+
+ - name: Set up Python
+ if: steps.check-trace.outputs.trace_exists == 'true'
+ uses: actions/setup-python@v6
+ with:
+ python-version: "3.12"
+
+ - name: Install dependencies
+ if: steps.check-trace.outputs.trace_exists == 'true'
+ run: pip install lmnr
+
+ - name: Run evaluation
+ if: steps.check-trace.outputs.trace_exists == 'true'
+ env:
+ LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ python extensions/plugins/pr-review/scripts/evaluate_review.py \
+ --trace-file trace-info/laminar_trace_info.json
+
+ - name: Upload evaluation logs
+ uses: actions/upload-artifact@v7
+ if: always() && steps.check-trace.outputs.trace_exists == 'true'
+ with:
+ name: pr-review-evaluation-${{ github.event.pull_request.number }}
+ path: "*.log"
+ retention-days: 30
diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml
deleted file mode 100644
index 06928739a0..0000000000
--- a/.github/workflows/pr-review.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-name: PR Review by OpenHands
-
-on:
- # Use pull_request_target to allow fork PRs to access secrets when triggered by maintainers
- # Security: This workflow runs when:
- # 1. A new PR is opened (non-draft), OR
- # 2. A draft PR is marked as ready for review, OR
- # 3. A maintainer adds the 'review-this' label, OR
- # 4. A maintainer requests openhands-agent or all-hands-bot as a reviewer
- # Adding labels and requesting reviewers requires write access.
- # The PR code is explicitly checked out for review, but secrets are only accessible
- # because the workflow runs in the base repository context.
- pull_request_target:
- types: [opened, ready_for_review, labeled, review_requested]
-
-permissions:
- contents: read
- pull-requests: write
- issues: write
-
-jobs:
- pr-review:
- # Run when one of the following conditions is met:
- # 1. A new non-draft PR is opened by a non-first-time contributor, OR
- # 2. A draft PR is converted to ready for review by a non-first-time contributor, OR
- # 3. 'review-this' label is added, OR
- # 4. openhands-agent or all-hands-bot is requested as a reviewer
- # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request.
- if: |
- (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
- (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
- github.event.label.name == 'review-this' ||
- github.event.requested_reviewer.login == 'openhands-agent' ||
- github.event.requested_reviewer.login == 'all-hands-bot'
- concurrency:
- group: pr-review-${{ github.event.pull_request.number }}
- cancel-in-progress: true
- runs-on: ubuntu-24.04
- steps:
- - name: Run PR Review
- uses: OpenHands/extensions/plugins/pr-review@main
- with:
- llm-model: litellm_proxy/claude-sonnet-4-5-20250929
- llm-base-url: https://llm-proxy.app.all-hands.dev
- # [DEPRECATED] review-style is no longer used; standard and roasted are merged
- # review-style: roasted
- use-sub-agents: 'false' # Disabled by default (issue #208: high cost / timeouts)
- llm-api-key: ${{ secrets.LLM_API_KEY }}
- github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC }}
- lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }}
diff --git a/AGENTS.md b/AGENTS.md
index a93c397499..26b5c51983 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -69,7 +69,8 @@
- OpenHands repo bootstrap files live under `.openhands/`:
- `.openhands/setup.sh` installs frontend dependencies with `npm ci` when needed, creates `.env` from `.env.sample` if missing, appends `VITE_WORKING_DIR` for this repo when unset, and generates `src/i18n/declaration.ts` via `npm run make-i18n`.
- `.openhands/pre-commit.sh` mirrors the repo's local quality gate with `npm run lint && npm run test`.
-- `.github/workflows/pr-review.yml` mirrors the OpenHands/extensions PR review workflow: it auto-runs for newly opened non-draft PRs and `ready_for_review` events from established contributors, still supports the `review-this` label / `openhands-agent` / `all-hands-bot` reviewer triggers, uses the OpenHands app LLM proxy defaults, and expects `LLM_API_KEY`, `OPENHANDS_BOT_GITHUB_PAT_PUBLIC`, and optional `LMNR_SKILLS_API_KEY` repository secrets.
+- GitHub PR-review automation should stay aligned with the current OpenHands repo conventions: keep the review workflow at `.github/workflows/pr-review-by-openhands.yml`, keep the companion `.github/workflows/pr-review-evaluation.yml`, auto-run on newly opened non-draft PRs and `ready_for_review` events from established contributors, still support the `review-this` label / `openhands-agent` / `all-hands-bot` reviewer triggers, use the OpenHands app LLM proxy defaults, and use the dual-trigger pattern (`pull_request` for same-repo PRs, `pull_request_target` for forks) so workflow changes can self-verify without widening fork secret exposure.
+
- HeroUI v3 migration notes:
- The repo now uses `@heroui/[email protected]` plus `@heroui/[email protected]`.
- `src/tailwind.css` should import `@heroui/styles`; the old `hero.ts` Tailwind plugin file was removed and should not be reintroduced.