π OpenHands: AI-Driven Development
ci: isolate verify-skill changes from implementation PRs (#18458)
πΎ smolpaws0774201613f4
diff --git a/.github/scripts/tests/verify-skill-isolation.cjs b/.github/scripts/tests/verify-skill-isolation.cjsnew file mode 100644index 0000000000..b3fa7f50ce--- /dev/null+++ b/.github/scripts/tests/verify-skill-isolation.cjs@@ -0,0 +1,106 @@+const assert = require("node:assert/strict");+const fs = require("node:fs");+const path = require("node:path");+const { test } = require("node:test");++// Execute the actual github-script body, without checking out or running PR code.+const workflow = fs.readFileSync(+ path.join(__dirname, "../../workflows/verify-skill-isolation.yml"), "utf8",+);+const match = workflow.match(/^([ \t]*)script: \|\n([\s\S]*)$/m);+assert.ok(match, "workflow must contain an inline github-script body");+const script = match[2].split("\n")+ .map((line) => line.slice(match[1].length + 4)).join("\n");+const run = new (Object.getPrototypeOf(async function () {}).constructor)(+ "github", "context", "core", script,+);+const verify = ".agents/skills/verify-openhands/SKILL.md";+const file = (filename, extra = {}) => ({ filename, status: "modified", ...extra });++async function check(files, options = {}) {+ const errors = [];+ let reads = 0;+ const pr = {+ head: { sha: "head" }, base: { sha: "base" },+ changed_files: options.count ?? files.length,+ };+ const get = async () => ({ data: ++reads === 2 && options.after ? options.after : pr });+ const listFiles = Symbol("listFiles");+ const github = {+ rest: { pulls: { get, listFiles } },+ paginate: async (endpoint, params) => {+ assert.equal(endpoint, listFiles);+ assert.equal(params.per_page, 100);+ assert.equal(params.pull_number, 42);+ if (options.apiError) throw new Error("GitHub unavailable");+ return files;+ },+ };+ await run(github, { repo: { owner: "org", repo: "repo" }, issue: { number: 42 } }, {+ info: () => {}, setFailed: (message) => errors.push(message),+ });+ return errors;+}++for (const [name, files, allowed] of [+ ["product-only", [file("src/app.py"), file("README.md")], true],+ ["AGENTS-only", [file("AGENTS.md"), file("nested/AGENTS.md")], true],+ ["skill-only additions and deletions", [file(verify, { status: "added" }),+ file(".agents/skills/verify-sdk/map.md", { status: "removed" })], true],+ ["skill plus AGENTS", [file(verify), file("nested/AGENTS.md")], true],+ ["mixed product and skill", [file(verify), file("src/app.py")], false],+ ["mixed other skill", [file(verify), file(".agents/skills/review/SKILL.md")], false],+ ["skill plus .pr evidence", [file(verify), file(".pr/map-maintenance/report.md")], true],+ ["nested .pr is not exempt", [file(verify), file("src/.pr/notes.md")], false],+ ["product plus .pr evidence", [file("src/app.py"), file(".pr/18160/proof.json")], true],+ ["README is not exempt", [file(verify), file("README.md")], false],+ ["lowercase agents is not exempt", [file(verify), file("agents.md")], false],+ ["verify prefix without dash is ordinary product", [file(".agents/skills/verify/SKILL.md"), file("src/app.py")], true],+ ["AGENTS suffix is not exempt", [file(verify), file("nested/NOT_AGENTS.md")], false],+ ["nested lookalike is ordinary product", [file("docs/.agents/skills/verify-x/a")], true],+ ["rename out of skill", [file("docs/map.md", { status: "renamed", previous_filename: verify })], false],+ ["rename into skill", [file(verify, { status: "renamed", previous_filename: "src/app.py" })], false],+ ["rename inside skills", [file(verify, { status: "renamed", previous_filename:+ ".agents/skills/verify-automation/SKILL.md" })], true],+ ["AGENTS renamed to product with skill", [file(verify), file("src/app.py", {+ status: "renamed", previous_filename: "AGENTS.md" })], false],+ ["AGENTS rename inside skill", [file(verify, { status: "renamed", previous_filename: "AGENTS.md" })], true],+ ["rename metadata missing", [file(verify, { status: "renamed" })], false],+]) {+ test(name, async () => assert.equal((await check(files)).length === 0, allowed));+}++test("more than one page still detects a mixed PR", async () => {+ const files = Array.from({ length: 101 }, (_, i) => file(`src/${i}.py`));+ files.push(file(verify));+ assert.equal((await check(files)).length, 1);+});+test("incomplete file listing fails closed", async () => {+ assert.match((await check([file(verify)], { count: 3001 }))[0], /complete/i);+});+test("API failures fail closed", async () => {+ assert.match((await check([], { apiError: true }))[0], /unavailable/);+});+test("PR changes during pagination fail closed", async () => {+ assert.match((await check([file(verify)], { after: {+ head: { sha: "new-head" }, base: { sha: "base" }, changed_files: 1,+ } }))[0], /changed/i);+});++test("base changes during pagination fail closed", async () => {+ assert.match((await check([file(verify)], { after: {+ head: { sha: "head" }, base: { sha: "new-base" }, changed_files: 1,+ } }))[0], /changed/i);+});+test("file count changes during pagination fail closed", async () => {+ assert.match((await check([file(verify)], { after: {+ head: { sha: "head" }, base: { sha: "base" }, changed_files: 2,+ } }))[0], /changed/i);+});++test("a mixed PR's failure says how to split it and lists both sides", async () => {+ const [message] = await check([file(verify), file("src/app.py")]);+ assert.match(message, /split this PR/);+ assert.match(message, new RegExp(`- ${verify.replaceAll(".", "\\.")}`));+ assert.match(message, /- src\/app\.py/);+});diff --git a/.github/workflows/verify-skill-isolation-tests.yml b/.github/workflows/verify-skill-isolation-tests.ymlnew file mode 100644index 0000000000..6129c8255b--- /dev/null+++ b/.github/workflows/verify-skill-isolation-tests.yml@@ -0,0 +1,29 @@+---+name: Verify skill isolation tests++on:+ pull_request:+ paths:+ - .github/workflows/verify-skill-isolation.yml+ - .github/workflows/verify-skill-isolation-tests.yml+ - .github/scripts/tests/verify-skill-isolation.cjs+ push:+ branches: [main]+ paths:+ - .github/workflows/verify-skill-isolation.yml+ - .github/workflows/verify-skill-isolation-tests.yml+ - .github/scripts/tests/verify-skill-isolation.cjs++permissions:+ contents: read++jobs:+ test:+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@v7+ - uses: actions/setup-node@v7+ with:+ node-version: '24'+ - run: node --test .github/scripts/tests/verify-skill-isolation.cjsdiff --git a/.github/workflows/verify-skill-isolation.yml b/.github/workflows/verify-skill-isolation.ymlnew file mode 100644index 0000000000..7ad369edaf--- /dev/null+++ b/.github/workflows/verify-skill-isolation.yml@@ -0,0 +1,110 @@+---+name: Verify skill isolation++# Why this check exists+# --------------------+# The verify-* skills under .agents/skills/ (feature maps, known-bug recipes,+# the control-openhands CLI and its fixtures) describe how OpenHands is+# expected to behave. Product code implements that behavior. The same PR, and+# so the same author or agent context, must not change both: whoever writes a+# fix must not also rewrite the rules the fix is checked against.+#+# Rule: a PR that changes anything under .agents/skills/verify-*/ may change+# nothing else, except files named exactly AGENTS.md (at any depth) and the+# temporary .pr/ evidence directory at the repository root.+# A PR that changes no verify-* skill file always passes.+#+# How to fix a failing PR: split it. Keep the product change (source, tests,+# CI, docs) in this PR and move the verify-* skill change to its own PR, or+# the other way around. A fix PR is verified against the skill as it stands+# on the base branch; update the skill in a follow-up PR after the fix lands.++on:+ pull_request_target:+ types: [opened, synchronize, reopened, edited, ready_for_review]++permissions:+ contents: read+ pull-requests: read++jobs:+ verify-skill-isolation:+ name: Verify skill isolation+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ # Use only trusted base-branch code and GitHub metadata. Never check out+ # the PR or execute its code in this pull_request_target workflow.+ # .github/scripts/tests/verify-skill-isolation.cjs runs this exact+ # script body, so keep `script: |` the last key in this file.+ - uses: actions/github-script@v9+ with:+ script: |+ try {+ // Read the PR metadata before and after listing its files, so a push+ // during pagination cannot slip a stale or partial list past the check.+ const params = { ...context.repo, pull_number: context.issue.number };+ const { data: before } = await github.rest.pulls.get(params);+ const files = await github.paginate(github.rest.pulls.listFiles, {+ ...params, per_page: 100,+ });+ const { data: after } = await github.rest.pulls.get(params);+ if (before.head.sha !== after.head.sha || before.base.sha !== after.base.sha ||+ before.changed_files !== after.changed_files) {+ throw new Error("PR changed while checking its files; rerun this check.");+ }+ // GitHub caps this endpoint at 3,000 files. Never accept a partial diff.+ if (files.length !== after.changed_files) {+ throw new Error("Could not read the complete PR file list; split the PR.");+ }+ // A rename touches both its old and new path, so check both.+ const paths = [];+ for (const file of files) {+ if (!file.filename || (file.status === "renamed" && !file.previous_filename)) {+ throw new Error("Incomplete file or rename metadata; rerun this check.");+ }+ paths.push(file.filename);+ if (file.previous_filename) paths.push(file.previous_filename);+ }+ const isVerify = (path) => /^\.agents\/skills\/verify-[^/]+\//.test(path);+ // AGENTS.md files and the temporary .pr/ evidence directory (cleaned up+ // by pr-artifacts.yml) may ride along with skill changes.+ const isExempt = (path) =>+ path.split("/").pop() === "AGENTS.md" || path.startsWith(".pr/");+ const skillPaths = [...new Set(paths.filter(isVerify))];+ const otherPaths = [...new Set(paths.filter((path) =>+ !isVerify(path) && !isExempt(path)))];+ if (skillPaths.length && otherPaths.length) {+ const list = (items) => items.map((item) => ` - ${item}`).join("\n");+ core.setFailed([+ "Verify skill isolation failed: this PR changes a verify-* skill AND other files.",+ "",+ "The verify-* skills (feature maps, known-bug recipes, control-openhands) define",+ "expected behavior. The same PR must not change both those rules and the code",+ "they check. Only AGENTS.md files and .pr/ evidence may accompany",+ ".agents/skills/verify-*/ changes.",+ "",+ "What to do: split this PR in two.",+ " 1. Keep the product change here and restore every verify-* skill path below",+ " to the base branch (for example `git checkout origin/<base> -- <path>`).",+ " 2. Open a separate PR with only the verify-* skill change. For a bug fix,",+ " open it after the fix merges, so a different context updates the skill",+ " than the one that wrote the fix.",+ "Do not edit this workflow, move content into AGENTS.md or .pr/, or relocate",+ "skill files to get around the check.",+ "",+ `verify-* skill paths (${skillPaths.length}):`,+ list(skillPaths),+ "",+ `Other paths (${otherPaths.length}):`,+ list(otherPaths),+ ].join("\n"));+ } else {+ core.info(skillPaths.length+ ? "Verify skill isolation passed: only verify-* skill files (plus AGENTS.md or .pr/) changed."+ : "Verify skill isolation passed: no verify-* skill files changed.");+ }+ } catch (error) {+ // Fail closed: when the file list cannot be trusted, do not pass.+ core.setFailed(`Verify skill isolation could not run: ${error.message}`);+ }diff --git a/AGENTS.md b/AGENTS.mdindex f12993fd67..119dce3b9b 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -62,9 +62,15 @@ Detailed contributor knowledge is split into skills so it loads only for relevan| [`frontend-development`](.agents/skills/frontend-development/SKILL.md) | React/UI work, i18n, named identifiers, MSW mock mode, lazy loading, bundle performance, and feature-specific UI invariants. || [`pr-design-doc`](.agents/skills/pr-design-doc/SKILL.md) | Writing the PR design document stored in the PR body. || [`release`](.agents/skills/release.md) | Cutting and verifying an `@openhands/agent-canvas` release. |-| [`verify-openhands`](.agents/skills/verify-openhands/SKILL.md) | Driving the real app like a user with `control-openhands`, the feature map of every user-facing behavior, and creating or maintaining that map. |+| [`verify-openhands`](.agents/skills/verify-openhands/SKILL.md) | Driving the real app like a user with `control-openhands` and checking behavior against the existing feature map. |-The detailed rules live in each skill's `references/guide.md`; do not copy them back into this file. Update the owning skill whenever an invariant changes.+The detailed rules live in each skill's `references/guide.md`; do not copy them back into this file. Update the owning contributor skill when its invariant changes, except for the independent verification skills described below.++### Verification Skill Independence++Implementor agents must use `.agents/skills/verify-*/` as read-only verification tools. Do not update their feature maps, expected results, known-bug recipes, drivers, or fixtures while implementing a product change. This separation applies even when an issue's acceptance criteria ask for verification-skill updates. Report a mismatch or missing coverage instead of changing the specification to match the implementation.++Verification-skill maintenance belongs in a separate PR and a separate agent context. For a product fix, make that follow-up after the fix lands. A verify-skill PR may include only verify-skill paths, `AGENTS.md` files, and root `.pr/` evidence. Product source, tests, CI, and other docs must stay separate.## Testing Baseline