Skip to content

πŸ™Œ OpenHands: AI-Driven Development

ci: isolate verify-skill changes from implementation PRs (#18458)

🐾 smolpaws0774201613f4
diff --git a/.github/scripts/tests/verify-skill-isolation.cjs b/.github/scripts/tests/verify-skill-isolation.cjs
new file mode 100644
index 0000000000..b3fa7f50ce
--- /dev/null
+++ b/.github/scripts/tests/verify-skill-isolation.cjs
@@ -0,0 +1,106 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const { test } = require("node:test");
+
+// Execute the actual github-script body, without checking out or running PR code.
+const workflow = fs.readFileSync(
+ path.join(__dirname, "../../workflows/verify-skill-isolation.yml"), "utf8",
+);
+const match = workflow.match(/^([ \t]*)script: \|\n([\s\S]*)$/m);
+assert.ok(match, "workflow must contain an inline github-script body");
+const script = match[2].split("\n")
+ .map((line) => line.slice(match[1].length + 4)).join("\n");
+const run = new (Object.getPrototypeOf(async function () {}).constructor)(
+ "github", "context", "core", script,
+);
+const verify = ".agents/skills/verify-openhands/SKILL.md";
+const file = (filename, extra = {}) => ({ filename, status: "modified", ...extra });
+
+async function check(files, options = {}) {
+ const errors = [];
+ let reads = 0;
+ const pr = {
+ head: { sha: "head" }, base: { sha: "base" },
+ changed_files: options.count ?? files.length,
+ };
+ const get = async () => ({ data: ++reads === 2 && options.after ? options.after : pr });
+ const listFiles = Symbol("listFiles");
+ const github = {
+ rest: { pulls: { get, listFiles } },
+ paginate: async (endpoint, params) => {
+ assert.equal(endpoint, listFiles);
+ assert.equal(params.per_page, 100);
+ assert.equal(params.pull_number, 42);
+ if (options.apiError) throw new Error("GitHub unavailable");
+ return files;
+ },
+ };
+ await run(github, { repo: { owner: "org", repo: "repo" }, issue: { number: 42 } }, {
+ info: () => {}, setFailed: (message) => errors.push(message),
+ });
+ return errors;
+}
+
+for (const [name, files, allowed] of [
+ ["product-only", [file("src/app.py"), file("README.md")], true],
+ ["AGENTS-only", [file("AGENTS.md"), file("nested/AGENTS.md")], true],
+ ["skill-only additions and deletions", [file(verify, { status: "added" }),
+ file(".agents/skills/verify-sdk/map.md", { status: "removed" })], true],
+ ["skill plus AGENTS", [file(verify), file("nested/AGENTS.md")], true],
+ ["mixed product and skill", [file(verify), file("src/app.py")], false],
+ ["mixed other skill", [file(verify), file(".agents/skills/review/SKILL.md")], false],
+ ["skill plus .pr evidence", [file(verify), file(".pr/map-maintenance/report.md")], true],
+ ["nested .pr is not exempt", [file(verify), file("src/.pr/notes.md")], false],
+ ["product plus .pr evidence", [file("src/app.py"), file(".pr/18160/proof.json")], true],
+ ["README is not exempt", [file(verify), file("README.md")], false],
+ ["lowercase agents is not exempt", [file(verify), file("agents.md")], false],
+ ["verify prefix without dash is ordinary product", [file(".agents/skills/verify/SKILL.md"), file("src/app.py")], true],
+ ["AGENTS suffix is not exempt", [file(verify), file("nested/NOT_AGENTS.md")], false],
+ ["nested lookalike is ordinary product", [file("docs/.agents/skills/verify-x/a")], true],
+ ["rename out of skill", [file("docs/map.md", { status: "renamed", previous_filename: verify })], false],
+ ["rename into skill", [file(verify, { status: "renamed", previous_filename: "src/app.py" })], false],
+ ["rename inside skills", [file(verify, { status: "renamed", previous_filename:
+ ".agents/skills/verify-automation/SKILL.md" })], true],
+ ["AGENTS renamed to product with skill", [file(verify), file("src/app.py", {
+ status: "renamed", previous_filename: "AGENTS.md" })], false],
+ ["AGENTS rename inside skill", [file(verify, { status: "renamed", previous_filename: "AGENTS.md" })], true],
+ ["rename metadata missing", [file(verify, { status: "renamed" })], false],
+]) {
+ test(name, async () => assert.equal((await check(files)).length === 0, allowed));
+}
+
+test("more than one page still detects a mixed PR", async () => {
+ const files = Array.from({ length: 101 }, (_, i) => file(`src/${i}.py`));
+ files.push(file(verify));
+ assert.equal((await check(files)).length, 1);
+});
+test("incomplete file listing fails closed", async () => {
+ assert.match((await check([file(verify)], { count: 3001 }))[0], /complete/i);
+});
+test("API failures fail closed", async () => {
+ assert.match((await check([], { apiError: true }))[0], /unavailable/);
+});
+test("PR changes during pagination fail closed", async () => {
+ assert.match((await check([file(verify)], { after: {
+ head: { sha: "new-head" }, base: { sha: "base" }, changed_files: 1,
+ } }))[0], /changed/i);
+});
+
+test("base changes during pagination fail closed", async () => {
+ assert.match((await check([file(verify)], { after: {
+ head: { sha: "head" }, base: { sha: "new-base" }, changed_files: 1,
+ } }))[0], /changed/i);
+});
+test("file count changes during pagination fail closed", async () => {
+ assert.match((await check([file(verify)], { after: {
+ head: { sha: "head" }, base: { sha: "base" }, changed_files: 2,
+ } }))[0], /changed/i);
+});
+
+test("a mixed PR's failure says how to split it and lists both sides", async () => {
+ const [message] = await check([file(verify), file("src/app.py")]);
+ assert.match(message, /split this PR/);
+ assert.match(message, new RegExp(`- ${verify.replaceAll(".", "\\.")}`));
+ assert.match(message, /- src\/app\.py/);
+});
diff --git a/.github/workflows/verify-skill-isolation-tests.yml b/.github/workflows/verify-skill-isolation-tests.yml
new file mode 100644
index 0000000000..6129c8255b
--- /dev/null
+++ b/.github/workflows/verify-skill-isolation-tests.yml
@@ -0,0 +1,29 @@
+---
+name: Verify skill isolation tests
+
+on:
+ pull_request:
+ paths:
+ - .github/workflows/verify-skill-isolation.yml
+ - .github/workflows/verify-skill-isolation-tests.yml
+ - .github/scripts/tests/verify-skill-isolation.cjs
+ push:
+ branches: [main]
+ paths:
+ - .github/workflows/verify-skill-isolation.yml
+ - .github/workflows/verify-skill-isolation-tests.yml
+ - .github/scripts/tests/verify-skill-isolation.cjs
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-node@v7
+ with:
+ node-version: '24'
+ - run: node --test .github/scripts/tests/verify-skill-isolation.cjs
diff --git a/.github/workflows/verify-skill-isolation.yml b/.github/workflows/verify-skill-isolation.yml
new file mode 100644
index 0000000000..7ad369edaf
--- /dev/null
+++ b/.github/workflows/verify-skill-isolation.yml
@@ -0,0 +1,110 @@
+---
+name: Verify skill isolation
+
+# Why this check exists
+# --------------------
+# The verify-* skills under .agents/skills/ (feature maps, known-bug recipes,
+# the control-openhands CLI and its fixtures) describe how OpenHands is
+# expected to behave. Product code implements that behavior. The same PR, and
+# so the same author or agent context, must not change both: whoever writes a
+# fix must not also rewrite the rules the fix is checked against.
+#
+# Rule: a PR that changes anything under .agents/skills/verify-*/ may change
+# nothing else, except files named exactly AGENTS.md (at any depth) and the
+# temporary .pr/ evidence directory at the repository root.
+# A PR that changes no verify-* skill file always passes.
+#
+# How to fix a failing PR: split it. Keep the product change (source, tests,
+# CI, docs) in this PR and move the verify-* skill change to its own PR, or
+# the other way around. A fix PR is verified against the skill as it stands
+# on the base branch; update the skill in a follow-up PR after the fix lands.
+
+on:
+ pull_request_target:
+ types: [opened, synchronize, reopened, edited, ready_for_review]
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ verify-skill-isolation:
+ name: Verify skill isolation
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ # Use only trusted base-branch code and GitHub metadata. Never check out
+ # the PR or execute its code in this pull_request_target workflow.
+ # .github/scripts/tests/verify-skill-isolation.cjs runs this exact
+ # script body, so keep `script: |` the last key in this file.
+ - uses: actions/github-script@v9
+ with:
+ script: |
+ try {
+ // Read the PR metadata before and after listing its files, so a push
+ // during pagination cannot slip a stale or partial list past the check.
+ const params = { ...context.repo, pull_number: context.issue.number };
+ const { data: before } = await github.rest.pulls.get(params);
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ ...params, per_page: 100,
+ });
+ const { data: after } = await github.rest.pulls.get(params);
+ if (before.head.sha !== after.head.sha || before.base.sha !== after.base.sha ||
+ before.changed_files !== after.changed_files) {
+ throw new Error("PR changed while checking its files; rerun this check.");
+ }
+ // GitHub caps this endpoint at 3,000 files. Never accept a partial diff.
+ if (files.length !== after.changed_files) {
+ throw new Error("Could not read the complete PR file list; split the PR.");
+ }
+ // A rename touches both its old and new path, so check both.
+ const paths = [];
+ for (const file of files) {
+ if (!file.filename || (file.status === "renamed" && !file.previous_filename)) {
+ throw new Error("Incomplete file or rename metadata; rerun this check.");
+ }
+ paths.push(file.filename);
+ if (file.previous_filename) paths.push(file.previous_filename);
+ }
+ const isVerify = (path) => /^\.agents\/skills\/verify-[^/]+\//.test(path);
+ // AGENTS.md files and the temporary .pr/ evidence directory (cleaned up
+ // by pr-artifacts.yml) may ride along with skill changes.
+ const isExempt = (path) =>
+ path.split("/").pop() === "AGENTS.md" || path.startsWith(".pr/");
+ const skillPaths = [...new Set(paths.filter(isVerify))];
+ const otherPaths = [...new Set(paths.filter((path) =>
+ !isVerify(path) && !isExempt(path)))];
+ if (skillPaths.length && otherPaths.length) {
+ const list = (items) => items.map((item) => ` - ${item}`).join("\n");
+ core.setFailed([
+ "Verify skill isolation failed: this PR changes a verify-* skill AND other files.",
+ "",
+ "The verify-* skills (feature maps, known-bug recipes, control-openhands) define",
+ "expected behavior. The same PR must not change both those rules and the code",
+ "they check. Only AGENTS.md files and .pr/ evidence may accompany",
+ ".agents/skills/verify-*/ changes.",
+ "",
+ "What to do: split this PR in two.",
+ " 1. Keep the product change here and restore every verify-* skill path below",
+ " to the base branch (for example `git checkout origin/<base> -- <path>`).",
+ " 2. Open a separate PR with only the verify-* skill change. For a bug fix,",
+ " open it after the fix merges, so a different context updates the skill",
+ " than the one that wrote the fix.",
+ "Do not edit this workflow, move content into AGENTS.md or .pr/, or relocate",
+ "skill files to get around the check.",
+ "",
+ `verify-* skill paths (${skillPaths.length}):`,
+ list(skillPaths),
+ "",
+ `Other paths (${otherPaths.length}):`,
+ list(otherPaths),
+ ].join("\n"));
+ } else {
+ core.info(skillPaths.length
+ ? "Verify skill isolation passed: only verify-* skill files (plus AGENTS.md or .pr/) changed."
+ : "Verify skill isolation passed: no verify-* skill files changed.");
+ }
+ } catch (error) {
+ // Fail closed: when the file list cannot be trusted, do not pass.
+ core.setFailed(`Verify skill isolation could not run: ${error.message}`);
+ }
diff --git a/AGENTS.md b/AGENTS.md
index f12993fd67..119dce3b9b 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -62,9 +62,15 @@ Detailed contributor knowledge is split into skills so it loads only for relevan
| [`frontend-development`](.agents/skills/frontend-development/SKILL.md) | React/UI work, i18n, named identifiers, MSW mock mode, lazy loading, bundle performance, and feature-specific UI invariants. |
| [`pr-design-doc`](.agents/skills/pr-design-doc/SKILL.md) | Writing the PR design document stored in the PR body. |
| [`release`](.agents/skills/release.md) | Cutting and verifying an `@openhands/agent-canvas` release. |
-| [`verify-openhands`](.agents/skills/verify-openhands/SKILL.md) | Driving the real app like a user with `control-openhands`, the feature map of every user-facing behavior, and creating or maintaining that map. |
+| [`verify-openhands`](.agents/skills/verify-openhands/SKILL.md) | Driving the real app like a user with `control-openhands` and checking behavior against the existing feature map. |
-The detailed rules live in each skill's `references/guide.md`; do not copy them back into this file. Update the owning skill whenever an invariant changes.
+The detailed rules live in each skill's `references/guide.md`; do not copy them back into this file. Update the owning contributor skill when its invariant changes, except for the independent verification skills described below.
+
+### Verification Skill Independence
+
+Implementor agents must use `.agents/skills/verify-*/` as read-only verification tools. Do not update their feature maps, expected results, known-bug recipes, drivers, or fixtures while implementing a product change. This separation applies even when an issue's acceptance criteria ask for verification-skill updates. Report a mismatch or missing coverage instead of changing the specification to match the implementation.
+
+Verification-skill maintenance belongs in a separate PR and a separate agent context. For a product fix, make that follow-up after the fix lands. A verify-skill PR may include only verify-skill paths, `AGENTS.md` files, and root `.pr/` evidence. Product source, tests, CI, and other docs must stay separate.
## Testing Baseline