Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(ci): let triage and repository writers own readiness (#17598)

Graham Neubigb5d6b2ff2c90
diff --git a/.github/scripts/check_issue_readiness.py b/.github/scripts/check_issue_readiness.py
deleted file mode 100644
index 1718e6c177..0000000000
--- a/.github/scripts/check_issue_readiness.py
+++ /dev/null
@@ -1,356 +0,0 @@
-"""Determine whether an issue meets the `ready-for-dev` readiness criteria.
-
-The criteria are type-specific, and the type is inferred from the body
-structure rather than the `bug`/`enhancement` label:
-
-- Bug reports (a `##`/`### Steps to Reproduce` or `##`/`### Actual Behavior`
- section):
- the Steps to Reproduce section must reference at least one supported run
- method (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`), the
- Actual Behavior section must embed a screenshot or video, and there must be
- a non-empty Acceptance Criteria section with at least one checklist item.
-
-- Enhancements (a `##`/`### Desired Behavior` section): the body must contain
- non-empty Desired Behavior and Acceptance Criteria sections, the latter
- with at least one checklist item.
-
-GitHub issue forms render each field as an `### <Label>` (h3) heading followed
-by the field text, with empty optional fields rendered as `_No response_`. The
-parser also accepts `## <Label>` headings used by hand-edited and free-form
-issues, and checks each criterion against the corresponding section.
-
-Local usage:
-
- python .github/scripts/check_issue_readiness.py --body-file /tmp/issue.md
- python .github/scripts/check_issue_readiness.py --event-path "$GITHUB_EVENT_PATH"
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import re
-import sys
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from markdown_sections import find_headings
-
-BUG_LABEL = "bug"
-ENHANCEMENT_LABEL = "enhancement"
-
-# Issue-form field headings that identify an issue's type. The `bug` and
-# `enhancement` labels are deliberately not consulted: the type is inferred from
-# which of these sections is present, so issues can qualify for `ready-for-dev`
-# without carrying a label.
-BUG_SECTION_LABELS = (
- "actual behavior",
- "actual",
- "steps to reproduce",
- "reproduction",
-)
-ENHANCEMENT_SECTION_LABELS = ("desired behavior", "desired")
-
-# Issue-form fields render as h3 headings, while hand-edited and free-form
-# issues commonly use h2. Capture the level so nested h3 headings can remain
-# part of an h2 section.
-HEADING_RE = re.compile(r"(?m)^(?P<level>#{2,3})\s+(?P<title>.+?)\s*$")
-
-READINESS_SECTION_LABELS = {
- *BUG_SECTION_LABELS,
- *ENHANCEMENT_SECTION_LABELS,
- "acceptance criteria",
- "acceptance",
-}
-
-# `_No response_` is what GitHub writes for an empty optional form field.
-NO_RESPONSE = "_No response_"
-
-# A supported way to run Agent Canvas. The Reproduction section must mention at
-# least one. `agent-canvas` covers `npx @openhands/agent-canvas`,
-# `agent-canvas --version`, the published binary, etc. `npm run` covers
-# `npm run dev` / `npm run dev:minimal`. The hosted canvas URL is matched
-# literally.
-RUN_METHOD_PATTERNS: tuple[re.Pattern[str], ...] = (
- re.compile(r"agent-canvas", re.IGNORECASE),
- re.compile(r"npm\s+run", re.IGNORECASE),
- re.compile(r"app\.all-hands\.dev/canvas", re.IGNORECASE),
-)
-
-# Markdown image: ![alt](url)
-MARKDOWN_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]+\)")
-# HTML <img ...> and <video ...> tags.
-HTML_IMG_RE = re.compile(r"<img\b[^>]*\bsrc\s*=", re.IGNORECASE)
-HTML_VIDEO_RE = re.compile(r"<video\b", re.IGNORECASE)
-# GitHub-uploaded assets (images and videos both use this URL shape).
-GITHUB_ATTACHMENT_RE = re.compile(
- r"https?://(?:www\.)?github\.com/user-attachments/assets/"
-)
-# Direct links to video files.
-VIDEO_FILE_RE = re.compile(
- r"https?://\S+\.(?:mp4|webm|mov|avi|mkv|ogv|3gp|m4v)(?:\S*)",
- re.IGNORECASE,
-)
-# Known video-hosting services.
-VIDEO_HOST_RE = re.compile(
- r"https?://(?:[a-z0-9-]+\.)?(?:youtube\.com|youtu\.be|loom\.com|vimeo\.com|asciinema\.org|streamable\.com)/",
- re.IGNORECASE,
-)
-
-# An Acceptance Criteria item is a markdown checklist bullet (`- [ ]` or
-# `- [x]`). We require at least one so the section is verifiable.
-CHECKLIST_ITEM_RE = re.compile(r"(?m)^\s*[-*]\s*\[[ xX]\]")
-
-
-@dataclass
-class ReadinessResult:
- """Outcome of a readiness check."""
-
- ready: bool
- reasons: list[str] = field(default_factory=list)
- issue_type: str | None = None
-
- def add(self, reason: str) -> None:
- self.reasons.append(reason)
- self.ready = False
-
-
-def visible_text(text: str) -> str:
- """Return field text with HTML comments stripped and emptiness normalized."""
- cleaned = re.sub(r"<!--[\s\S]*?-->", "", text).strip()
- if cleaned == NO_RESPONSE:
- return ""
- return cleaned
-
-
-def extract_sections(body: str) -> dict[str, str]:
- """Split the body into a {heading: text} map using h2 or h3 boundaries.
-
- Issue forms render fields as h3 headings. Free-form and hand-edited issues
- may use h2 headings instead, including a mix of both levels.
- """
- matches = find_headings(body, HEADING_RE)
- has_h2 = any(match.group("level") == "##" for match in matches)
- boundaries = [
- match
- for match in matches
- if match.group("level") == "##"
- or not has_h2
- or match.group("title").strip().lower() in READINESS_SECTION_LABELS
- ]
-
- sections: dict[str, str] = {}
- for index, match in enumerate(boundaries):
- start = match.end()
- end = (
- boundaries[index + 1].start()
- if index + 1 < len(boundaries)
- else len(body)
- )
- sections[match.group("title").strip().lower()] = body[start:end]
- return sections
-
-
-def find_section(sections: dict[str, str], *labels: str) -> str:
- """Return the first matching section text by case-insensitive label."""
- for label in labels:
- if label in sections:
- return sections[label]
- return ""
-
-
-def has_screenshot_or_video(text: str) -> bool:
- if MARKDOWN_IMAGE_RE.search(text):
- return True
- if HTML_IMG_RE.search(text):
- return True
- if HTML_VIDEO_RE.search(text):
- return True
- if GITHUB_ATTACHMENT_RE.search(text):
- return True
- if VIDEO_FILE_RE.search(text):
- return True
- if VIDEO_HOST_RE.search(text):
- return True
- return False
-
-
-def references_run_method(text: str) -> bool:
- return any(pattern.search(text) for pattern in RUN_METHOD_PATTERNS)
-
-
-def has_checklist_item(text: str) -> bool:
- return bool(CHECKLIST_ITEM_RE.search(text))
-
-
-def check_bug(sections: dict[str, str]) -> ReadinessResult:
- result = ReadinessResult(ready=True, issue_type=BUG_LABEL)
-
- reproduction = visible_text(find_section(sections, "steps to reproduce", "reproduction"))
- if not reproduction:
- result.add(
- "Fill in the `### Steps to Reproduce` section showing how you reproduced "
- "the bug in a live Agent Canvas session."
- )
- elif not references_run_method(reproduction):
- result.add(
- "The Steps to Reproduce section must reference a supported run method: "
- "`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`."
- )
-
- actual = visible_text(find_section(sections, "actual behavior", "actual"))
- if not actual:
- result.add("Fill in the `### Actual Behavior` section describing the observed bug.")
- elif not has_screenshot_or_video(actual):
- result.add(
- "The Actual Behavior section must include a screenshot or video of "
- "the bug (drag a file into the field or paste a link)."
- )
-
- acceptance = visible_text(find_section(sections, "acceptance criteria", "acceptance"))
- if not acceptance:
- result.add("Add an `### Acceptance Criteria` section with testable checklist items.")
- elif not has_checklist_item(acceptance):
- result.add(
- "The Acceptance Criteria section must contain at least one checklist item "
- "(`- [ ] …`)."
- )
-
- return result
-
-
-def check_enhancement(sections: dict[str, str]) -> ReadinessResult:
- result = ReadinessResult(ready=True, issue_type=ENHANCEMENT_LABEL)
-
- desired = visible_text(find_section(sections, "desired behavior", "desired"))
- if not desired:
- result.add(
- "Add a `### Desired Behavior` section describing the behavior you want."
- )
-
- acceptance = visible_text(find_section(sections, "acceptance criteria", "acceptance"))
- if not acceptance:
- result.add("Add an `### Acceptance Criteria` section with testable checklist items.")
- elif not has_checklist_item(acceptance):
- result.add(
- "The Acceptance Criteria section must contain at least one checklist item "
- "(`- [ ] …`)."
- )
-
- return result
-
-
-def evaluate_readiness(body: str, labels: list[str]) -> ReadinessResult:
- """Return the readiness result for an issue body.
-
- The type is inferred from the body structure, not the `bug`/`enhancement`
- label: a body with a `##`/`### Steps to Reproduce` or `##`/`### Actual
- Behavior` section is treated as a bug report, and a body with a
- `##`/`### Desired Behavior` section is treated as an enhancement. `labels`
- is accepted for backwards compatibility but is not consulted.
- """
- sections = extract_sections(body or "")
- has_bug_sections = any(label in sections for label in BUG_SECTION_LABELS)
- has_enhancement_sections = any(
- label in sections for label in ENHANCEMENT_SECTION_LABELS
- )
-
- if has_bug_sections and has_enhancement_sections:
- return ReadinessResult(
- ready=False,
- reasons=[
- "The issue contains both bug-report and feature-request sections, "
- "so its type cannot be inferred. Use only one issue template."
- ],
- )
- if has_bug_sections:
- return check_bug(sections)
- if has_enhancement_sections:
- return check_enhancement(sections)
-
- return ReadinessResult(
- ready=False,
- reasons=[
- "The issue has neither a `### Steps to Reproduce`/`### Actual "
- "Behavior` nor a `### Desired Behavior` section, so its readiness "
- "criteria cannot be evaluated. Use the bug report or feature "
- "request template."
- ],
- )
-
-
-def body_and_labels_from_event(event_path: Path) -> tuple[str, list[str]]:
- payload = json.loads(event_path.read_text())
- issue = payload.get("issue") or payload.get("pull_request")
- if not isinstance(issue, dict):
- raise ValueError("GitHub event payload does not contain an issue object")
- body = issue.get("body")
- body = body if isinstance(body, str) else ""
- labels = [label["name"] for label in issue.get("labels", []) if isinstance(label, dict)]
- return body, labels
-
-
-def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(
- description="Evaluate whether an issue meets the ready-for-dev criteria."
- )
- parser.add_argument("--body-file", type=Path, help="Read the issue body from a file.")
- parser.add_argument(
- "--labels",
- help="Comma-separated issue labels. Accepted for backwards compatibility "
- "but no longer consulted (the type is inferred from the body).",
- default="",
- )
- parser.add_argument(
- "--event-path",
- type=Path,
- default=Path(os.environ["GITHUB_EVENT_PATH"])
- if "GITHUB_EVENT_PATH" in os.environ
- else None,
- help="Read body and labels from a GitHub event payload.",
- )
- parser.add_argument(
- "--json",
- action="store_true",
- help="Emit a JSON result instead of human-readable text.",
- )
- return parser.parse_args()
-
-
-def main() -> int:
- args = parse_args()
-
- if args.body_file is not None:
- body = args.body_file.read_text()
- labels = [label.strip() for label in args.labels.split(",") if label.strip()]
- elif args.event_path is not None:
- body, labels = body_and_labels_from_event(args.event_path)
- else:
- raise SystemExit("Pass --body-file or set GITHUB_EVENT_PATH.")
-
- result = evaluate_readiness(body, labels)
-
- if args.json:
- print(
- json.dumps(
- {
- "ready": result.ready,
- "reasons": result.reasons,
- "issue_type": result.issue_type,
- }
- )
- )
- else:
- if result.ready:
- print("Issue meets ready-for-dev criteria.")
- else:
- print("Issue does not meet ready-for-dev criteria:")
- for reason in result.reasons:
- print(f" - {reason}")
-
- return 0 if result.ready else 1
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/.github/scripts/post-readiness-comment.mjs b/.github/scripts/post-readiness-comment.mjs
deleted file mode 100644
index 2666ed09b1..0000000000
--- a/.github/scripts/post-readiness-comment.mjs
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Post (or update) a `ready-for-dev` readiness comment on an issue.
- *
- * Upserts by a hidden HTML marker so repeated runs update the same comment
- * instead of spamming duplicates. Uses the GITHUB_TOKEN available to the
- * workflow via the GH_TOKEN env var.
- *
- * Usage:
- * node post-readiness-comment.mjs \
- * --issue-number 123 \
- * --repo OpenHands/OpenHands \
- * --reasons-file /tmp/reasons.txt \
- * --ready # optional: post a "ready" message instead of "not ready"
- */
-
-import { readFileSync } from "node:fs";
-
-const MARKER = "<!-- issue-readiness-check -->";
-const API_ROOT = process.env.GITHUB_API_URL ?? "https://api.github.com";
-
-function parseArgs(argv) {
- const args = {};
- for (let i = 0; i < argv.length; i += 1) {
- const arg = argv[i];
- if (!arg.startsWith("--")) continue;
- const [rawKey, inlineValue] = arg.slice(2).split("=", 2);
- const key = rawKey.replaceAll("-", "_");
- if (inlineValue !== undefined) {
- args[key] = inlineValue;
- continue;
- }
- const next = argv[i + 1];
- if (next && !next.startsWith("--")) {
- args[key] = next;
- i += 1;
- } else {
- args[key] = "";
- }
- }
- return args;
-}
-
-function requireValue(name, value) {
- if (!value) throw new Error(`Missing required value: ${name}`);
- return value;
-}
-
-async function listComments(repo, issueNumber, token) {
- // Paginate through all comments so the marker is found even on issues
- // with >100 comments.
- const all = [];
- let page = 1;
- while (true) {
- const url = `${API_ROOT}/repos/${repo}/issues/${issueNumber}/comments?per_page=100&page=${page}`;
- const resp = await fetch(url, {
- headers: {
- Authorization: `Bearer ${token}`,
- Accept: "application/vnd.github+json",
- "X-GitHub-Api-Version": "2022-11-28",
- },
- });
- if (!resp.ok) {
- throw new Error(`Failed to list comments: ${resp.status} ${await resp.text()}`);
- }
- const batch = await resp.json();
- all.push(...batch);
- if (batch.length < 100) break;
- page += 1;
- }
- return all;
-}
-
-async function createComment(repo, issueNumber, body, token) {
- const url = `${API_ROOT}/repos/${repo}/issues/${issueNumber}/comments`;
- const resp = await fetch(url, {
- method: "POST",
- headers: {
- Authorization: `Bearer ${token}`,
- Accept: "application/vnd.github+json",
- "X-GitHub-Api-Version": "2022-11-28",
- "Content-Type": "application/json",
- },
- body: JSON.stringify({ body }),
- });
- if (!resp.ok) {
- throw new Error(`Failed to create comment: ${resp.status} ${await resp.text()}`);
- }
- return resp.json();
-}
-
-async function updateComment(repo, commentId, body, token) {
- const url = `${API_ROOT}/repos/${repo}/issues/comments/${commentId}`;
- const resp = await fetch(url, {
- method: "PATCH",
- headers: {
- Authorization: `Bearer ${token}`,
- Accept: "application/vnd.github+json",
- "X-GitHub-Api-Version": "2022-11-28",
- "Content-Type": "application/json",
- },
- body: JSON.stringify({ body }),
- });
- if (!resp.ok) {
- throw new Error(`Failed to update comment: ${resp.status} ${await resp.text()}`);
- }
- return resp.json();
-}
-
-async function main() {
- const args = parseArgs(process.argv.slice(2));
- const issueNumber = requireValue("issue-number", args.issue_number);
- const repo = requireValue("repo", args.repo);
- const token = process.env.GH_TOKEN;
- if (!token) throw new Error("GH_TOKEN env var is required");
-
- const isReady = args.ready === "" || args.ready === "true";
- const reasonsFile = args.reasons_file || args.reasons;
- const reasons = reasonsFile
- ? readFileSync(reasonsFile, "utf8")
- .split("\n")
- .map((line) => line.trim())
- .filter(Boolean)
- : [];
-
- const reasonList = reasons.length
- ? reasons.map((r) => `- ${r}`).join("\n")
- : "- The issue does not meet the ready-for-dev criteria.";
-
- const body = isReady
- ? [
- MARKER,
- "### βœ… `ready-for-dev` label applied",
- "",
- "This issue meets the readiness criteria and is ready for contribution.",
- "A contributor can now open a PR that links this issue (`Fixes #<number>`).",
- "",
- "_This is an automated check β€” no AI was used to generate this comment._",
- ].join("\n")
- : [
- MARKER,
- "### ⚠️ Not yet `ready-for-dev`",
- "",
- "This issue does not yet meet the readiness criteria, so the",
- "`ready-for-dev` label has not been applied (or has been removed).",
- "The criteria are type-specific, and the type is inferred from the body:",
- "",
- "**Bug reports** (a `##`/`### Steps to Reproduce` or `##`/`### Actual Behavior` section):",
- "- The `### Steps to Reproduce` section must reference a supported run method",
- " (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`).",
- "- The `### Actual Behavior` section must include a screenshot or video of the bug.",
- "- An `### Acceptance Criteria` section with at least one checklist item (`- [ ] …`).",
- "",
- "**Enhancements** (a `##`/`### Desired Behavior` section):",
- "- A `### Desired Behavior` section.",
- "- An `### Acceptance Criteria` section with at least one checklist item.",
- "",
- "What is missing:",
- "",
- reasonList,
- "",
- "Edit the issue to address the gaps and the `ready-for-dev` label will be",
- "applied automatically.",
- "",
- "_This is an automated check β€” no AI was used to generate this comment._",
- ].join("\n");
-
- const comments = await listComments(repo, issueNumber, token);
- const existing = comments.find((comment) => comment.body?.includes(MARKER));
-
- if (existing) {
- await updateComment(repo, existing.id, body, token);
- console.log(`Updated readiness comment ${existing.id} on issue #${issueNumber}`);
- } else {
- const created = await createComment(repo, issueNumber, body, token);
- console.log(`Created readiness comment ${created.id} on issue #${issueNumber}`);
- }
-}
-
-main().catch((error) => {
- console.error(error);
- process.exit(1);
-});
diff --git a/.github/scripts/tests/test_issue_readiness.py b/.github/scripts/tests/test_issue_readiness.py
deleted file mode 100644
index 5ee6bae953..0000000000
--- a/.github/scripts/tests/test_issue_readiness.py
+++ /dev/null
@@ -1,518 +0,0 @@
-"""Tests for check_issue_readiness.py β€” the ready-for-dev gate logic."""
-
-import sys
-from pathlib import Path
-
-# Make the sibling script importable.
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-
-from check_issue_readiness import (
- evaluate_readiness,
- extract_sections,
- has_screenshot_or_video,
- references_run_method,
- has_checklist_item,
- visible_text,
- main,
- BUG_LABEL,
- ENHANCEMENT_LABEL,
-)
-
-# ---------------------------------------------------------------------------
-# Helper builders
-# ---------------------------------------------------------------------------
-
-BUG_BODY_READY = """### Steps to Reproduce
-Run `npm run dev` and click the button.
-
-### Actual Behavior
-The button was misaligned.
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-
-### Acceptance Criteria
-- [ ] Button is centered
-"""
-
-BUG_BODY_NO_RUN_METHOD = """### Steps to Reproduce
-Click the button.
-
-### Actual Behavior
-The button was misaligned.
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-
-"""
-
-BUG_BODY_NO_SCREENSHOT = """### Steps to Reproduce
-I ran `npm run dev` and clicked the button.
-
-### Actual Behavior
-The button was misaligned.
-
-"""
-
-BUG_BODY_NO_ACCEPTANCE = """### Steps to Reproduce
-I ran `npm run dev` and clicked the button.
-
-### Actual Behavior
-The button was misaligned.
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-"""
-
-BUG_BODY_EMPTY_ACTUAL = """### Steps to Reproduce
-I ran `npm run dev` and clicked the button.
-
-### Actual Behavior
-_No response_
-
-### Acceptance Criteria
-- [ ] Button is centered
-"""
-
-BUG_BODY_AGENT_CANVAS = """### Steps to Reproduce
-I used agent-canvas to reproduce this.
-
-### Actual Behavior
-The button was misaligned.
-
-![screenshot](https://example.com/screenshot.png)
-
-### Acceptance Criteria
-- [ ] Fixed
-"""
-
-BUG_BODY_HOSTED_URL = """### Steps to Reproduce
-Reproduced on app.all-hands.dev/canvas.
-
-### Actual Behavior
-The button was misaligned.
-
-<video src="https://example.com/bug.mp4"></video>
-
-### Acceptance Criteria
-- [ ] Fixed
-"""
-
-BUG_BODY_MISSING_REPRODUCTION = """### Actual Behavior
-The button was misaligned.
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-
-### Acceptance Criteria
-- [ ] Button is centered
-"""
-
-ENHANCEMENT_BODY_READY = """### Desired Behavior
-The button should animate on hover.
-
-### Acceptance Criteria
-- [ ] Hover animation works
-- [ ] No perf regression
-"""
-
-ENHANCEMENT_BODY_NO_DESIRED = """### Acceptance Criteria
-- [ ] Something
-"""
-
-ENHANCEMENT_BODY_NO_ACCEPTANCE = """### Desired Behavior
-The button should animate on hover.
-"""
-
-ENHANCEMENT_BODY_PROSE_ACCEPTANCE = """### Desired Behavior
-The button should animate on hover.
-
-### Acceptance Criteria
-Make it look nice.
-"""
-
-
-# ---------------------------------------------------------------------------
-# Bug readiness
-# ---------------------------------------------------------------------------
-
-def test_bug_ready_npm_run_screenshot():
- result = evaluate_readiness(BUG_BODY_READY, [BUG_LABEL])
- assert result.ready, result.reasons
-
-def test_bug_ready_agent_canvas():
- result = evaluate_readiness(BUG_BODY_AGENT_CANVAS, [BUG_LABEL])
- assert result.ready, result.reasons
-
-def test_bug_ready_hosted_url():
- result = evaluate_readiness(BUG_BODY_HOSTED_URL, [BUG_LABEL])
- assert result.ready, result.reasons
-
-def test_bug_not_ready_no_run_method():
- result = evaluate_readiness(BUG_BODY_NO_RUN_METHOD, [BUG_LABEL])
- assert not result.ready
- assert any("run method" in r for r in result.reasons)
-
-def test_bug_not_ready_no_screenshot():
- result = evaluate_readiness(BUG_BODY_NO_SCREENSHOT, [BUG_LABEL])
- assert not result.ready
- assert any("screenshot" in r for r in result.reasons)
-
-def test_bug_not_ready_no_acceptance():
- result = evaluate_readiness(BUG_BODY_NO_ACCEPTANCE, [BUG_LABEL])
- assert not result.ready
- assert any("Acceptance Criteria" in r for r in result.reasons)
-
-def test_bug_not_ready_empty_actual():
- result = evaluate_readiness(BUG_BODY_EMPTY_ACTUAL, [BUG_LABEL])
- assert not result.ready
- assert any("Actual Behavior" in r for r in result.reasons)
-
-
-
-def test_bug_not_ready_missing_reproduction():
- result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [BUG_LABEL])
- assert not result.ready
- assert any("Steps to Reproduce" in r for r in result.reasons)
-
-# ---------------------------------------------------------------------------
-# Enhancement readiness
-# ---------------------------------------------------------------------------
-
-def test_enhancement_ready():
- result = evaluate_readiness(ENHANCEMENT_BODY_READY, [ENHANCEMENT_LABEL])
- assert result.ready, result.reasons
-
-def test_enhancement_not_ready_no_desired():
- result = evaluate_readiness(ENHANCEMENT_BODY_NO_DESIRED, [ENHANCEMENT_LABEL])
- assert not result.ready
- assert any("Desired Behavior" in r for r in result.reasons)
-
-def test_enhancement_not_ready_no_acceptance():
- result = evaluate_readiness(ENHANCEMENT_BODY_NO_ACCEPTANCE, [ENHANCEMENT_LABEL])
- assert not result.ready
- assert any("Acceptance Criteria" in r for r in result.reasons)
-
-def test_enhancement_not_ready_prose_acceptance():
- result = evaluate_readiness(ENHANCEMENT_BODY_PROSE_ACCEPTANCE, [ENHANCEMENT_LABEL])
- assert not result.ready
- assert any("checklist" in r for r in result.reasons)
-
-
-# ---------------------------------------------------------------------------
-# No type section
-# ---------------------------------------------------------------------------
-
-def test_no_type_section_not_ready():
- result = evaluate_readiness("### Something\nSome text", [])
- assert not result.ready
- assert result.issue_type is None
- assert any("neither" in r.lower() for r in result.reasons)
-
-
-def test_type_inferred_from_body_ignores_labels():
- # A bug-shaped body is treated as a bug even without a `bug` label.
- result = evaluate_readiness(BUG_BODY_READY, [])
- assert result.ready, result.reasons
- assert result.issue_type == BUG_LABEL
-
- # A feature-shaped body is treated as an enhancement even with a `bug` label.
- result = evaluate_readiness(ENHANCEMENT_BODY_READY, ["bug"])
- assert result.ready, result.reasons
- assert result.issue_type == ENHANCEMENT_LABEL
-
- # A body with only an `### Actual Behavior` section (no Steps to Reproduce)
- # is still recognized as a bug report.
- result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [])
- assert not result.ready
- assert any("Steps to Reproduce" in r for r in result.reasons)
-
-
-def test_empty_type_section_is_still_classified():
- result = evaluate_readiness("### Actual Behavior\n### Acceptance Criteria\n", [])
- assert not result.ready
- assert result.issue_type == BUG_LABEL
- assert any("Steps to Reproduce" in r for r in result.reasons)
-
-
-def test_ambiguous_body_does_not_infer_type():
- body = f"{BUG_BODY_READY}\n{ENHANCEMENT_BODY_READY}"
- result = evaluate_readiness(body, [])
- assert not result.ready
- assert result.issue_type is None
- assert any("both" in reason.lower() for reason in result.reasons)
-
-
-# ---------------------------------------------------------------------------
-# Unit-level helpers
-# ---------------------------------------------------------------------------
-
-def test_has_screenshot_markdown_image():
- assert has_screenshot_or_video("![alt](https://example.com/img.png)")
-
-def test_has_screenshot_github_attachment():
- assert has_screenshot_or_video("https://github.com/user-attachments/assets/abc123")
-
-def test_has_screenshot_html_video():
- assert has_screenshot_or_video('<video src="bug.mp4"></video>')
-
-def test_has_screenshot_youtube():
- assert has_screenshot_or_video("https://youtube.com/watch?v=abc123")
-
-def test_has_screenshot_none():
- assert not has_screenshot_or_video("Just text, no media")
-
-def test_references_run_method_npm():
- assert references_run_method("I ran npm run dev")
-
-def test_references_run_method_agent_canvas():
- assert references_run_method("Used agent-canvas to test")
-
-def test_references_run_method_hosted():
- assert references_run_method("Reproduced on app.all-hands.dev/canvas")
-
-def test_references_run_method_none():
- assert not references_run_method("I clicked the button")
-
-def test_has_checklist_item():
- assert has_checklist_item("- [ ] Do something")
- assert has_checklist_item("- [x] Done")
- assert has_checklist_item(" * [ ] Indented")
-
-def test_has_checklist_item_none():
- assert not has_checklist_item("Just prose, no checklist")
-
-def test_visible_text_strips_html_comments():
- assert visible_text("<!-- hidden -->visible text") == "visible text"
-
-def test_visible_text_no_response():
- assert visible_text("_No response_") == ""
-
-def test_extract_sections():
- sections = extract_sections("### Title One\nText 1\n### Title Two\nText 2")
- assert "title one" in sections
- assert "title two" in sections
- assert "Text 1" in sections["title one"]
- assert "Text 2" in sections["title two"]
-
-
-def test_bug_ready_with_h2_sections():
- body = BUG_BODY_READY.replace("### ", "## ")
- result = evaluate_readiness(body, [])
- assert result.ready, result.reasons
-
-
-def test_enhancement_ready_with_mixed_h2_h3_sections():
- body = ENHANCEMENT_BODY_READY.replace(
- "### Desired Behavior", "## Desired Behavior"
- )
- result = evaluate_readiness(body, [ENHANCEMENT_LABEL])
- assert result.ready, result.reasons
-
-
-def test_extract_sections_excludes_h1_and_h4_headings():
- body = "# Document Title\nintro\n#### Nested Detail\ntext"
- assert extract_sections(body) == {}
-
-
-def test_nested_h3_stays_inside_h2_readiness_section():
- body = """## Steps to Reproduce
-Run `npm run dev`.
-
-## Actual Behavior
-The page is broken.
-
-### Screenshot
-![broken page](https://github.com/user-attachments/assets/abc123)
-
-## Acceptance Criteria
-- [ ] The page works
-"""
- result = evaluate_readiness(body, [BUG_LABEL])
- assert result.ready, result.reasons
- assert "### Screenshot" in extract_sections(body)["actual behavior"]
-
-
-def test_extract_sections_ignores_h2_heading_inside_fence():
- body = """## Notes
-The template says:
-
-```markdown
-## Acceptance Criteria
-- [ ] Add criteria here
-```
-"""
- sections = extract_sections(body)
- assert set(sections) == {"notes"}
-
-
-def test_extract_sections_ignores_heading_inside_fence():
- body = """### Notes
-The template says:
-
-```markdown
-### Acceptance Criteria
-- [ ] Add criteria here
-```
-"""
- sections = extract_sections(body)
- assert set(sections) == {"notes"}
-
-def test_fenced_heading_does_not_truncate_actual_behavior():
- body = """### Steps to Reproduce
-Run `npm run dev` and reproduce the error.
-
-### Actual Behavior
-I ran `npm run dev` and saw:
-
-~~~text
-### Error detail
-something went wrong
-~~~
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-
-### Acceptance Criteria
-- [ ] The bug is fixed
-"""
- sections = extract_sections(body)
- assert "error detail" not in sections
- assert "user-attachments" in sections["actual behavior"]
- assert evaluate_readiness(body, [BUG_LABEL]).ready
-
-
-def test_unclosed_fence_does_not_swallow_later_sections():
- """One stray marker in a log paste must not reject an otherwise-ready report."""
- body = """### Steps to Reproduce
-Run `npm run dev` and reproduce the crash.
-
-### Relevant Logs
-```shell
-Traceback (most recent call last):
- the paste was cut off before the closing fence
-
-### Actual Behavior
-I ran `npm run dev` and saw the crash above.
-
-![screenshot](https://github.com/user-attachments/assets/abc123)
-
-### Acceptance Criteria
-- [ ] The bug is fixed
-"""
- sections = extract_sections(body)
- assert {"relevant logs", "actual behavior", "acceptance criteria"} <= set(sections)
- assert evaluate_readiness(body, [BUG_LABEL]).ready
-
-def test_main_json_not_ready(tmp_path, capsys, monkeypatch):
- import json
- body_file = tmp_path / "issue.md"
- body_file.write_text(BUG_BODY_NO_SCREENSHOT)
- monkeypatch.setattr(
- "sys.argv",
- [
- "check_issue_readiness.py",
- "--body-file",
- str(body_file),
- "--labels",
- "bug",
- "--json",
- ],
- )
- exit_code = main()
- assert exit_code == 1
- captured = capsys.readouterr()
- data = json.loads(captured.out)
- assert data["ready"] is False
- assert data["issue_type"] == BUG_LABEL
- assert len(data["reasons"]) > 0
-
-
-def test_main_json_ready(tmp_path, capsys, monkeypatch):
- import json
-
- body_file = tmp_path / "issue.md"
- body_file.write_text(BUG_BODY_READY)
- monkeypatch.setattr(
- "sys.argv",
- [
- "check_issue_readiness.py",
- "--body-file",
- str(body_file),
- "--labels",
- "bug",
- "--json",
- ],
- )
- exit_code = main()
- assert exit_code == 0
- captured = capsys.readouterr()
- data = json.loads(captured.out)
- assert data["ready"] is True
- assert data["issue_type"] == BUG_LABEL
- assert len(data["reasons"]) == 0
-
-
-def test_main_text_ready(tmp_path, capsys, monkeypatch):
- body_file = tmp_path / "issue.md"
- body_file.write_text(BUG_BODY_READY)
- monkeypatch.setattr(
- "sys.argv",
- [
- "check_issue_readiness.py",
- "--body-file",
- str(body_file),
- "--labels",
- "bug",
- ],
- )
- exit_code = main()
- assert exit_code == 0
- captured = capsys.readouterr()
- assert "Issue meets ready-for-dev criteria." in captured.out
-
-
-def test_main_text_not_ready(tmp_path, capsys, monkeypatch):
- body_file = tmp_path / "issue.md"
- body_file.write_text(BUG_BODY_NO_SCREENSHOT)
- monkeypatch.setattr(
- "sys.argv",
- [
- "check_issue_readiness.py",
- "--body-file",
- str(body_file),
- "--labels",
- "bug",
- ],
- )
- exit_code = main()
- assert exit_code == 1
- captured = capsys.readouterr()
- assert "Issue does not meet ready-for-dev criteria:" in captured.out
-
-
-def test_main_event_path_json_ready(tmp_path, capsys, monkeypatch):
- import json
-
- event_file = tmp_path / "event.json"
- event_file.write_text(
- json.dumps(
- {
- "issue": {
- "body": BUG_BODY_READY,
- "labels": [{"name": "bug"}],
- }
- }
- )
- )
- monkeypatch.setattr(
- "sys.argv",
- [
- "check_issue_readiness.py",
- "--event-path",
- str(event_file),
- "--json",
- ],
- )
- exit_code = main()
- assert exit_code == 0
- captured = capsys.readouterr()
- data = json.loads(captured.out)
- assert data["ready"] is True
- assert data["issue_type"] == BUG_LABEL
- assert len(data["reasons"]) == 0
diff --git a/.github/workflows/issue-readiness-check.yml b/.github/workflows/issue-readiness-check.yml
index c92b70960e..68dba2a0bf 100644
--- a/.github/workflows/issue-readiness-check.yml
+++ b/.github/workflows/issue-readiness-check.yml
@@ -1,155 +1,43 @@
---
-name: Issue Readiness Check
+name: Ready-for-dev Label Policy
-# Manages the `ready-for-dev` label based on type-specific readiness criteria.
-#
-# The type is inferred from the issue body rather than its labels: a body with
-# a `##`/`### Steps to Reproduce` or `##`/`### Actual Behavior` section is
-# checked as a bug report, and a body with a `##`/`### Desired Behavior`
-# section is checked as an enhancement. The criteria themselves are unchanged.
-# When exactly one type is inferred, the workflow also synchronizes the `bug`
-# and `enhancement` labels so API-created issues do not require manual labeling.
-#
-# When criteria are met the label is added (idempotently); when they are not the
-# label is removed. A feedback comment is posted (or updated) in three cases:
-# 1. The issue is first opened or reopened β€” always comment so the author
-# knows whether their issue is ready or what is missing.
-# 2. The label is being added (issue became ready) β€” comment celebrating it.
-# 3. The label is being removed (issue was ready but is no longer) β€” comment
-# explaining what changed.
-# Edits that do not change the label state do not produce a new comment. All
-# comments are upserted by a hidden marker so there is at most one per issue.
+# The issue-triage automation and repository writers may grant ready-for-dev.
+# Labels from any other actor are removed.
on:
issues:
- types: [opened, edited, reopened, labeled, unlabeled]
-
-permissions:
- issues: write
- contents: read
+ types: [labeled]
concurrency:
- group: issue-readiness-${{ github.event.issue.number }}
+ group: ready-for-dev-policy-${{ github.event.issue.number }}
cancel-in-progress: false
jobs:
- check:
- if: github.event.issue.state == 'open' && github.event.issue.pull_request == null
+ enforce-label-policy:
+ name: Enforce ready-for-dev label policy
+ if: >-
+ github.event.issue.state == 'open'
+ && github.event.issue.pull_request == null
+ && github.event.label.name == 'ready-for-dev'
runs-on: ubuntu-24.04
timeout-minutes: 5
+ permissions:
+ issues: write
steps:
- - name: Checkout trusted workflow scripts
- uses: actions/checkout@v7
-
- - name: Evaluate issue readiness
- id: readiness
- env:
- GITHUB_EVENT_PATH: ${{ github.event_path }}
- run: |
- set -euo pipefail
- python .github/scripts/check_issue_readiness.py --json > /tmp/result.json || true
- echo "ready=$(jq -r '.ready' /tmp/result.json)" >> "$GITHUB_OUTPUT"
- echo "issue_type=$(jq -r '.issue_type // empty' /tmp/result.json)" >> "$GITHUB_OUTPUT"
- # Write reasons to a file so the comment step can read them
- # without shell-escaping multiline text.
- jq -r '.reasons[]?' /tmp/result.json > /tmp/reasons.txt || true
- echo "reasons_file=/tmp/reasons.txt" >> "$GITHUB_OUTPUT"
-
- - name: Synchronize inferred issue type label
- if: steps.readiness.outputs.issue_type != ''
+ - name: Check label actor
env:
GH_TOKEN: ${{ github.token }}
- ISSUE_TYPE: ${{ steps.readiness.outputs.issue_type }}
+ ACTOR: ${{ github.event.sender.login }}
+ REPOSITORY: ${{ github.repository }}
+ ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
set -euo pipefail
- if ! issue_labels_json=$(gh issue view \
- "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- --json labels --jq '[.labels[].name]' 2>/tmp/gh-err.txt); then
- echo "::warning::Failed to read current issue labels:" \
- >> "$GITHUB_STEP_SUMMARY"
- cat /tmp/gh-err.txt >> "$GITHUB_STEP_SUMMARY"
- exit 0
- fi
-
- if [ "$ISSUE_TYPE" = "bug" ]; then
- other_type="enhancement"
- else
- other_type="bug"
- fi
-
- label_args=()
- if ! jq -e --arg label "$ISSUE_TYPE" \
- 'index($label) != null' <<< "$issue_labels_json" >/dev/null; then
- label_args+=(--add-label "$ISSUE_TYPE")
- fi
- if jq -e --arg label "$other_type" \
- 'index($label) != null' <<< "$issue_labels_json" >/dev/null; then
- label_args+=(--remove-label "$other_type")
- fi
-
- if [ "${#label_args[@]}" -eq 0 ]; then
- exit 0
- fi
-
- if ! gh issue edit "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- "${label_args[@]}" 2>/tmp/gh-err.txt; then
- echo "::warning::Failed to synchronize inferred issue type label:" \
- >> "$GITHUB_STEP_SUMMARY"
- cat /tmp/gh-err.txt >> "$GITHUB_STEP_SUMMARY"
- fi
-
- - name: Add ready-for-dev label
- if: steps.readiness.outputs.ready == 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- # Capture stderr β€” tolerate "label already present" (exit 0
- # from gh) but surface unexpected errors to the step summary.
- if ! gh issue edit "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- --add-label "ready-for-dev" 2>/tmp/gh-err.txt; then
- echo "::warning::Failed to add ready-for-dev label:" >> "$GITHUB_STEP_SUMMARY"
- cat /tmp/gh-err.txt >> "$GITHUB_STEP_SUMMARY"
- fi
-
- - name: Remove ready-for-dev label
- if: steps.readiness.outputs.ready != 'true' && contains(github.event.issue.labels.*.name, 'ready-for-dev')
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- # Capture stderr β€” tolerate "label already removed" but
- # surface unexpected errors.
- if ! gh issue edit "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- --remove-label "ready-for-dev" 2>/tmp/gh-err.txt; then
- echo "::warning::Failed to remove ready-for-dev label:" >> "$GITHUB_STEP_SUMMARY"
- cat /tmp/gh-err.txt >> "$GITHUB_STEP_SUMMARY"
- fi
-
- - name: Post feedback comment
- if: >-
- github.event.action == 'opened'
- || github.event.action == 'reopened'
- || (github.event.action == 'labeled' && github.event.label.name == 'ready-for-dev')
- || (steps.readiness.outputs.ready == 'true' && !contains(github.event.issue.labels.*.name, 'ready-for-dev'))
- || (steps.readiness.outputs.ready != 'true' && contains(github.event.issue.labels.*.name, 'ready-for-dev'))
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- if [ "${{ steps.readiness.outputs.ready }}" = "true" ]; then
- node .github/scripts/post-readiness-comment.mjs \
- --issue-number "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- --ready
- else
- node .github/scripts/post-readiness-comment.mjs \
- --issue-number "${{ github.event.issue.number }}" \
- --repo "${{ github.repository }}" \
- --reasons-file "${{ steps.readiness.outputs.reasons_file }}"
- fi
+ permission="$(gh api "repos/$REPOSITORY/collaborators/$ACTOR/permission" --jq .permission 2>/dev/null || true)"
+ case "$permission" in
+ write|maintain|admin)
+ ;;
+ *)
+ gh issue edit "$ISSUE_NUMBER" --repo "$REPOSITORY" --remove-label ready-for-dev
+ ;;
+ esac