Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(ci): ignore fenced headings in readiness checks (#16595)

ump45nose9e8ba84f7547
diff --git a/.github/scripts/check_issue_readiness.py b/.github/scripts/check_issue_readiness.py
index 8f685b9d4a..05943d0bb0 100644
--- a/.github/scripts/check_issue_readiness.py
+++ b/.github/scripts/check_issue_readiness.py
@@ -33,6 +33,8 @@ import sys
from dataclasses import dataclass, field
from pathlib import Path
+from markdown_sections import find_headings
+
BUG_LABEL = "bug"
ENHANCEMENT_LABEL = "enhancement"
@@ -107,7 +109,7 @@ def extract_sections(body: str) -> dict[str, str]:
form) may still use `###` headings; if they don't, the map is empty and the
caller falls back to whole-body checks.
"""
- matches = list(HEADING_RE.finditer(body))
+ matches = find_headings(body, HEADING_RE)
sections: dict[str, str] = {}
for index, match in enumerate(matches):
start = match.end()
diff --git a/.github/scripts/check_pr_description.py b/.github/scripts/check_pr_description.py
index 185eef34e2..9930cb23f5 100644
--- a/.github/scripts/check_pr_description.py
+++ b/.github/scripts/check_pr_description.py
@@ -32,6 +32,8 @@ import re
import sys
from pathlib import Path
+from markdown_sections import find_headings, without_fenced_code_blocks
+
# Reject placeholders while allowing a concise human-written sentence.
MIN_HUMAN_NOTE_CHARS = 20
@@ -132,7 +134,7 @@ def first_visible_line(text: str) -> str:
def extract_sections(body: str) -> dict[str, str]:
- matches = list(HEADING_RE.finditer(body))
+ matches = find_headings(body, HEADING_RE)
sections: dict[str, str] = {}
for index, match in enumerate(matches):
start = match.end()
@@ -142,12 +144,19 @@ def extract_sections(body: str) -> dict[str, str]:
def extract_human_note(body: str) -> str:
- """Return human-written text in the required location before `AGENT:`."""
- human_match = HUMAN_HEADING_RE.search(body)
+ """Return human-written text in the required location before `AGENT:`.
+
+ The markers are located outside fenced code blocks so that quoting the
+ template does not stand in for filling it out. Offsets are preserved by the
+ masking, so the note itself is still read from the original body.
+ """
+ outside_fences = without_fenced_code_blocks(body)
+
+ human_match = HUMAN_HEADING_RE.search(outside_fences)
if human_match is None:
return ""
- agent_match = AGENT_HEADING_RE.search(body, human_match.end())
+ agent_match = AGENT_HEADING_RE.search(outside_fences, human_match.end())
if agent_match is None:
return ""
@@ -387,7 +396,7 @@ def validate_pr_body(body: str, files: list[str] | None = None) -> list[str]:
if len(human_note) < MIN_HUMAN_NOTE_CHARS:
errors.append("Add a short human-written note between `HUMAN:` and `AGENT:`.")
- if AGENT_HEADING_RE.search(body) is None:
+ if AGENT_HEADING_RE.search(without_fenced_code_blocks(body)) is None:
errors.append("Keep the `AGENT:` marker from the PR template.")
sections = extract_sections(body)
diff --git a/.github/scripts/markdown_sections.py b/.github/scripts/markdown_sections.py
new file mode 100644
index 0000000000..872264420d
--- /dev/null
+++ b/.github/scripts/markdown_sections.py
@@ -0,0 +1,94 @@
+"""Helpers for parsing Markdown sections in GitHub issue and PR bodies."""
+
+import re
+from dataclasses import dataclass
+
+# CommonMark allows an opening fence to carry leading indentation, and inside a
+# list item that indentation is measured from the list's content column rather
+# than the left margin. Rather than parse lists, accept any indentation on the
+# opening fence and bound the closing fence relative to it (below).
+FENCE_LINE_RE = re.compile(
+ r"^(?P<indent>[ ]*)(?P<marker>`{3,}|~{3,})(?P<rest>[^\r\n]*)"
+)
+
+# CommonMark lets a closing fence sit up to three spaces further in than the
+# fence it closes; beyond that it is content, not a terminator.
+MAX_CLOSING_INDENT_OFFSET = 3
+
+
+def _mask_line(line: str) -> str:
+ """Replace line content while preserving offsets and line endings."""
+ return "".join(char if char in "\r\n" else " " for char in line)
+
+
+@dataclass(frozen=True)
+class _Fence:
+ """The marker used to open a fenced Markdown code block."""
+
+ char: str
+ length: int
+ indent: int
+
+ @classmethod
+ def opened_by(cls, line: str) -> "_Fence | None":
+ """Return the fence opened by ``line``, if any."""
+ match = FENCE_LINE_RE.match(line)
+ if match is None:
+ return None
+ marker = match.group("marker")
+ return cls(
+ char=marker[0],
+ length=len(marker),
+ indent=len(match.group("indent")),
+ )
+
+ def closed_by(self, line: str) -> bool:
+ """Return whether ``line`` closes this fence."""
+ match = FENCE_LINE_RE.match(line)
+ if match is None:
+ return False
+ marker = match.group("marker")
+ return (
+ marker[0] == self.char
+ and len(marker) >= self.length
+ and len(match.group("indent")) <= self.indent + MAX_CLOSING_INDENT_OFFSET
+ and not match.group("rest").strip()
+ )
+
+
+def without_fenced_code_blocks(body: str) -> str:
+ """Mask fenced code blocks without changing character offsets.
+
+ A fence that is never closed is left untouched. CommonMark would run it to
+ the end of the document, but these parsers gate contributions: one stray
+ marker in a pasted log would hide every heading after it and reject a report
+ whose sections are plainly there. Masking only balanced fences keeps the
+ failure on the side of accepting.
+ """
+ masked_lines: list[str] = []
+ fenced_lines: list[str] = []
+ fence: _Fence | None = None
+
+ for line in body.splitlines(keepends=True):
+ if fence is None:
+ fence = _Fence.opened_by(line)
+ if fence is None:
+ masked_lines.append(line)
+ else:
+ fenced_lines.append(line)
+ else:
+ fenced_lines.append(line)
+ if fence.closed_by(line):
+ masked_lines.extend(_mask_line(text) for text in fenced_lines)
+ fenced_lines.clear()
+ fence = None
+
+ # Whatever is left belongs to an unclosed fence, so it stays as written.
+ masked_lines.extend(fenced_lines)
+
+ return "".join(masked_lines)
+
+
+def find_headings(body: str, heading_re: re.Pattern[str]) -> list[re.Match[str]]:
+ """Return heading matches outside fenced code blocks."""
+ return list(heading_re.finditer(without_fenced_code_blocks(body)))
diff --git a/.github/scripts/tests/test_issue_readiness.py b/.github/scripts/tests/test_issue_readiness.py
index 51b5bcfd94..f6d5b784d3 100644
--- a/.github/scripts/tests/test_issue_readiness.py
+++ b/.github/scripts/tests/test_issue_readiness.py
@@ -227,3 +227,54 @@ def test_extract_sections():
assert "title two" in sections
assert "Text 1" in sections["title one"]
assert "Text 2" in sections["title two"]
+
+def test_extract_sections_ignores_heading_inside_fence():
+ body = """### Notes
+The template says:
+
+```markdown
+### Acceptance Criteria
+- [ ] Add criteria here
+```
+"""
+ sections = extract_sections(body)
+ assert set(sections) == {"notes"}
+
+def test_fenced_heading_does_not_truncate_actual_behavior():
+ body = """### Actual Behavior
+I ran `npm run dev` and saw:
+
+~~~text
+### Error detail
+something went wrong
+~~~
+
+![screenshot](https://github.com/user-attachments/assets/abc123)
+
+### Acceptance Criteria
+- [ ] The bug is fixed
+"""
+ sections = extract_sections(body)
+ assert "error detail" not in sections
+ assert "user-attachments" in sections["actual behavior"]
+ assert evaluate_readiness(body, [BUG_LABEL]).ready
+
+
+def test_unclosed_fence_does_not_swallow_later_sections():
+ """One stray marker in a log paste must not reject an otherwise-ready report."""
+ body = """### Relevant Logs
+```shell
+Traceback (most recent call last):
+ the paste was cut off before the closing fence
+
+### Actual Behavior
+I ran `npm run dev` and saw the crash above.
+
+![screenshot](https://github.com/user-attachments/assets/abc123)
+
+### Acceptance Criteria
+- [ ] The bug is fixed
+"""
+ sections = extract_sections(body)
+ assert {"relevant logs", "actual behavior", "acceptance criteria"} <= set(sections)
+ assert evaluate_readiness(body, [BUG_LABEL]).ready
diff --git a/.github/scripts/tests/test_markdown_sections.py b/.github/scripts/tests/test_markdown_sections.py
new file mode 100644
index 0000000000..2ef26cc38d
--- /dev/null
+++ b/.github/scripts/tests/test_markdown_sections.py
@@ -0,0 +1,135 @@
+"""Tests for markdown_sections.py β€” the fence-aware heading scanner."""
+
+import re
+import sys
+from pathlib import Path
+
+# Make the sibling script importable.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from markdown_sections import find_headings, without_fenced_code_blocks
+
+H3_RE = re.compile(r"(?m)^###\s+(.+?)\s*$")
+
+
+def headings(body: str) -> list[str]:
+ return [match.group(1) for match in find_headings(body, H3_RE)]
+
+
+# ---------------------------------------------------------------------------
+# Offsets
+# ---------------------------------------------------------------------------
+
+
+def test_masking_preserves_offsets():
+ body = "### One\n```py\n### Two\n```\n### Three\n"
+ masked = without_fenced_code_blocks(body)
+ assert len(masked) == len(body)
+ assert masked.splitlines() == [
+ "### One",
+ " ",
+ " ",
+ " ",
+ "### Three",
+ ]
+
+
+def test_masking_preserves_crlf_offsets():
+ body = "### One\r\n```py\r\n### Two\r\n```\r\n### Three\r\n"
+ masked = without_fenced_code_blocks(body)
+ assert len(masked) == len(body)
+ assert headings(body) == ["One", "Three"]
+
+
+# ---------------------------------------------------------------------------
+# Unclosed fences
+# ---------------------------------------------------------------------------
+
+
+def test_unclosed_fence_does_not_hide_later_headings():
+ """A stray marker in a pasted log must not swallow the rest of the body."""
+ body = """### Relevant Logs
+```shell
+Traceback (most recent call last):
+ the paste was cut off here
+
+### Actual Behavior
+It broke.
+"""
+ assert headings(body) == ["Relevant Logs", "Actual Behavior"]
+
+
+def test_unclosed_fence_leaves_text_untouched():
+ body = "### One\n~~~\nstill here\n"
+ assert without_fenced_code_blocks(body) == body
+
+
+def test_second_fence_reopens_after_a_balanced_one():
+ body = """### One
+```
+### Masked
+```
+### Two
+```
+### Not masked, fence never closes
+"""
+ assert headings(body) == ["One", "Two", "Not masked, fence never closes"]
+
+
+# ---------------------------------------------------------------------------
+# Fence markers
+# ---------------------------------------------------------------------------
+
+
+def test_tilde_fence_is_not_closed_by_backticks():
+ body = "### One\n~~~\n### Masked\n```\n### Also masked\n~~~\n### Two\n"
+ assert headings(body) == ["One", "Two"]
+
+
+def test_longer_fence_is_not_closed_by_a_shorter_one():
+ body = "### One\n````\n### Masked\n```\n### Also masked\n````\n### Two\n"
+ assert headings(body) == ["One", "Two"]
+
+
+def test_info_string_does_not_close_a_fence():
+ body = "### One\n```\n### Masked\n```py\n### Also masked\n```\n### Two\n"
+ assert headings(body) == ["One", "Two"]
+
+
+# ---------------------------------------------------------------------------
+# Indentation
+# ---------------------------------------------------------------------------
+
+
+def test_fence_indented_inside_a_list_item_is_masked():
+ """CommonMark measures a nested fence from the list's content column."""
+ body = """### Steps to Reproduce
+
+1. Quote the template:
+
+ ```markdown
+### Acceptance Criteria
+ - [ ] not a real criterion
+ ```
+
+### Actual Behavior
+It broke.
+"""
+ assert headings(body) == ["Steps to Reproduce", "Actual Behavior"]
+
+
+def test_closing_fence_indented_past_the_limit_does_not_close():
+ body = """### One
+```text
+### Masked
+ ```
+### Also masked
+```
+### Two
+"""
+ assert headings(body) == ["One", "Two"]
+
+
+def test_closing_fence_may_be_indented_three_spaces():
+ body = "### One\n```\n### Masked\n ```\n### Two\n"
+ assert headings(body) == ["One", "Two"]
diff --git a/.github/scripts/tests/test_pr_description.py b/.github/scripts/tests/test_pr_description.py
index 8fe26393af..b16bcf4921 100644
--- a/.github/scripts/tests/test_pr_description.py
+++ b/.github/scripts/tests/test_pr_description.py
@@ -8,6 +8,9 @@ from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from check_pr_description import (
+ extract_human_note,
+ extract_sections,
+ validate_pr_body,
is_frontend_file,
touches_frontend,
extract_linked_issue_numbers,
@@ -19,6 +22,37 @@ from check_pr_description import (
READY_FOR_DEV_LABEL,
)
+
+def test_extract_sections_ignores_heading_inside_fence():
+ body = """## Summary
+The template says:
+
+```markdown
+## How to Test
+Describe testing here
+```
+"""
+ assert set(extract_sections(body)) == {"Summary"}
+
+
+def test_fenced_heading_does_not_truncate_how_to_test():
+ body = """## How to Test
+Run the focused checker tests.
+
+~~~text
+## Example output
+all tests passed
+~~~
+
+Then run the reproduction script.
+
+## Issue Number
+Fixes #16553
+"""
+ sections = extract_sections(body)
+ assert "Example output" not in sections
+ assert "Then run the reproduction script." in sections["How to Test"]
+
# ---------------------------------------------------------------------------
# extract_linked_issue_numbers
# ---------------------------------------------------------------------------
@@ -244,3 +278,46 @@ def test_docs_only_change_does_not_require_frontend_evidence():
def test_mixed_change_still_requires_frontend_evidence():
assert touches_frontend(["__tests__/router.md", "src/app.tsx"])
+
+
+# ---------------------------------------------------------------------------
+# HUMAN: / AGENT: markers
+# ---------------------------------------------------------------------------
+
+
+FENCED_TEMPLATE_BODY = """HUMAN:
+
+Quoting the template I was asked to fill in:
+
+```
+HUMAN:
+I really did test this thoroughly by hand, twice, on my own machine.
+AGENT:
+```
+"""
+
+
+def test_fenced_human_marker_does_not_supply_the_note():
+ assert extract_human_note(FENCED_TEMPLATE_BODY) == ""
+
+
+def test_fenced_agent_marker_does_not_satisfy_validation():
+ errors = validate_pr_body(FENCED_TEMPLATE_BODY)
+ assert "Add a short human-written note between `HUMAN:` and `AGENT:`." in errors
+ assert "Keep the `AGENT:` marker from the PR template." in errors
+
+
+def test_real_markers_still_supply_the_note():
+ body = """HUMAN:
+
+I ran the checker suite and reproduced the fenced-heading case by hand.
+
+AGENT:
+
+## Why
+x
+"""
+ assert "reproduced the fenced-heading case" in extract_human_note(body)
+ errors = validate_pr_body(body)
+ assert "Keep the `AGENT:` marker from the PR template." not in errors
+ assert "Add a short human-written note between `HUMAN:` and `AGENT:`." not in errors