Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Give the one-step run back the guarantees the collapse dropped

Daoud Clarkecea10b76881b
diff --git a/.github/claude/implement-issue.md b/.github/claude/implement-issue.md
index d710ed8..bd8d3c0 100644
--- a/.github/claude/implement-issue.md
+++ b/.github/claude/implement-issue.md
@@ -6,8 +6,8 @@ unbuilt part of one issue, verified green.
`ISSUE_NUMBER` is in the prompt; `$N` below means that number. You are inside the run
described by `.github/claude/run.md`: come back to its section 4 when you have committed,
-which is where the change is reviewed, pushed and turned into a pull request. Commit and
-stop.
+which is where the change is reviewed, and then its section 5, which pushes it and turns
+it into a pull request. Commit and stop.
## 1. Decide what to build
@@ -65,9 +65,10 @@ git commit -m "<section title>"
```
Stop there — do not push, do not open a pull request. Return to section 4 of
-`.github/claude/run.md`, which has the change reviewed and writes the pull request body.
-You need to carry this into it: what changed and why, how you verified it, and whether this
-was the last section of the plan, which decides between `Part of #$N` and `Closes #$N`.
+`.github/claude/run.md`, which has the change reviewed; its section 5 then writes the pull
+request body. You need to carry this into them: what changed and why, how you verified it,
+and whether this was the last section of the plan, which decides between `Part of #$N` and
+`Closes #$N`.
## Rules
diff --git a/.github/claude/plan-issue.md b/.github/claude/plan-issue.md
index 7d65ae8..4fee1dd 100644
--- a/.github/claude/plan-issue.md
+++ b/.github/claude/plan-issue.md
@@ -6,8 +6,8 @@ to turn one issue into a written plan: one commit adding exactly one file.
`ISSUE_NUMBER` is in the prompt; `$N` below means that number. You are inside the run
described by `.github/claude/run.md`: come back to its section 4 when you have committed,
-which is where the plan is reviewed, pushed and turned into a pull request. Commit and
-stop.
+which is where the plan is reviewed, and then its section 5, which pushes it and turns it
+into a pull request. Commit and stop.
## 1. Understand the issue
@@ -58,10 +58,10 @@ git commit -m "Plan issue #$N: <issue title>"
```
Stop there — do not push, do not open a pull request. Return to section 4 of
-`.github/claude/run.md`, which has the plan reviewed and writes the pull request body. You
-need to carry this into it: one paragraph on the approach and why it is split this way,
-then the sections with their line estimates. Merging that pull request is what starts
-implementation, so make the trade-offs easy to review.
+`.github/claude/run.md`, which has the plan reviewed; its section 5 then writes the pull
+request body. You need to carry this into them: one paragraph on the approach and why it is
+split this way, then the sections with their line estimates. Merging that pull request is
+what starts implementation, so make the trade-offs easy to review.
## Rules
diff --git a/.github/scripts/select-claude-work.sh b/.github/scripts/select-claude-work.sh
index 72056a0..fbf17ea 100755
--- a/.github/scripts/select-claude-work.sh
+++ b/.github/scripts/select-claude-work.sh
@@ -58,8 +58,16 @@ readonly only_issue="${CLAUDE_ONLY_ISSUE:-}"
# changed nothing. Only a run that commits nothing writes that one, because a run that
# pushes has not yet had a turn at what it just pushed — CI has not run on it. Kept in step
# with section 5 of .github/claude/run.md, which writes both.
-readonly marker='<!-- claude-run:'
-readonly turn_taken='<!-- claude-run: done -->'
+#
+# Both are anchored at the end of the body rather than matched anywhere in it, because
+# GitHub's "Quote reply" copies the quoted comment's raw markdown, HTML comments included.
+# A maintainer quote-replying to one of these comments would otherwise be read as the
+# automation's own voice: their reply would be dropped from the feedback that wakes a run,
+# which is the exact case the markers exist to unblock, and it would also count as a turn
+# taken and silence the failing checks on that commit. A quoted marker fails both halves:
+# it carries a "> " in front of it, and there is normally a reply after it.
+readonly marker='(^|\n)<!-- claude-run: [a-z]+ -->[[:space:]]*$'
+readonly turn_taken='(^|\n)<!-- claude-run: done -->[[:space:]]*$'
open_issues=$(gh issue list --state open --limit 200 --json number,title,labels)
open_pull_requests=$(gh pr list --state open --limit 200 \
@@ -81,8 +89,8 @@ open_pull_requests=$(gh pr list --state open --limit 200 \
# reasons stop counting for that commit, and only something said afterwards counts as
# feedback. A new commit, or a human replying, is what starts it moving again.
readonly wake_filter='
-def ours: (.body // "") | contains($marker);
-def had_a_turn: (.body // "") | contains($turn_taken);
+def ours: (.body // "") | test($marker);
+def had_a_turn: (.body // "") | test($turn_taken);
def maintainer: .authorAssociation | IN("OWNER", "MEMBER", "COLLABORATOR");
def failing: [.statusCheckRollup[] | select(.conclusion == "FAILURE") | .name];
diff --git a/.github/workflows/claude-issues.yml b/.github/workflows/claude-issues.yml
index 469ba15..a0683fb 100644
--- a/.github/workflows/claude-issues.yml
+++ b/.github/workflows/claude-issues.yml
@@ -155,8 +155,15 @@ jobs:
if: needs.select.outputs.has_items == 'true'
runs-on: ubuntu-latest
# One Claude run and the review subagent inside it, both iterating on a 67-second test
- # suite.
- timeout-minutes: 120
+ # suite, on a change capped at about 500 added lines.
+ #
+ # The bound is the App token, not the work. claude-code-action mints an installation
+ # token when the step starts and GitHub expires those after an hour, so a step that
+ # outlives the hour reaches `git push` with dead credentials and throws the whole run
+ # away. Every work job measured so far has finished between 2 and 16 minutes, so 45 is
+ # roughly three times the worst case and still safely inside the token's life. A run
+ # that reaches it is stuck rather than slow, and killing it is the right outcome.
+ timeout-minutes: 45
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.select.outputs.items_matrix) }}
@@ -186,8 +193,6 @@ jobs:
env:
ISSUE_NUMBER: ${{ matrix.item.issue }}
- DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_mwmbl
- DJANGO_SETTINGS_MODULE: mwmbl.settings_test
steps:
# The log lives outside the checkout. The run holds an unrestricted Bash tool and is
@@ -197,8 +202,21 @@ jobs:
# It is set here rather than in the job's `env` above because the `runner` context
# does not exist at job level: a `${{ runner.temp }}` there is not an empty string but
# a workflow that fails to validate, which is what broke every run from 2f0de72 on.
- - name: Point the run log outside the checkout
- run: echo "CLAUDE_RUN_LOG=$RUNNER_TEMP/claude-runs.jsonl" >>"$GITHUB_ENV"
+ #
+ # The gates the run has to get green need the same database and settings as ci.yml,
+ # and they are exported here rather than in the job's `env` above because a composite
+ # action's steps do not inherit the calling job's environment: claude-code-action's
+ # manifest says so itself, and forwards only the variables it lists by name. A
+ # `DATABASE_URL` at job level reaches every plain step and never reaches the CLI, so
+ # `uv run pytest` would fail for want of a database in a way nothing else reports.
+ # $GITHUB_ENV is the route the action's own note recommends.
+ - name: Set up the environment the run works in
+ run: |
+ {
+ echo "CLAUDE_RUN_LOG=$RUNNER_TEMP/claude-runs.jsonl"
+ echo "DATABASE_URL=postgres://postgres:postgres@localhost:5432/test_mwmbl"
+ echo "DJANGO_SETTINGS_MODULE=mwmbl.settings_test"
+ } >>"$GITHUB_ENV"
# Full history: the run diffs the branch against main and pushes it.
#
# The ref is pinned because `pull_request` hands github.ref as refs/pull/<n>/merge:
diff --git a/test/test_claude_automation_workflow.py b/test/test_claude_automation_workflow.py
index 9c88df4..48b5859 100644
--- a/test/test_claude_automation_workflow.py
+++ b/test/test_claude_automation_workflow.py
@@ -1,4 +1,4 @@
-"""The issue automation is one Claude session wired into GitHub, and a run costs an hour.
+"""The issue automation is one Claude session wired into GitHub, and a run costs real money.
Almost all of it is prompt: the run decides what the issue needs, has the change reviewed
by a subagent, pushes, and opens or comments on the pull request itself. Only two pieces of
@@ -13,8 +13,11 @@ that feedback belongs. What the run does with its instructions is not testable h
deliberately not faked: this file checks the wiring, not the judgement.
"""
+import json
import os
import re
+import shutil
+import subprocess
from pathlib import Path
import pytest
@@ -89,6 +92,24 @@ def test_instruction_files_only_cross_reference_files_that_exist() -> None:
assert (REPOSITORY_ROOT / path).is_file(), f"{instruction_file.name} -> {path}"
+def test_instruction_files_only_cross_reference_sections_that_exist() -> None:
+ """The filename guard above does not catch a section that moved. Sending a run back to
+ the section that reviews the change when it wanted the one that pushes is how a push
+ ends up skipping the rules that bound the retries."""
+ for instruction_file in sorted((REPOSITORY_ROOT / ".github/claude").glob("*.md")):
+ for block in re.split(r"\n\s*\n", instruction_file.read_text()):
+ sections = re.findall(r"\bsections? (\d+)(?:\s+and\s+(\d+))?", block)
+ if not sections:
+ continue
+ # A block that names no file is talking about itself.
+ named = set(re.findall(r"\.github/claude/[\w-]+\.md", block))
+ assert len(named) <= 1, f"{instruction_file.name}: ambiguous target {named}"
+ target = REPOSITORY_ROOT / named.pop() if named else instruction_file
+ headings = re.findall(r"^## (\d+)\.", target.read_text(), re.MULTILINE)
+ for number in {n for pair in sections for n in pair if n}:
+ assert number in headings, f"{instruction_file.name} -> {target.name} #{number}"
+
+
def test_the_run_is_told_to_have_the_change_reviewed_before_it_pushes(run_instructions: str) -> None:
"""Nothing enforces this any more, so the ordering has to be in the instructions and
has to stay there: a review after the push is a review a human has already seen past."""
@@ -288,6 +309,61 @@ def test_forks_are_not_the_automations_to_work_on() -> None:
assert "isCrossRepository" in SELECTOR_PATH.read_text()
+def test_a_run_cannot_outlive_the_app_token_it_pushes_with(workflow: dict) -> None:
+ """claude-code-action mints a GitHub App installation token when the step starts, and
+ GitHub expires those after an hour. The run pushes at the end of that same step, so a
+ job allowed to run longer than the token lives can spend its whole budget and then fail
+ to publish any of it — and a discarded run looks like a stuck issue, not a dead token.
+ Every work job measured so far finished well inside 20 minutes."""
+ timeout = workflow["jobs"]["work"]["timeout-minutes"]
+ assert timeout < 60, timeout
+
+
+def test_the_gates_environment_reaches_the_cli_and_not_only_the_shell(workflow: dict, work_steps: list[dict]) -> None:
+ """A composite action's steps do not inherit the calling job's env, and
+ claude-code-action forwards only the variables it names — DATABASE_URL is not one of
+ them. Set at job level it reaches every plain step and never the CLI, so the run's own
+ `uv run pytest` fails for want of a database while everything around it looks right."""
+ required = {"DATABASE_URL", "DJANGO_SETTINGS_MODULE"}
+ job_level = set(workflow["jobs"]["work"].get("env", {}))
+ assert not (required & job_level), required & job_level
+ exported = "".join(step.get("run", "") for step in work_steps if "GITHUB_ENV" in step.get("run", ""))
+ for name in required:
+ assert f"{name}=" in exported, name
+
+
[email protected](shutil.which("jq") is None, reason="jq drives the selector's filter")
+ "body,is_ours,is_turn",
+ [
+ ("Nothing changed.\n\n<!-- claude-run: done -->", True, True),
+ ("Pushed a fix.\n\n<!-- claude-run: update -->", True, False),
+ ("> earlier\n> <!-- claude-run: done -->\n\nBut what about X?", False, False),
+ ("What about X?\n\n> earlier\n> <!-- claude-run: done -->", False, False),
+ ],
+)
+def test_a_quoted_marker_is_not_the_automations_own_voice(body: str, is_ours: bool, is_turn: bool) -> None:
+ """GitHub's "Quote reply" copies the quoted comment's raw markdown, HTML comments
+ included. Read as the automation's own voice, a maintainer's reply is dropped from the
+ feedback that wakes a run — the exact case the markers exist to unblock — and counts as
+ a turn taken, silencing the failing checks on that commit."""
+ selector = SELECTOR_PATH.read_text()
+ patterns = dict(re.findall(r"^readonly (marker|turn_taken)='(.+)'$", selector, re.MULTILINE))
+ assert patterns.keys() == {"marker", "turn_taken"}, patterns
+
+ def matches(pattern: str) -> bool:
+ result = subprocess.run(
+ ["jq", "-nr", "--arg", "b", body, "--arg", "rx", pattern, "$b | test($rx)"],
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return json.loads(result.stdout)
+
+ assert matches(patterns["marker"]) is is_ours
+ assert matches(patterns["turn_taken"]) is is_turn
+
+
def test_referenced_scripts_are_executable(work_steps: list[dict], workflow: dict) -> None:
steps = work_steps + workflow["jobs"]["select"]["steps"]
scripts = {step["run"].split()[0] for step in steps if step.get("run", "").startswith(".github/")}