graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Publication text: a base-reproduced failure never permits non-draft (#1197)
justrache76a9d59fd03
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex 92d0d68e..f17997a5 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -116,7 +116,7 @@{"args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "github_fixture": {"checks": "SUCCESS", "runs": "success", "review_files": ["packages/client/package.json", "packages/client/test_runner.py", "packages/client/value.py", ".github/workflows/client.yml"]}, "workspace_seed": {"prepare.py": "import json, pathlib, subprocess\nrun=lambda *args: subprocess.check_output(args,text=True).strip()\nstate=json.loads(pathlib.Path('gh-state.json').read_text())\nstate['base_sha']=run('git','rev-parse','HEAD')\npathlib.Path('packages/client/value.py').write_text('VALUE = 2\\n')\nrun('git','add','packages/client/value.py')\nrun('git','-c','user.name=Fixture','-c','[email protected]','-c','commit.gpgsign=false','commit','-qm','change')\nstate['initial_head']=run('git','rev-parse','HEAD')\npathlib.Path('gh-state.json').write_text(json.dumps(state))\n", "packages/client/package.json": "{\"scripts\": {\"test\": \"python3 test_runner.py\"}}", "packages/client/test_runner.py": "import os\nfrom value import VALUE\nassert VALUE == int(os.environ.get('EXPECTED', '2'))\nprint('CHECK_RECEIPT_OK')\n", "packages/client/value.py": "VALUE = 1\n", ".github/workflows/client.yml": "name: client\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: cd packages/client && python3 test_runner.py\n"}, "prompt": "Run the isolated publication fixture and report its result.", "id": "publication-rlm-check-evidence", "why": "#1168/#1178/#1180: canonical shell checks and unchanged nested runner configuration reach immutable publication review.", "script": [{"tool": "shell", "arguments": {"command": "python3 prepare.py"}}, {"tool": "rlm", "arguments": {"code": "print(bash(\"cd packages/client && python3 test_runner.py\"))"}}, {"tool": "shell", "arguments": {"command": "gh pr ready 1"}}, {"text": "{\"verdict\":\"supported\",\"reason\":\"Fixture confirms supplied committed runner and observed check.\"}"}, {"text": "Publication fixture complete."}], "assert": [{"request_contains": {"index": 3, "text": "CHECK_RECEIPT_OK"}}, {"request_contains": {"index": 3, "text": "packages/client/package.json"}}, {"request_contains": {"index": 3, "text": "packages/client/test_runner.py"}}, {"request_contains": {"index": 3, "text": ".github/workflows/client.yml"}}, {"file_equals": {"path": "mutations.jsonl", "content": "[\"pr\", \"ready\", \"1\"]\n"}}]}{"args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "github_fixture": {"checks": "SUCCESS", "runs": "success", "review_files": ["packages/client/package.json", "packages/client/test_runner.py", "packages/client/value.py", ".github/workflows/client.yml"]}, "workspace_seed": {"prepare.py": "import json, pathlib, subprocess\nrun=lambda *args: subprocess.check_output(args,text=True).strip()\nstate=json.loads(pathlib.Path('gh-state.json').read_text())\nstate['base_sha']=run('git','rev-parse','HEAD')\npathlib.Path('packages/client/value.py').write_text('VALUE = 2\\n')\nrun('git','add','packages/client/value.py')\nrun('git','-c','user.name=Fixture','-c','[email protected]','-c','commit.gpgsign=false','commit','-qm','change')\nstate['initial_head']=run('git','rev-parse','HEAD')\npathlib.Path('gh-state.json').write_text(json.dumps(state))\n", "packages/client/package.json": "{\"scripts\": {\"test\": \"python3 test_runner.py\"}}", "packages/client/test_runner.py": "import os\nfrom value import VALUE\nassert VALUE == int(os.environ.get('EXPECTED', '2'))\nprint('CHECK_RECEIPT_OK')\n", "packages/client/value.py": "VALUE = 1\n", ".github/workflows/client.yml": "name: client\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: cd packages/client && python3 test_runner.py\n"}, "prompt": "Run the isolated publication fixture and report its result.", "id": "publication-rlm-failed-check-retained", "why": "#1168: recording canonical shell successes must also preserve its failures in the publication gate.", "script": [{"tool": "shell", "arguments": {"command": "python3 prepare.py"}}, {"tool": "rlm", "arguments": {"code": "print(bash(\"cd packages/client && EXPECTED=3 python3 test_runner.py\"))"}}, {"tool": "shell", "arguments": {"command": "gh pr ready 1"}}, {"text": "Publication remains blocked by the observed failed check."}], "assert": [{"request_contains": {"text": "observed local check has no successful completion"}}, {"no_event": {"type": "tool_call_finished", "name": "shell", "is_error": false, "text": "https://example.invalid/pull/1"}}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path('mutations.jsonl').exists()"]}{"args": ["--old"], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "Run the isolated publication fixture and report its result.", "id": "publication-pending-ci-success", "why": "#1179: pending exact-head branch CI is reread before deciding whether publication is ready.", "github_fixture": {"checks": "SUCCESS", "runs": "pending", "run_sequence": ["pending", "success"], "review_files": ["notes.md"]}, "script": [{"tool": "shell", "arguments": {"command": "gh pr create --title fixture --body-file notes.md"}}, {"text": "{\"verdict\":\"supported\",\"reason\":\"Controlled committed fixture review.\"}"}, {"text": "Ready publication complete."}], "assert": [{"file_equals": {"path": "mutations.jsonl", "content": "[\"pr\", \"create\", \"--title\", \"fixture\", \"--body-file\", \"notes.md\"]\n"}}]}-{"args": ["--old"], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "Run the isolated publication fixture and report its result.", "id": "publication-pending-ci-failure", "why": "#1179: pending exact-head branch CI is reread before deciding whether publication is ready.", "github_fixture": {"checks": "SUCCESS", "runs": "pending", "run_sequence": ["pending", "failure"], "review_files": ["notes.md"]}, "script": [{"tool": "shell", "arguments": {"command": "gh pr create --title fixture --body-file notes.md"}}, {"text": "Publication remains blocked by failed CI."}], "assert": [{"request_contains": {"text": "exact head SHA has an unexplained failed branch run"}}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path('mutations.jsonl').exists()"]}+{"args": ["--old"], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "Run the isolated publication fixture and report its result.", "id": "publication-pending-ci-failure", "why": "#1179: pending exact-head branch CI is reread before deciding whether publication is ready.", "github_fixture": {"checks": "SUCCESS", "runs": "pending", "run_sequence": ["pending", "failure"], "review_files": ["notes.md"]}, "script": [{"tool": "shell", "arguments": {"command": "gh pr create --title fixture --body-file notes.md"}}, {"text": "Publication remains blocked by failed CI."}], "assert": [{"request_contains": {"text": "exact head SHA has a failed branch run"}}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path('mutations.jsonl').exists()"]}{"id": "btw-advertised-tools-never-execute", "why": "ADR0011: cached tools remain advertised to /btw, but an unsolicited tool call must never dispatch or pollute the parent history.", "workspace_seed": {"sentinel.txt": "unchanged"}, "prompt": "Describe the current task briefly.", "controls_after_turn": [{"type": "btw", "text": "SIDE-QUESTION-SENTINEL"}], "second_prompt": "Continue the original discussion.", "script": [{"text": "Initial discussion."}, {"tool": "write_file", "arguments": {"path": "sentinel.txt", "content": "SIDE-EFFECT-SENTINEL"}}, {"text": "Original discussion continues."}], "assert": [{"event": {"type": "error", "message": "side question failed; the conversation is unchanged"}}, {"no_event": {"type": "tool_call"}}, {"file_equals": {"path": "sentinel.txt", "content": "unchanged"}}, {"request_contains": {"index": 1, "text": "\"tools\""}}, {"request_lacks": {"index": -1, "text": "SIDE-QUESTION-SENTINEL"}}, {"request_lacks": {"index": -1, "text": "SIDE-EFFECT-SENTINEL"}}, {"final_text_contains": "Original discussion continues."}]}{"id": "completion-result-keeps-evidence-and-limits", "why": "A result-only completion must deliver the full verified answer and remaining limitations without requiring duplicate assistant prose; this checks delivery, not model compliance with the wording.", "prompt": "Run the fixture verification command, then report the exact evidence and the remaining integration-test limitation.", "script": [{"tool": "shell", "arguments": {"command": "printf 'fixture verification: OK\\n'"}}, {"tool": "attempt_completion", "arguments": {"result": "Verified: fixture verification: OK.\nRemaining: integration tests were not run."}}], "assert": [{"event": {"type": "tool_result", "name": "shell", "is_error": false}}, {"final_text_contains": "Verified: fixture verification: OK."}, {"final_text_contains": "Remaining: integration tests were not run."}, {"events_at_most": {"match": {"type": "model_call_started"}, "count": 2}}]}{"args": ["--old"], "prompt": "Run the scripted verification fixture and report only observed results.", "id": "completion-pending-automatic", "why": "An unfinished shell receipt cannot satisfy same-batch completion; pending is not command failure.", "script": [{"tools": [{"tool": "shell", "arguments": {"command": "sleep 1; exit 0", "timeout": 1}}, {"tool": "attempt_completion", "arguments": {"result": "PREMATURE_COMPLETION"}}]}, {"text": "Verification is still pending."}], "assert": [{"event": {"type": "tool_call_finished", "name": "shell", "is_error": false}}, {"event": {"type": "tool_call_finished", "name": "attempt_completion", "is_error": true}}, {"no_event": {"type": "tool_call_finished", "name": "attempt_completion", "is_error": false}}, {"final_text_contains": "Verification is still pending."}]}diff --git a/src/pr_publish.zig b/src/pr_publish.zigindex 347e2125..12697d1f 100644--- a/src/pr_publish.zig+++ b/src/pr_publish.zig@@ -208,7 +208,8 @@ pub fn decide(draft: bool, ev: Evidence) Decision {pub fn reason(decision: Decision, ev: Evidence) []const u8 {return switch (decision) {.allow => "ready",- .draft_only => "create as a draft, or disclose the base-reproduced failure in the PR body",+ // #1197: disclosure explains a blocker; it never permits non-draft publication (ADR 0120).+ .draft_only => "create as a draft: a failure that also happens on the base explains the blocker but does not permit non-draft publication",.block => if (claimOverreach(ev.body, ev.coverage))"behavior claim overreaches the committed regression (helper/one-separator coverage is not the changed dispatch path)"else if (!hasVerificationSection(ev.body))@@ -216,7 +217,7 @@ pub fn reason(decision: Decision, ev: Evidence) []const u8 {else if (ev.head_status == .pending)"non-draft publication blocked: the exact head SHA still has a pending branch run"else if (ev.head_status == .failed)- "non-draft publication blocked: the exact head SHA has an unexplained failed branch run"+ "non-draft publication blocked: the exact head SHA has a failed branch run; a failure that also happens on the base still blocks, so fix it or publish a draft"else"non-draft publication blocked: head readiness is unresolved",};