graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Review checkpoint case: the long review reads a different file each call
Rach Pradhan6a06c046be13
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex d74d2ed8..c63991e8 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -100,7 +100,7 @@{"id":"human-workflow-clarify-edit-recover-followup","why":"#106: a two-file task preserves user clarification, recovers from a failed edit, and handles a follow-up; independent checks reject a plausible but absent result.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"workspace_seed":{"normalize.py":"def normalize(text):\n return text.lower()\n","client.py":"def render(items):\n return \",\".join(items)\n","README.md":"Keep this documentation unchanged.\n"},"prompt":"Make render trim labels using the existing normalizer. Clarify whether to preserve case before editing. Keep the README unchanged, and verify the actual result.","controls_after_turn":[{"type":"answer","text":"Preserve case exactly; trim surrounding whitespace only."}],"second_prompt":"Now skip empty and whitespace-only labels too. Keep the earlier case behavior, and test that input lists stay unchanged.","script":[{"tool":"read_file","arguments":{"path":"normalize.py"}},{"tool":"read_file","arguments":{"path":"client.py"}},{"tool":"ask_user","arguments":{"question":"Should label case be preserved or normalized?","options":["Preserve case","Lowercase"]}},{"tool":"bash","arguments":{"command":"python3 -c 'from client import render; assert render([\" A \", \"B \"]) == \"A,B\"; print(\"render checks passed\")'"}},{"tool":"edit_file","arguments":{"path":"normalize.py","old_string":" return text.upper()","new_string":" return text.strip()"}},{"tool":"read_file","arguments":{"path":"normalize.py"}},{"tool":"edit_file","arguments":{"path":"normalize.py","old_string":" return text.lower()","new_string":" return text.strip()"}},{"tool":"edit_file","arguments":{"path":"client.py","old_string":"def render(items):\n return \",\".join(items)\n","new_string":"from normalize import normalize\n\ndef render(items):\n return \",\".join(normalize(item) for item in items)\n"}},{"tool":"bash","arguments":{"command":"python3 -c 'from client import render; assert render([\" A \", \"B \"]) == \"A,B\"; print(\"render checks passed\")'"}},{"text":"Both files now trim labels while preserving case. The render checks passed; README is unchanged."},{"tool":"read_file","arguments":{"path":"client.py"}},{"tool":"edit_file","arguments":{"path":"client.py","old_string":"from normalize import normalize\n\ndef render(items):\n return \",\".join(normalize(item) for item in items)\n","new_string":"from normalize import normalize\n\ndef render(items):\n return \",\".join(normalize(item) for item in items if normalize(item))\n"}},{"tool":"bash","arguments":{"command":"python3 -c 'from client import render; x=[\" A \",\" \",\"B\",\"\"]; before=x.copy(); assert render(x)==\"A,B\"; assert x==before; assert render([])==\"\"; print(\"follow-up checks passed\")'"}},{"text":"Blank labels are skipped. Case and input lists are preserved; follow-up checks passed."}],"verify_command":["python3","-c","from client import render; from normalize import normalize; x=[\" Ab \",\" \",\"cD\",\"\"]; before=x.copy(); assert render(x)==\"Ab,cD\"; assert x==before; assert render([])==\"\"; assert normalize(\" X \")==\"X\"; print(\"independent outcomes passed\")"],"assert":[{"event":{"type":"ask_user","question":"Should label case be preserved or normalized?"}},{"event":{"type":"tool_result","name":"ask_user","is_error":false,"text":"Preserve case exactly; trim surrounding whitespace only."}},{"event":{"type":"tool_call_finished","name":"edit_file","is_error":true}},{"event":{"type":"tool_call_finished","name":"bash","is_error":true}},{"events_at_least":{"match":{"type":"turn"},"count":2}},{"request_contains":{"text":"Preserve case exactly"}},{"final_text_contains":"follow-up checks passed"},{"file_equals":{"path":"normalize.py","content":"def normalize(text):\n return text.strip()\n"}},{"file_equals":{"path":"client.py","content":"from normalize import normalize\n\ndef render(items):\n return \",\".join(normalize(item) for item in items if normalize(item))\n"}},{"file_equals":{"path":"README.md","content":"Keep this documentation unchanged.\n"}},{"request_contains":{"index":3,"text":"Preserve case exactly"}},{"request_tool_calls_at_most":{"index":3,"name":"edit_file","count":0}}]}{"id":"human-workflow-missing-config-clarify-followup","why":"#106: failed navigation cannot become an invented setting; a human supplies the actual path, then a later requested edit preserves unrelated configuration.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"workspace_seed":{"config/settings.json":"{\"retries\": 3, \"timeout\": 10}\n","README.md":"Do not change me.\n"},"prompt":"Check settings.json for the retry count. If it is missing, ask me where it lives; do not create a replacement or change anything yet.","controls_after_turn":[{"type":"answer","text":"It is config/settings.json. Read that existing file."}],"second_prompt":"Raise retries to 4 in that file, preserve timeout, and validate the JSON.","script":[{"tool":"read_file","arguments":{"path":"settings.json"}},{"tool":"ask_user","arguments":{"question":"Where is the existing settings file?"}},{"tool":"read_file","arguments":{"path":"config/settings.json"}},{"text":"The existing retry count is 3. No files were changed."},{"tool":"edit_file","arguments":{"path":"config/settings.json","old_string":"\"retries\": 3","new_string":"\"retries\": 4"}},{"tool":"bash","arguments":{"command":"python3 -c 'import json; x=json.load(open(\"config/settings.json\")); assert x=={\"retries\":4,\"timeout\":10}; print(\"configuration validated\")'"}},{"text":"Retries are 4; timeout remains 10. Configuration validated."}],"verify_command":["python3","-c","import json,pathlib; assert json.loads(pathlib.Path(\"config/settings.json\").read_text())=={\"retries\":4,\"timeout\":10}; assert not pathlib.Path(\"settings.json\").exists(); print(\"independent configuration passed\")"],"assert":[{"event":{"type":"tool_call_finished","name":"read_file","is_error":true}},{"event":{"type":"ask_user","question":"Where is the existing settings file?"}},{"events_at_least":{"match":{"type":"turn"},"count":2}},{"no_event":{"type":"tool_call","name":"write_file"}},{"request_contains":{"text":"Read that existing file"}},{"final_text_contains":"timeout remains 10"},{"file_equals":{"path":"config/settings.json","content":"{\"retries\": 4, \"timeout\": 10}\n"}},{"file_equals":{"path":"README.md","content":"Do not change me.\n"}},{"request_contains":{"index":2,"text":"config/settings.json. Read that existing file."}},{"request_tool_calls_at_most":{"index":2,"name":"edit_file","count":0}}]}{"id": "current-head-receipt-names-actual-checks", "why": "#853: preserve the actual named CI outcomes; local schema success cannot replace failing packed SDK evidence.", "args": ["--old", "--goal", "Deliver a PR with passing current-head CI"], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "github_fixture": {"runs": "none", "check_rollup": [{"name": "schema drift", "status": "COMPLETED", "conclusion": "SUCCESS"}, {"name": "packed SDK", "status": "COMPLETED", "conclusion": "FAILURE"}, {"context": "platform contract", "state": "PENDING"}], "review_files": ["notes.md"]}, "prompt": "Run the local publication fixture and fulfill the objective.", "script": [{"tool": "todo_write", "arguments": {"todos": [{"content": "inspect exact-head CI", "status": "pending"}]}}, {"tool": "bash", "arguments": {"command": "gh pr create --title fixture --body-file notes.md"}}, {"text": "{\"verdict\": \"supported\", \"reason\": \"Controlled committed fixture review.\"}"}, {"tool": "todo_write", "arguments": {"todos": [{"content": "provide draft handoff", "status": "completed"}]}}, {"tool": "attempt_completion", "arguments": {"result": "Draft handoff; the base checks also fail."}}, {"tool": "attempt_completion", "arguments": {"result": "Draft handoff; the base checks also fail."}}, {"text": "CI is unresolved."}], "assert": [{"events_at_least": {"match": {"type": "tool_call_finished", "name": "attempt_completion", "is_error": true}, "count": 2}}, {"no_event": {"type": "tool_call_finished", "name": "attempt_completion", "is_error": false}}, {"request_contains": {"text": "failed: CI reported a failure"}}, {"request_contains": {"text": "schema drift"}}, {"request_contains": {"text": "packed SDK"}}, {"request_contains": {"text": "platform contract"}}]}-{"id":"review-checkpoint-preserves-unfinished-scope","why":"#306: a long read-only review requests an evidence checkpoint without forcing completion or granting edits.","args":["--old"],"prompt_options":{"type":"review"},"prompt":"Review target.txt without modifying it.","workspace_seed":{"target.txt":"original\n"},"script":[{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"tool":"read_file","arguments":{"path":"target.txt"}},{"text":"No findings. Review finished."}],"assert":[{"events_at_least":{"match":{"type":"review_checkpoint","complete":false},"count":1}},{"events_at_most":{"match":{"type":"review_checkpoint"},"count":1}},{"request_contains":{"text":"Review checkpoint:"}},{"request_contains":{"text":"next inspection tool call"}},{"file_equals":{"path":"target.txt","content":"original\n"}},{"final_text_contains":"No findings. Review finished."}]}+{"id": "review-checkpoint-preserves-unfinished-scope", "why": "#306: a long read-only review requests an evidence checkpoint without forcing completion or granting edits. The long review reads a different note each call: one identical read repeated would now end the turn at the #1706 identical-call guard.", "args": ["--old"], "prompt_options": {"type": "review"}, "prompt": "Review target.txt without modifying it.", "workspace_seed": {"target.txt": "original\n", "notes/note-01.txt": "review note 1\n", "notes/note-02.txt": "review note 2\n", "notes/note-03.txt": "review note 3\n", "notes/note-04.txt": "review note 4\n", "notes/note-05.txt": "review note 5\n", "notes/note-06.txt": "review note 6\n", "notes/note-07.txt": "review note 7\n", "notes/note-08.txt": "review note 8\n", "notes/note-09.txt": "review note 9\n", "notes/note-10.txt": "review note 10\n", "notes/note-11.txt": "review note 11\n", "notes/note-12.txt": "review note 12\n", "notes/note-13.txt": "review note 13\n", "notes/note-14.txt": "review note 14\n", "notes/note-15.txt": "review note 15\n", "notes/note-16.txt": "review note 16\n", "notes/note-17.txt": "review note 17\n", "notes/note-18.txt": "review note 18\n", "notes/note-19.txt": "review note 19\n"}, "script": [{"tool": "read_file", "arguments": {"path": "target.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-01.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-02.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-03.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-04.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-05.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-06.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-07.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-08.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-09.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-10.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-11.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-12.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-13.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-14.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-15.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-16.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-17.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-18.txt"}}, {"tool": "read_file", "arguments": {"path": "notes/note-19.txt"}}, {"text": "No findings. Review finished."}], "assert": [{"events_at_least": {"match": {"type": "review_checkpoint", "complete": false}, "count": 1}}, {"events_at_most": {"match": {"type": "review_checkpoint"}, "count": 1}}, {"request_contains": {"text": "Review checkpoint:"}}, {"request_contains": {"text": "next inspection tool call"}}, {"file_equals": {"path": "target.txt", "content": "original\n"}}, {"final_text_contains": "No findings. Review finished."}]}{"id": "same-turn-two-writes-land", "why": "Two write_file calls in one assistant message must both land. Serial mutating batches must not drop the second write; parallel must not race-lose it either.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "create a.txt and b.txt", "script": [{"tools": [{"tool": "write_file", "arguments": {"path": "a.txt", "content": "alpha\n"}}, {"tool": "write_file", "arguments": {"path": "b.txt", "content": "beta\n"}}]}, {"tool": "bash", "arguments": {"command": "grep -qx alpha a.txt && grep -qx beta b.txt && echo BOTH_LANDED"}}, {"text": "both files are in"}], "assert": [{"events_at_least": {"match": {"type": "tool_call", "name": "write_file"}, "count": 2}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 0}}, {"event": {"type": "tool_result", "name": "bash", "text": "BOTH_LANDED\n"}}, {"final_text_contains": "both files are in"}]}{"id": "same-turn-write-then-edit-same-file", "why": "write_file then edit_file on the same path in one assistant message used to race (#337). Serial batches make the edit see the write; a lost race fails old_string or missing file.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "create n.txt then change v1 to v2", "script": [{"tools": [{"tool": "write_file", "arguments": {"path": "n.txt", "content": "v1\n"}}, {"tool": "edit_file", "arguments": {"path": "n.txt", "old_string": "v1", "new_string": "v2"}}]}, {"tool": "bash", "arguments": {"command": "grep -qx v2 n.txt && echo EDIT_SAW_WRITE"}}, {"text": "the file now says v2"}], "assert": [{"event": {"type": "tool_call", "name": "write_file"}}, {"event": {"type": "tool_call", "name": "edit_file"}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 0}}, {"event": {"type": "tool_result", "name": "bash", "text": "EDIT_SAW_WRITE\n"}}, {"final_text_contains": "v2"}]}{"id": "parallel-fast-shell-batch-resumes", "why": "#1166: fast shell results must appear exactly once and reach the second model request before final completion and clean process exit. Repeated varied-size allocator stress lives in scripts/eval/parallel_shell_stress.py.", "args": [], "prompt": "run the three checks", "script": [{"tools": [{"tool": "bash", "arguments": {"command": "printf AAA"}}, {"tool": "bash", "arguments": {"command": "printf BBBBB"}}, {"tool": "bash", "arguments": {"command": "printf CCCCCCC"}}]}, {"text": "all three checks returned"}], "assert": [{"events_at_least": {"match": {"type": "tool_result", "name": "bash"}, "count": 3}}, {"events_at_most": {"match": {"type": "tool_call_finished", "name": "bash", "is_error": true}, "count": 0}}, {"request_contains": {"index": -1, "text": "AAA"}}, {"request_contains": {"index": -1, "text": "BBBBB"}}, {"request_contains": {"index": -1, "text": "CCCCCCC"}}, {"final_text_contains": "all three checks returned"}, {"events_at_most": {"match": {"type": "tool_result", "name": "bash"}, "count": 3}}, {"events_at_least": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_0", "is_error": false}, "count": 1}}, {"events_at_most": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_0", "is_error": false}, "count": 1}}, {"request_contains": {"index": 1, "text": "AAA"}}, {"events_at_least": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_1", "is_error": false}, "count": 1}}, {"events_at_most": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_1", "is_error": false}, "count": 1}}, {"request_contains": {"index": 1, "text": "BBBBB"}}, {"events_at_least": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_2", "is_error": false}, "count": 1}}, {"events_at_most": {"match": {"type": "tool_result", "name": "bash", "id": "call_1_2", "is_error": false}, "count": 1}}, {"request_contains": {"index": 1, "text": "CCCCCCC"}}]}