Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Merge current release tool-preview diagnostics

justrachcbc978f6966a
diff --cc evals/harness_behavior.jsonl
index f3289e4e,83d6dec6..9813a93a
--- a/evals/harness_behavior.jsonl
+++ b/evals/harness_behavior.jsonl
@@@ -134,5 -134,6 +134,7 @@@
{"id": "rlm-indexed-print-refuses-and-recovers", "why": "Unsupported Python-style result indexing must report an error without pretending to execute the nested host or executing later side effects; the supported text binding form recovers.", "args": ["--lean"], "prompt": "Run the scripted checks and report only observed results.", "script": [{"tool": "rlm", "arguments": {"code": "print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"tail-marker\", content=\"ran\")"}}, {"tool": "rlm", "arguments": {"code": "r = bash(\"printf VERIFIED\")\nprint(r)"}}, {"text": "Verified using the supported text result."}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path(\"nested-marker\").exists(); assert not Path(\"tail-marker\").exists()"], "assert": [{"event": {"type": "tool_call_finished", "name": "rlm", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 1}}, {"request_contains": {"index": 1, "text": "host results are text"}}, {"event": {"type": "tool_result", "name": "rlm", "is_error": false, "text": "VERIFIED"}}, {"final_text_contains": "Verified using the supported text result."}]}
{"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]}
{"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]}
+{"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"api error (service_unavailable model_unavailable): This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]}
{"id":"atomic-edit-batch-rejects-later-span","why":"A later ambiguous edit span must reject the entire batch without leaving earlier spans on disk; a corrected dependent batch then commits once and verifies.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"workspace_seed":{"f.txt":"alpha beta\nreturn v\nreturn v\n"},"prompt":"Apply the three edits to f.txt, check that a rejected batch left the file unchanged, then correct the ambiguous span and verify the final contents.","script":[{"tool":"edit_file","arguments":{"path":"f.txt","edits":[{"old_string":"alpha","new_string":"ALPHA"},{"old_string":"beta","new_string":"BETA"},{"old_string":"return v","new_string":"return value"}]}},{"tool":"bash","arguments":{"command":"python3 -c \"from pathlib import Path; assert Path('f.txt').read_text() == 'alpha beta\\nreturn v\\nreturn v\\n'; print('UNCHANGED_AFTER_REJECTION')\""}},{"tool":"edit_file","arguments":{"path":"f.txt","edits":[{"old_string":"alpha","new_string":"ALPHA"},{"old_string":"beta","new_string":"BETA"},{"old_string":"return v\nreturn v","new_string":"return value\nreturn value"}]}},{"tool":"bash","arguments":{"command":"python3 -c \"from pathlib import Path; assert Path('f.txt').read_text() == 'ALPHA BETA\\nreturn value\\nreturn value\\n'; print('CORRECTED_BATCH_VERIFIED')\""}},{"text":"The corrected batch passed both file checks."}],"assert":[{"event":{"type":"tool_call_finished","name":"edit_file","is_error":true}},{"event":{"type":"tool_call_finished","name":"edit_file","is_error":false}},{"events_at_most":{"match":{"type":"tool_call_finished","name":"edit_file","is_error":true},"count":1}},{"request_contains":{"index":1,"text":"edit span 3/3 failed"}},{"request_contains":{"index":1,"text":"no batch changes written"}},{"event":{"type":"tool_result","name":"bash","text":"UNCHANGED_AFTER_REJECTION\n"}},{"event":{"type":"tool_result","name":"bash","text":"CORRECTED_BATCH_VERIFIED\n"}},{"file_equals":{"path":"f.txt","content":"ALPHA BETA\nreturn value\nreturn value\n"}},{"final_text_contains":"The corrected batch passed both file checks."}]}
+ {"id":"large-log-preview-early-omitted-error","why":"A spilled tool result must surface useful omitted error lines in its first model request without an extra pager call.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"prompt":"Run the diagnostic report and summarize its errors.","script":[{"tool":"shell","arguments":{"command":"python3 -c '[print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(24)]; print('\"'\"'ERROR '\"'\"'+ '\"'\"'EARLY_'\"'\"'+'\"'\"'GAP_VISIBLE'\"'\"'); [print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(24,625)]'"}},{"text":"Report reviewed."}],"assert":[{"request_count":2},{"request_contains":{"index":1,"text":"ERROR EARLY_GAP_VISIBLE"}},{"request_contains":{"index":1,"text":"tool result handle"}},{"final_text_contains":"Report reviewed."}]}
+ {"id":"large-log-preview-short-error-after-long-line","why":"A spilled tool result must surface useful omitted error lines in its first model request without an extra pager call.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"prompt":"Run the diagnostic report and summarize its errors.","script":[{"tool":"shell","arguments":{"command":"python3 -c '[print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(100)]; print('\"'\"'ERROR '\"'\"'+ '\"'\"'z'\"'\"'*2500); print('\"'\"'ERROR '\"'\"'+ '\"'\"'SHORT_'\"'\"'+'\"'\"'DETAIL_VISIBLE'\"'\"'); [print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(100,700)]'"}},{"text":"Report reviewed."}],"assert":[{"request_count":2},{"request_contains":{"index":1,"text":"ERROR SHORT_DETAIL_VISIBLE"}},{"request_contains":{"index":1,"text":"tool result handle"}},{"final_text_contains":"Report reviewed."}]}