graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Keep unavailable-model errors terminal across retry gates
Rach Pradhana5a23d56b8bc
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex 9813a93a..8c053a31 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -135,6 +135,7 @@{"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]}{"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]}{"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"api error (service_unavailable model_unavailable): This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]}+{"id":"terminal-model-unavailable-outranks-retry-wording","why":"The structured unavailable-model code remains terminal even if gateway diagnostic text also contains transient retry wording.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"Gateway overloaded; this model is unavailable."}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"api error (service_unavailable model_unavailable): Gateway overloaded; this model is unavailable."}},{"no_event":{"type":"model_call_finished","ok":true}}]}{"id":"atomic-edit-batch-rejects-later-span","why":"A later ambiguous edit span must reject the entire batch without leaving earlier spans on disk; a corrected dependent batch then commits once and verifies.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"workspace_seed":{"f.txt":"alpha beta\nreturn v\nreturn v\n"},"prompt":"Apply the three edits to f.txt, check that a rejected batch left the file unchanged, then correct the ambiguous span and verify the final contents.","script":[{"tool":"edit_file","arguments":{"path":"f.txt","edits":[{"old_string":"alpha","new_string":"ALPHA"},{"old_string":"beta","new_string":"BETA"},{"old_string":"return v","new_string":"return value"}]}},{"tool":"bash","arguments":{"command":"python3 -c \"from pathlib import Path; assert Path('f.txt').read_text() == 'alpha beta\\nreturn v\\nreturn v\\n'; print('UNCHANGED_AFTER_REJECTION')\""}},{"tool":"edit_file","arguments":{"path":"f.txt","edits":[{"old_string":"alpha","new_string":"ALPHA"},{"old_string":"beta","new_string":"BETA"},{"old_string":"return v\nreturn v","new_string":"return value\nreturn value"}]}},{"tool":"bash","arguments":{"command":"python3 -c \"from pathlib import Path; assert Path('f.txt').read_text() == 'ALPHA BETA\\nreturn value\\nreturn value\\n'; print('CORRECTED_BATCH_VERIFIED')\""}},{"text":"The corrected batch passed both file checks."}],"assert":[{"event":{"type":"tool_call_finished","name":"edit_file","is_error":true}},{"event":{"type":"tool_call_finished","name":"edit_file","is_error":false}},{"events_at_most":{"match":{"type":"tool_call_finished","name":"edit_file","is_error":true},"count":1}},{"request_contains":{"index":1,"text":"edit span 3/3 failed"}},{"request_contains":{"index":1,"text":"no batch changes written"}},{"event":{"type":"tool_result","name":"bash","text":"UNCHANGED_AFTER_REJECTION\n"}},{"event":{"type":"tool_result","name":"bash","text":"CORRECTED_BATCH_VERIFIED\n"}},{"file_equals":{"path":"f.txt","content":"ALPHA BETA\nreturn value\nreturn value\n"}},{"final_text_contains":"The corrected batch passed both file checks."}]}{"id":"large-log-preview-early-omitted-error","why":"A spilled tool result must surface useful omitted error lines in its first model request without an extra pager call.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"prompt":"Run the diagnostic report and summarize its errors.","script":[{"tool":"shell","arguments":{"command":"python3 -c '[print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(24)]; print('\"'\"'ERROR '\"'\"'+ '\"'\"'EARLY_'\"'\"'+'\"'\"'GAP_VISIBLE'\"'\"'); [print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(24,625)]'"}},{"text":"Report reviewed."}],"assert":[{"request_count":2},{"request_contains":{"index":1,"text":"ERROR EARLY_GAP_VISIBLE"}},{"request_contains":{"index":1,"text":"tool result handle"}},{"final_text_contains":"Report reviewed."}]}{"id":"large-log-preview-short-error-after-long-line","why":"A spilled tool result must surface useful omitted error lines in its first model request without an extra pager call.","args":["--old"],"env":{"GRAFF_NO_CODEDB_GUARD":"1"},"prompt":"Run the diagnostic report and summarize its errors.","script":[{"tool":"shell","arguments":{"command":"python3 -c '[print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(100)]; print('\"'\"'ERROR '\"'\"'+ '\"'\"'z'\"'\"'*2500); print('\"'\"'ERROR '\"'\"'+ '\"'\"'SHORT_'\"'\"'+'\"'\"'DETAIL_VISIBLE'\"'\"'); [print('\"'\"'ok %04d '\"'\"'%i+'\"'\"'x'\"'\"'*32) for i in range(100,700)]'"}},{"text":"Report reviewed."}],"assert":[{"request_count":2},{"request_contains":{"index":1,"text":"ERROR SHORT_DETAIL_VISIBLE"}},{"request_contains":{"index":1,"text":"tool result handle"}},{"final_text_contains":"Report reviewed."}]}diff --git a/src/agent_gateway_retry.zig b/src/agent_gateway_retry.zigindex 2211be76..6525c307 100644--- a/src/agent_gateway_retry.zig+++ b/src/agent_gateway_retry.zig@@ -88,6 +88,10 @@ pub fn noteFlake(self: *Agent, state: *GatewayRetryState, err: anyerror) void {/// ladder below), then a short Codegraff follow-up flake (ADR 0053 DeepSeek/// `-j 6` 110-byte / ~450ms `api_error`), then the timeout-gated body-parse retry.pub fn afterServerErrorOrParseReject(self: *Agent, etype: []const u8, code: ?[]const u8, msg: []const u8, server_retries: *usize, state: *GatewayRetryState) !bool {+ // A structured terminal code outranks transient wording in the message.+ // In particular, do not retry a model the gateway cannot serve even if+ // its diagnostic also mentions capacity or a request-body parse error.+ if (isModelUnavailable(code)) return false;if (try retryTransientServerError(self, etype, code, msg, server_retries)) return true;if (try retryShortGatewayFlake(self, etype, code, msg, server_retries)) return true;return retryBodyParseAfterTimeouts(self, msg, state);@@ -95,6 +99,10 @@ pub fn afterServerErrorOrParseReject(self: *Agent, etype: []const u8, code: ?[]cpub const max_short_flake_retries: usize = 2;+fn isModelUnavailable(code: ?[]const u8) bool {+ return if (code) |c| std.mem.eql(u8, c, "model_unavailable") else false;+}+/// Tiny / generic envelopes that are not a real client bug. The DeepSeek/// SWE `-j 6` follow-up was ~110 bytes, ~450 ms, `is_error`, no/// "overloaded" / "server_error" needle — isolated serial retry passed.@@ -102,7 +110,7 @@ pub const max_short_flake_retries: usize = 2;pub fn isShortGatewayFlake(etype: []const u8, code: ?[]const u8, msg: []const u8) bool {// Gateway 110-byte follow-up. etype is often invalid_request_error, which// would otherwise hard-fail on the "invalid" needle. Auth/quota still die.- if (code) |c| if (std.mem.eql(u8, c, "model_unavailable")) return false;+ if (isModelUnavailable(code)) return false;if (isBodyParseRejection(msg)) return true;const hard = [_][]const u8{ "invalid", "authentication", "unauthorized", "insufficient", "quota", "permission", "tool_choice", "not found" };for (hard) |n| {@@ -170,6 +178,7 @@ pub const max_server_retries: usize = 3; // #opencode-parity: bounded retries fo/// surfaced mid-stream and should be retried, not hard-failed. Billing / quota //// invalid-input errors are NOT transient and fall through to a hard fail.pub fn isTransientServerError(etype: []const u8, code: ?[]const u8, msg: []const u8) bool {+ if (isModelUnavailable(code)) return false;const needles = [_][]const u8{ "overloaded", "server_error", "server_is_overloaded", "token parsing" };for (needles) |n| {if (util.indexOfIgnoreCase(etype, n) != null) return true;@@ -293,6 +302,7 @@ test "isShortGatewayFlake: internal/empty api_error retry; invalid/auth/quota dotry std.testing.expect(!isShortGatewayFlake("insufficient_quota", null, "You exceeded your current quota"));try std.testing.expect(!isShortGatewayFlake("api_error", null, "model not found"));try std.testing.expect(!isShortGatewayFlake("service_unavailable", "model_unavailable", "Codegraff cannot serve this model right now"));+ try std.testing.expect(!isShortGatewayFlake("service_unavailable", "model_unavailable", "Malformed JSON in request body"));try std.testing.expect(isShortGatewayFlake("invalid_request_error", null, "Body must be valid JSON"));try std.testing.expect(isShortGatewayFlake("api_error", null, "Malformed JSON in request body"));try std.testing.expect(!isShortGatewayFlake("invalid_request_error", null, "invalid prompt"));@@ -308,6 +318,7 @@ test "isTransientServerError (#opencode-parity): overload/server_error retry; qutry std.testing.expect(!isTransientServerError("insufficient_quota", "insufficient_quota", "You exceeded your current quota"));try std.testing.expect(!isTransientServerError("invalid_request_error", null, "invalid prompt"));try std.testing.expect(!isTransientServerError("authentication_error", null, "invalid api key"));+ try std.testing.expect(!isTransientServerError("service_unavailable", "model_unavailable", "server overloaded; retry after 3 seconds"));// a bare "Internal Server Error" stays on the shorter gateway-flake laddertry std.testing.expect(!isTransientServerError("api_error", null, "Internal Server Error"));}