graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Stop retrying unavailable gateway models
justrach048cf167c91c
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex 66f3a4b9..f5cc0cb6 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -134,3 +134,4 @@{"id": "rlm-indexed-print-refuses-and-recovers", "why": "Unsupported Python-style result indexing must report an error without pretending to execute the nested host or executing later side effects; the supported text binding form recovers.", "args": ["--lean"], "prompt": "Run the scripted checks and report only observed results.", "script": [{"tool": "rlm", "arguments": {"code": "print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"tail-marker\", content=\"ran\")"}}, {"tool": "rlm", "arguments": {"code": "r = bash(\"printf VERIFIED\")\nprint(r)"}}, {"text": "Verified using the supported text result."}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path(\"nested-marker\").exists(); assert not Path(\"tail-marker\").exists()"], "assert": [{"event": {"type": "tool_call_finished", "name": "rlm", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 1}}, {"request_contains": {"index": 1, "text": "host results are text"}}, {"event": {"type": "tool_result", "name": "rlm", "is_error": false, "text": "VERIFIED"}}, {"final_text_contains": "Verified using the supported text result."}]}{"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]}{"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]}+{"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"api error (service_unavailable model_unavailable): This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]}diff --git a/src/agent_gateway_retry.zig b/src/agent_gateway_retry.zigindex 1b45ceae..2211be76 100644--- a/src/agent_gateway_retry.zig+++ b/src/agent_gateway_retry.zig@@ -102,6 +102,7 @@ pub const max_short_flake_retries: usize = 2;pub fn isShortGatewayFlake(etype: []const u8, code: ?[]const u8, msg: []const u8) bool {// Gateway 110-byte follow-up. etype is often invalid_request_error, which// would otherwise hard-fail on the "invalid" needle. Auth/quota still die.+ if (code) |c| if (std.mem.eql(u8, c, "model_unavailable")) return false;if (isBodyParseRejection(msg)) return true;const hard = [_][]const u8{ "invalid", "authentication", "unauthorized", "insufficient", "quota", "permission", "tool_choice", "not found" };for (hard) |n| {@@ -291,6 +292,7 @@ test "isShortGatewayFlake: internal/empty api_error retry; invalid/auth/quota dotry std.testing.expect(!isShortGatewayFlake("authentication_error", null, "invalid api key"));try std.testing.expect(!isShortGatewayFlake("insufficient_quota", null, "You exceeded your current quota"));try std.testing.expect(!isShortGatewayFlake("api_error", null, "model not found"));+ try std.testing.expect(!isShortGatewayFlake("service_unavailable", "model_unavailable", "Codegraff cannot serve this model right now"));try std.testing.expect(isShortGatewayFlake("invalid_request_error", null, "Body must be valid JSON"));try std.testing.expect(isShortGatewayFlake("api_error", null, "Malformed JSON in request body"));try std.testing.expect(!isShortGatewayFlake("invalid_request_error", null, "invalid prompt"));diff --git a/src/trace.zig b/src/trace.zigindex 4d7b75d0..3a407a84 100644--- a/src/trace.zig+++ b/src/trace.zig@@ -124,6 +124,22 @@ pub const Tracer = struct {behavior.recordApiMetric(from_subagent, ms, req_bytes, resp_bytes, context_tokens, cache_read, is_error)else0;+ if (is_error) {+ // Failed calls did not report usage. Omit it rather than claiming+ // an observed zero; request/response byte counts remain known.+ self.write(.{+ .t = self.elapsedMs(),+ .ev = "api",+ .turn = behavior_turn,+ .agent = label,+ .model = model,+ .ms = ms,+ .req_bytes = req_bytes,+ .resp_bytes = resp_bytes,+ .is_error = true,+ });+ return;+ }self.write(.{.t = self.elapsedMs(),.ev = "api",@@ -500,6 +516,30 @@ test "writeJsonLine: one complete newline-terminated JSON record per call" {}}+test "failed api trace leaves unavailable usage unknown, successful zero stays zero" {+ const io = std.testing.io;+ var aw: Io.Writer.Allocating = .init(std.testing.allocator);+ defer aw.deinit();+ var tracer: Tracer = .{+ .io = io,+ .gpa = std.testing.allocator,+ .out = &aw.writer,+ .start = Io.Timestamp.now(io, .awake),+ };+ tracer.api("repl", false, "model", 20, 100, 0, 0, 0, true);+ tracer.api("repl", false, "model", 20, 100, 80, 0, 0, false);+ var it = std.mem.splitScalar(u8, std.mem.trimEnd(u8, aw.writer.buffered(), "\n"), '\n');+ var failure = try std.json.parseFromSlice(Value, std.testing.allocator, it.next().?, .{});+ defer failure.deinit();+ try std.testing.expect(failure.value.object.get("is_error").?.bool);+ try std.testing.expect(failure.value.object.get("context_tokens") == null);+ try std.testing.expect(failure.value.object.get("cache_read_tokens") == null);+ var success = try std.json.parseFromSlice(Value, std.testing.allocator, it.next().?, .{});+ defer success.deinit();+ try std.testing.expectEqual(@as(i64, 0), success.value.object.get("context_tokens").?.integer);+ try std.testing.expectEqual(@as(i64, 0), success.value.object.get("cache_read_tokens").?.integer);+}+test "operational tool outcomes carry monotonic call ids without rich tracing (#602)" {const io = std.testing.io;var aw: Io.Writer.Allocating = .init(std.testing.allocator);