Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Stop retrying unavailable gateway models

justrach048cf167c91c
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonl
index 66f3a4b9..f5cc0cb6 100644
--- a/evals/harness_behavior.jsonl
+++ b/evals/harness_behavior.jsonl
@@ -134,3 +134,4 @@
{"id": "rlm-indexed-print-refuses-and-recovers", "why": "Unsupported Python-style result indexing must report an error without pretending to execute the nested host or executing later side effects; the supported text binding form recovers.", "args": ["--lean"], "prompt": "Run the scripted checks and report only observed results.", "script": [{"tool": "rlm", "arguments": {"code": "print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"tail-marker\", content=\"ran\")"}}, {"tool": "rlm", "arguments": {"code": "r = bash(\"printf VERIFIED\")\nprint(r)"}}, {"text": "Verified using the supported text result."}], "verify_command": ["python3", "-c", "from pathlib import Path; assert not Path(\"nested-marker\").exists(); assert not Path(\"tail-marker\").exists()"], "assert": [{"event": {"type": "tool_call_finished", "name": "rlm", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 1}}, {"request_contains": {"index": 1, "text": "host results are text"}}, {"event": {"type": "tool_result", "name": "rlm", "is_error": false, "text": "VERIFIED"}}, {"final_text_contains": "Verified using the supported text result."}]}
{"id": "gateway-failure-usage-unknown", "why": "A typed gateway failure with no usage must preserve the provider diagnostic and label totals as incomplete instead of implying a free zero-token call.", "args": ["-p", "summarize the fixture"], "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "gateway_error", "code": "request_failed", "message": "fixture request failed"}}], "expected_exit": 1, "assert": [{"stderr_contains": "gateway_error request_failed"}, {"stderr_contains": "known subtotal: 0 api call(s)"}, {"stderr_contains": "1 failed request attempt(s) without usage (tokens and cost unknown)"}]}
{"id": "gateway-retry-usage-remains-incomplete", "why": "A recovered retry retains known successful usage while the failed attempt remains explicitly unknown.", "prompt": "summarize the fixture", "script": [{"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"text": "recovered fixture"}], "assert": [{"event": {"type": "turn", "usage_complete": false, "unreported_failed_attempts": 1, "api_calls": 1}}, {"final_text_contains": "recovered fixture"}]}
+{"id":"terminal-model-unavailable-does-not-retry","why":"#1191: a structured model_unavailable error is terminal when the gateway cannot serve the chosen model; a retry would repeat the same failure. Keep the request id so support can find the failed call.","prompt":"Say hello","script":[{"stream_error":{"type":"service_unavailable","code":"model_unavailable","message":"This model is not available right now. Choose another model or contact Codegraff support with the request ID.","request_id":"cg_req_fixture"}}],"assert":[{"events_at_most":{"match":{"type":"model_call_started"},"count":1}},{"event":{"type":"error","message":"api error (service_unavailable model_unavailable): This model is not available right now. Choose another model or contact Codegraff support with the request ID. [cg_req_fixture]"}},{"no_event":{"type":"model_call_finished","ok":true}}]}
diff --git a/src/agent_gateway_retry.zig b/src/agent_gateway_retry.zig
index 1b45ceae..2211be76 100644
--- a/src/agent_gateway_retry.zig
+++ b/src/agent_gateway_retry.zig
@@ -102,6 +102,7 @@ pub const max_short_flake_retries: usize = 2;
pub fn isShortGatewayFlake(etype: []const u8, code: ?[]const u8, msg: []const u8) bool {
// Gateway 110-byte follow-up. etype is often invalid_request_error, which
// would otherwise hard-fail on the "invalid" needle. Auth/quota still die.
+ if (code) |c| if (std.mem.eql(u8, c, "model_unavailable")) return false;
if (isBodyParseRejection(msg)) return true;
const hard = [_][]const u8{ "invalid", "authentication", "unauthorized", "insufficient", "quota", "permission", "tool_choice", "not found" };
for (hard) |n| {
@@ -291,6 +292,7 @@ test "isShortGatewayFlake: internal/empty api_error retry; invalid/auth/quota do
try std.testing.expect(!isShortGatewayFlake("authentication_error", null, "invalid api key"));
try std.testing.expect(!isShortGatewayFlake("insufficient_quota", null, "You exceeded your current quota"));
try std.testing.expect(!isShortGatewayFlake("api_error", null, "model not found"));
+ try std.testing.expect(!isShortGatewayFlake("service_unavailable", "model_unavailable", "Codegraff cannot serve this model right now"));
try std.testing.expect(isShortGatewayFlake("invalid_request_error", null, "Body must be valid JSON"));
try std.testing.expect(isShortGatewayFlake("api_error", null, "Malformed JSON in request body"));
try std.testing.expect(!isShortGatewayFlake("invalid_request_error", null, "invalid prompt"));
diff --git a/src/trace.zig b/src/trace.zig
index 4d7b75d0..3a407a84 100644
--- a/src/trace.zig
+++ b/src/trace.zig
@@ -124,6 +124,22 @@ pub const Tracer = struct {
behavior.recordApiMetric(from_subagent, ms, req_bytes, resp_bytes, context_tokens, cache_read, is_error)
else
0;
+ if (is_error) {
+ // Failed calls did not report usage. Omit it rather than claiming
+ // an observed zero; request/response byte counts remain known.
+ self.write(.{
+ .t = self.elapsedMs(),
+ .ev = "api",
+ .turn = behavior_turn,
+ .agent = label,
+ .model = model,
+ .ms = ms,
+ .req_bytes = req_bytes,
+ .resp_bytes = resp_bytes,
+ .is_error = true,
+ });
+ return;
+ }
self.write(.{
.t = self.elapsedMs(),
.ev = "api",
@@ -500,6 +516,30 @@ test "writeJsonLine: one complete newline-terminated JSON record per call" {
}
}
+test "failed api trace leaves unavailable usage unknown, successful zero stays zero" {
+ const io = std.testing.io;
+ var aw: Io.Writer.Allocating = .init(std.testing.allocator);
+ defer aw.deinit();
+ var tracer: Tracer = .{
+ .io = io,
+ .gpa = std.testing.allocator,
+ .out = &aw.writer,
+ .start = Io.Timestamp.now(io, .awake),
+ };
+ tracer.api("repl", false, "model", 20, 100, 0, 0, 0, true);
+ tracer.api("repl", false, "model", 20, 100, 80, 0, 0, false);
+ var it = std.mem.splitScalar(u8, std.mem.trimEnd(u8, aw.writer.buffered(), "\n"), '\n');
+ var failure = try std.json.parseFromSlice(Value, std.testing.allocator, it.next().?, .{});
+ defer failure.deinit();
+ try std.testing.expect(failure.value.object.get("is_error").?.bool);
+ try std.testing.expect(failure.value.object.get("context_tokens") == null);
+ try std.testing.expect(failure.value.object.get("cache_read_tokens") == null);
+ var success = try std.json.parseFromSlice(Value, std.testing.allocator, it.next().?, .{});
+ defer success.deinit();
+ try std.testing.expectEqual(@as(i64, 0), success.value.object.get("context_tokens").?.integer);
+ try std.testing.expectEqual(@as(i64, 0), success.value.object.get("cache_read_tokens").?.integer);
+}
+
test "operational tool outcomes carry monotonic call ids without rich tracing (#602)" {
const io = std.testing.io;
var aw: Io.Writer.Allocating = .init(std.testing.allocator);