graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Add bounded jitter to every retry backoff (#1274)
Rach Pradhana61f3571092a
diff --git a/scripts/test-acp-http2-retry.py b/scripts/test-acp-http2-retry.pyindex c5e86b9b..f80e8025 100644--- a/scripts/test-acp-http2-retry.py+++ b/scripts/test-acp-http2-retry.py@@ -116,9 +116,11 @@ def exercise(root, binary):result, updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Say OK.'}]}, 3)assert result.get('result', {}).get('stopReason') == 'end_turn', resultreply_chunks = chunks(updates)- assert len(reply_chunks) == 2 and re.fullmatch(- r'\[network error: EndOfStream — retrying in 250ms \(1/6\)\]', reply_chunks[0]- ) and reply_chunks[1] == 'OK', reply_chunks+ # #1274: the 250 ms first flake step is jittered upward by at most 25%.+ retry_line = re.fullmatch(+ r'\[network error: EndOfStream — retrying in (\d+)ms \(1/6\)\]', reply_chunks[0]+ ) if len(reply_chunks) == 2 else None+ assert retry_line and 250 <= int(retry_line.group(1)) <= 312 and reply_chunks[1] == 'OK', reply_chunksrows = [json.loads(line) for line in log.read_text().splitlines()]posts = [row for row in rows if row['event'] == 'post']assert len(posts) == 2, postsdiff --git a/src/agent_gateway_retry.zig b/src/agent_gateway_retry.zigindex 6525c307..6d45d58b 100644--- a/src/agent_gateway_retry.zig+++ b/src/agent_gateway_retry.zig@@ -18,13 +18,14 @@//! fail-fast.//! 4. The transient-server ladder (overloaded / server_error, and xAI's//! mid-stream "Internal error during token parsing" — ADR 0148): a 5xx that-//! surfaced in-band is retried 3× with 1·2·4 s backoff, never treated as+//! surfaced in-band is retried 3× with 1·2·4 s (+jitter) backoff, never treated as//! context overflow. Moved here from agent_request_policy.zig (600-line cap).const std = @import("std");const Agent = @import("agent.zig").Agent;const http = @import("http.zig");const RetryPlan = http.RetryPlan;+const jitter = @import("retry_jitter.zig");const util = @import("util.zig");const policy = @import("agent_request_policy.zig");const telemetry = @import("telemetry.zig");@@ -149,7 +150,7 @@ pub fn retryDegenerateBody(self: *Agent, body: []const u8, retries: *usize) !booif (retries.* >= max_short_flake_retries) return false;retries.* += 1;self.partial_text.clearRetainingCapacity();- const delay_ms = RetryPlan.delayMs(true, retries.* - 1);+ const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(true, retries.* - 1));const what: []const u8 = if (policy.sseKeepAliveOnly(body)) "keep-alive only, no tokens" else "truncated gateway body";try self.say("[provider queued the request ({s}) — retrying in {d}s ({d}/{d})]\n", .{ what, delay_ms / 1000, retries.*, max_short_flake_retries });if (self.tracer) |tr| tr.note("retry", what);@@ -162,7 +163,7 @@ fn retryShortGatewayFlake(self: *Agent, etype: []const u8, code: ?[]const u8, msif (retries.* >= max_short_flake_retries) return false;retries.* += 1;self.partial_text.clearRetainingCapacity();- const delay_ms = RetryPlan.delayMs(true, retries.* - 1);+ const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(true, retries.* - 1));try self.say("[gateway flake — retrying in {d}s ({d}/{d})]\n", .{ delay_ms / 1000, retries.*, max_short_flake_retries });if (self.tracer) |tr| tr.note("retry", "short gateway flake");self.sleepInterruptible(delay_ms) catch return error.Interrupted;@@ -243,7 +244,7 @@ pub fn retryTransientServerError(self: *Agent, etype: []const u8, code: ?[]constif (retries.* >= max_server_retries) return false;retries.* += 1;self.partial_text.clearRetainingCapacity(); // fresh re-stream after the retry, no concat- const delay_ms = serverRetryDelayMs(msg) orelse RetryPlan.delayMs(true, retries.* - 1); // 1·2·4s, or the provider's wait+ const delay_ms = jitter.ms(self.io, serverRetryDelayMs(msg) orelse RetryPlan.delayMs(true, retries.* - 1)); // 1·2·4s, or the provider's wait; +jitter (#1274)const label = transientServerLabel(msg);try announceTransientRetry(self, label, delay_ms, retries.*);if (self.tracer) |tr| tr.note("retry", label);@@ -259,7 +260,7 @@ fn retryBodyParseAfterTimeouts(self: *Agent, msg: []const u8, state: *GatewayRetif (!shouldRetryBodyParseAfterTimeouts(msg, state.transport_timeouts, state.parse_retries)) return false;state.parse_retries += 1;self.partial_text.clearRetainingCapacity(); // fresh re-stream after the retry, no concat- const delay_ms = RetryPlan.delayMs(true, state.parse_retries - 1); // 1·2s+ const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(true, state.parse_retries - 1)); // 1·2stry self.say("[gateway answered after {d} timeouts with a body-parse rejection — retrying in {d}s ({d}/{d})]\n", .{ state.transport_timeouts, delay_ms / 1000, state.parse_retries, max_gateway_parse_retries });if (self.tracer) |tr| tr.note("retry", "body-parse rejection after transport timeouts (gateway artifact?)");self.sleepInterruptible(delay_ms) catch return error.Interrupted;diff --git a/src/agent_request.zig b/src/agent_request.zigindex b02cb3fa..c538d804 100644--- a/src/agent_request.zig+++ b/src/agent_request.zig@@ -65,6 +65,7 @@ const codex_chain = @import("codex_chain.zig");const req_stats = @import("req_stats.zig"); // GRAFF_REQ_STATS anatomy (session_settings arms req_stats.g_armed)const scratch = @import("agent_request_scratch.zig");+const jitter = @import("retry_jitter.zig"); // #1274: every computed backoff is jittered upward/// #390 — appended once, on the run's final admitted model call, right where/// the tools disappear, so the model knows WHY and lands instead of retrying.@@ -276,7 +277,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {if (telemetry.g_telem) |t| t.errorEvent("stream_retry", what);@import("agent_stream.zig").noteStallRetry(self, stall_retries); // #680: clears the partial (fresh stream, no concat) + widens the between-lines budgetself.closeCodexWs(); // tear down the dead WS + null codex_prev_id for a full re-send (no-op off codex WS)- self.sleepInterruptible(stall_reconnect_backoff_ms) catch return error.Interrupted;+ self.sleepInterruptible(jitter.ms(self.io, stall_reconnect_backoff_ms)) catch return error.Interrupted; // #1274: desync parallel reconnectscontinue :rebuild;}// Budget gone: the turn is ending. Mid-turn cuts stayed silent.@@ -306,7 +307,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {openai404_retries += 1;try self.say("[openai 404 (often spurious) — retrying ({d}/{d})]\n", .{ openai404_retries, max_openai404_retries });if (self.tracer) |tr| tr.note("retry", "openai 404");- self.sleepInterruptible(RetryPlan.delayMs(false, openai404_retries - 1)) catch return error.Interrupted;+ self.sleepInterruptible(jitter.ms(self.io, RetryPlan.delayMs(false, openai404_retries - 1))) catch return error.Interrupted;continue;}self.last_api_error = std.fmt.allocPrint(self.arena, "openai 404 (model not found?): {s}", .{if (main_mod.g_5xx_body_len > 0) main_mod.g_5xx_body_buf[0..main_mod.g_5xx_body_len] else "not found"}) catch "openai 404: model not found";@@ -345,9 +346,10 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {if (throttled) {// #retry-after: prefer the provider's Retry-After// (429/503) over our computed backoff, capped — like- // opencode, so we wait exactly as long as the server asked.+ // opencode, so we wait at least as long as the server asked.+ // #1274: jitter only adds on top, so parallel agents desync.const server_ra = main_mod.g_retry_after_ms;- const delay_ms = if (server_ra > 0) server_ra else RetryPlan.delayMs(throttled, attempt);+ const delay_ms = jitter.ms(self.io, if (server_ra > 0) server_ra else RetryPlan.delayMs(throttled, attempt));const ra_note: []const u8 = if (server_ra > 0) " (server retry-after)" else "";const what: []const u8 = if (err == error.RateLimited) "rate limited (429)" else "server error (5xx)";// Never echo the captured 429/5xx body. Envelopes from@@ -359,7 +361,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {} else {// Transport flakes back off; rapid retries against a// just-closed keep-alive re-fail (#86). Cap: 4s/6 tries.- const delay_ms = RetryPlan.delayMs(throttled, attempt);+ const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(throttled, attempt));@import("turn_chrome.zig").emitRetryNotice(self.io, @errorName(err), attempt + 1, max_attempts);if (scratch.showRecoveredTransportRetry(self.call_kind))try scratch.announceNetworkRetry(self, err, delay_ms, attempt + 1, max_attempts);diff --git a/src/remote_control.zig b/src/remote_control.zigindex adcb61d4..787346b8 100644--- a/src/remote_control.zig+++ b/src/remote_control.zig@@ -31,6 +31,7 @@ const serve_create = @import("serve_create.zig");const events = @import("serve_events.zig");const upload = @import("remote_upload.zig");const util = @import("util.zig");+const jitter = @import("retry_jitter.zig");const ServeState = serve.ServeState;const ServeSession = serve.ServeSession;@@ -102,7 +103,7 @@ pub fn remoteControlMain(gpa: Allocator, io: Io, cfg: Config, exe: []const u8) !if (!registered) {register(&self, arena, label) catch |err| {serve.serveLog(io, "remote-control: register failed ({t}) — retrying in {d}s", .{ err, @divTrunc(backoff, 1000) });- io.sleep(.fromMilliseconds(backoff), .awake) catch return;+ io.sleep(.fromMilliseconds(@intCast(jitter.ms(io, @intCast(backoff)))), .awake) catch return;backoff = @min(backoff * 2, backoff_max_ms);continue;};@@ -114,7 +115,7 @@ pub fn remoteControlMain(gpa: Allocator, io: Io, cfg: Config, exe: []const u8) !const cmds = poll(&self, arena) catch |err| {if (err == error.UnknownDevice) registered = false;serve.serveLog(io, "remote-control: poll failed ({t}) — retrying in {d}s", .{ err, @divTrunc(backoff, 1000) });- io.sleep(.fromMilliseconds(backoff), .awake) catch return;+ io.sleep(.fromMilliseconds(@intCast(jitter.ms(io, @intCast(backoff)))), .awake) catch return;backoff = @min(backoff * 2, backoff_max_ms);continue;};diff --git a/src/retry_jitter.zig b/src/retry_jitter.zignew file mode 100644index 00000000..17efd954--- /dev/null+++ b/src/retry_jitter.zig@@ -0,0 +1,88 @@+//! Bounded retry jitter (#1274).+//!+//! Every retry ladder here (http.RetryPlan's 1·2·4·8 s throttle and+//! .25·.5·1·2·4 s flake steps, the gateway-flake ladder, the subagent re-ask,+//! the remote-control reconnect) used to be deterministic, so parallel+//! subagents or workflow items that hit the same rate limit slept the same+//! amount and retried in lockstep, colliding again. Each backoff now passes+//! through here once before it is slept.+//!+//! The rule: jitter only ADDS time, never more than `max_extra_pct` of the+//! base. A server-provided Retry-After / "try again in N" hint is passed in as+//! the base, so the jittered wait is always >= what the server asked for.+//! Worst case is 1.25x the old wait; the ladders' own caps are unchanged.++const std = @import("std");+const Io = std.Io;++/// Upper bound of the added wait, as a percentage of the base.+pub const max_extra_pct: u64 = 25;++/// Pure seam: `base_ms` plus a uniform extra in [0, base_ms * max_extra_pct / 100],+/// drawn from the caller's random word `r`. Never returns less than `base_ms`.+pub fn withJitter(base_ms: u64, r: u64) u64 {+ const span = base_ms / 100 * max_extra_pct + base_ms % 100 * max_extra_pct / 100;+ if (span == 0) return base_ms;+ return base_ms +| r % (span + 1);+}++/// Seeded-RNG form for tests and any caller that owns a std.Random.+pub fn withRandom(rand: std.Random, base_ms: u64) u64 {+ return withJitter(base_ms, rand.int(u64));+}++/// Production: jitter `base_ms` from the process entropy source.+pub fn ms(io: Io, base_ms: u64) u64 {+ var raw: [8]u8 = undefined;+ io.random(&raw);+ return withJitter(base_ms, std.mem.readInt(u64, &raw, .little));+}++test "withJitter (#1274): only adds, at most max_extra_pct of the base" {+ var prng = std.Random.DefaultPrng.init(0x1274);+ const rand = prng.random();+ for ([_]u64{ 250, 500, 750, 1000, 2000, 4000, 8000, 30_000 }) |base| {+ var i: usize = 0;+ while (i < 200) : (i += 1) {+ const d = withRandom(rand, base);+ try std.testing.expect(d >= base);+ try std.testing.expect(d <= base + base * max_extra_pct / 100);+ }+ }+ // The extremes of the random word land on the two bounds.+ try std.testing.expectEqual(@as(u64, 1000), withJitter(1000, 0));+ try std.testing.expectEqual(@as(u64, 1250), withJitter(1000, 250));+ // Nothing to spread: a zero or tiny base passes through untouched.+ try std.testing.expectEqual(@as(u64, 0), withJitter(0, 12345));+ try std.testing.expectEqual(@as(u64, 3), withJitter(3, 12345));+ // No overflow at the top of the range.+ try std.testing.expect(withJitter(std.math.maxInt(u64), std.math.maxInt(u64)) == std.math.maxInt(u64));+}++test "withJitter (#1274): a server Retry-After is a floor, never shortened" {+ var prng = std.Random.DefaultPrng.init(7);+ const rand = prng.random();+ // The call sites pass the server hint as the base (e.g. 3 s from+ // "try again in 3 seconds", or a capped Retry-After header).+ for ([_]u64{ 1000, 3000, 30_000 }) |server_ms| {+ var i: usize = 0;+ while (i < 200) : (i += 1) try std.testing.expect(withRandom(rand, server_ms) >= server_ms);+ }+}++test "withJitter (#1274): seeded draws are deterministic and actually spread" {+ var a = std.Random.DefaultPrng.init(42);+ var b = std.Random.DefaultPrng.init(42);+ var seen_min: u64 = std.math.maxInt(u64);+ var seen_max: u64 = 0;+ var i: usize = 0;+ while (i < 64) : (i += 1) {+ const x = withRandom(a.random(), 4000);+ try std.testing.expectEqual(x, withRandom(b.random(), 4000));+ seen_min = @min(seen_min, x);+ seen_max = @max(seen_max, x);+ }+ // Two callers retrying the same step must not wait the same amount: the+ // lockstep this fixes. 64 draws over a 1000 ms span spread well past 100 ms.+ try std.testing.expect(seen_max - seen_min > 100);+}diff --git a/src/subagent_run.zig b/src/subagent_run.zigindex 7f727e56..af223858 100644--- a/src/subagent_run.zig+++ b/src/subagent_run.zig@@ -399,7 +399,7 @@ pub fn runSub(ctx: ToolCtx, kind: []const u8, label: []const u8, prompt: []const// A /loop deadline the run is already past makes waiting pointless.if (subagent_retry.pastDeadline(util.unixMs(ctx.io), ctx.loop_deadline_ms)) break :ask r;subagent_retry.traceAttempt(ctx.tracer, sub_id, attempts, fail_kind, agent.last_api_error);- agent.sleepInterruptible(subagent_retry.backoffMs(fail_kind, attempts)) catch break :ask r;+ agent.sleepInterruptible(@import("retry_jitter.zig").ms(agent.io, subagent_retry.backoffMs(fail_kind, attempts))) catch break :ask r; // #1274}};try @import("subagent_retained.zig").checkpoint(&agent, ctx);