graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
fix: keep retries out of answers and interrupt backoff
yxlyx1b9edc8bd82c
diff --git a/build.zig b/build.zigindex aee1d528..8c762e9b 100644--- a/build.zig+++ b/build.zig@@ -224,6 +224,9 @@ pub fn build(b: *std.Build) void {const acp_stream_drop_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-stream-drop.py" });acp_stream_drop_test.addArtifactArg(exe);test_step.dependOn(&acp_stream_drop_test.step);+ const retry_cancellation_test = b.addSystemCommand(&.{ "python3", "scripts/test-retry-cancellation.py" });+ retry_cancellation_test.addArtifactArg(exe);+ test_step.dependOn(&retry_cancellation_test.step);const run_tool_budget_test = b.addSystemCommand(&.{ "python3", "scripts/test-run-tool-budget.py" });run_tool_budget_test.addArtifactArg(exe);test_step.dependOn(&run_tool_budget_test.step);diff --git a/scripts/test-acp-http2-retry.py b/scripts/test-acp-http2-retry.pyindex f80e8025..5a2c4bf3 100644--- a/scripts/test-acp-http2-retry.py+++ b/scripts/test-acp-http2-retry.py@@ -2,7 +2,7 @@"""Offline ACP regression for an ambiguous HTTP/2 failure after POST flush.Builds with a verified copy of the pinned http-zig dependency. Only the local-CA trust source and a one-shot post-flush fault are injected into that copy.+CA trust source and controlled post-flush faults are injected into that copy.The production harness and retry policy are used unchanged."""import importlib.util@@ -48,8 +48,22 @@ def build(root):site = ' try self.sendBody(sid, req.body);'position = contents.index(site, start)assert contents.find(site, position + len(site), contents.index('\n }', start)) == -1- contents = contents[:position + len(site)] + '\n // Test-only fault after the complete POST flush.\n if (!retry_fault_once.swap(true, .acq_rel)) return error.EndOfStream;' + contents[position + len(site):]- contents = contents.replace('const std = @import("std");', 'const std = @import("std");\nvar retry_fault_once: std.atomic.Value(bool) = .init(false);', 1)+ fault = '''+ // Test-only faults before the engine receives any response bytes:+ // recover once, exhaust a turn, then fail four times for cancellation.+ if (std.mem.eql(u8, req.authority, "localhost") and std.mem.eql(u8, req.method, "POST")) {+ const n = retry_fault_calls.fetchAdd(1, .acq_rel);+ if (n == 0 or (n >= 2 and n < 8) or (n >= 9 and n < 13)) {+ // Synchronize with the peer: an immediate close can race delivery+ // of the flushed POST, making the duplicate-body assertion flaky.+ var received = try self.readResponse(sid);+ received.deinit();+ return error.EndOfStream;+ }+ }+'''+ contents = contents[:position + len(site)] + fault + contents[position + len(site):]+ contents = contents.replace('const std = @import("std");', 'const std = @import("std");\nvar retry_fault_calls: std.atomic.Value(usize) = .init(0);', 1)conn.write_text(contents)fixture.run(['zig', 'build', '--system', str(packages), '--cache-dir', str(root/'cache'),'-p', str(root/'install')], cwd=REPO, timeout=240)@@ -116,11 +130,11 @@ def exercise(root, binary):result, updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Say OK.'}]}, 3)assert result.get('result', {}).get('stopReason') == 'end_turn', resultreply_chunks = chunks(updates)- # #1274: the 250 ms first flake step is jittered upward by at most 25%.- retry_line = re.fullmatch(- r'\[network error: EndOfStream — retrying in (\d+)ms \(1/6\)\]', reply_chunks[0]- ) if len(reply_chunks) == 2 else None- assert retry_line and 250 <= int(retry_line.group(1)) <= 312 and reply_chunks[1] == 'OK', reply_chunks+ # #625: retry housekeeping is chrome, never answer/thought content.+ # ACP has no retry-status channel; do not counterfeit a message.+ assert reply_chunks == ['OK'], reply_chunks+ assert not any('network error:' in json.dumps(event) or 'retrying in' in json.dumps(event)+ for event in updates), updatesrows = [json.loads(line) for line in log.read_text().splitlines()]posts = [row for row in rows if row['event'] == 'post']assert len(posts) == 2, posts@@ -137,7 +151,104 @@ def exercise(root, binary):assert posts[1]['atMs'] - posts[0]['atMs'] >= 200, 'Second POST bypassed the 250ms request-level retry'after_cost = [json.loads(line) for line in log.read_text().splitlines()]assert len([row for row in after_cost if row['event'] == 'post']) == 2, '/cost caused a model POST'- print('ACP HTTP/2 post-flush retry: 2 matching h2 POSTs, outer retry observed, 1 failed attempt reported')+ failed, failed_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Exercise the exhausted transport fixture.'}]}, 5)+ assert failed.get('error', {}).get('code') == -32603, failed+ assert 'EndOfStream (gave up after 6 attempts)' in failed['error']['message'], failed+ assert chunks(failed_updates) == [], failed_updates+ assert not any('network error:' in json.dumps(event) or 'retrying in' in json.dumps(event)+ for event in failed_updates), failed_updates+ failed_posts = [json.loads(line) for line in log.read_text().splitlines()+ if json.loads(line)['event'] == 'post'][2:]+ assert len(failed_posts) == 6, failed_posts+ assert len({row['sha256'] for row in failed_posts}) == 1, failed_posts+ assert all(row['protocol'] == 'h2' for row in failed_posts), failed_posts+ resumed, resumed_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Try again explicitly.'}]}, 6)+ assert resumed.get('result', {}).get('stopReason') == 'end_turn', resumed+ assert chunks(resumed_updates) == ['OK'], resumed_updates+ for mode in ('json', 'oneshot'):+ work = case/mode+ work.mkdir()+ before = len([row for row in map(json.loads, log.read_text().splitlines())+ if row['event'] == 'post'])+ args = [str(binary), '--model', 'vercel', '--yolo']+ stdin = ''+ if mode == 'json':+ args += ['--json']+ stdin = json.dumps({'type':'user', 'text':'Say OK.'}) + '\n'+ else:+ args += ['-p', 'Say OK.']+ run = subprocess.Popen(args, cwd=work, env=env, stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ text=True, start_new_session=True)+ try:+ stdout, stderr = run.communicate(stdin, timeout=45)+ assert run.returncode == 0, (mode, stderr)+ finally:+ fixture.terminate(run)+ assert 'network error:' not in stdout and 'retrying in' not in stdout, stdout+ assert '· retry' not in stdout, stdout+ if mode == 'json':+ wire = [json.loads(line) for line in stdout.splitlines()]+ turns = [event for event in wire if event.get('type') == 'turn']+ assert len(turns) == 1 and turns[0]['text'] == 'OK', turns+ text = ''.join(event.get('text', '') for event in wire if event.get('type') == 'text')+ assert text == 'OK', text+ sequence = [event['seq'] for event in wire]+ assert sequence == list(range(sequence[0], sequence[0] + len(sequence))), sequence+ else:+ assert stdout.strip() == 'OK', stdout+ posts = [row for row in map(json.loads, log.read_text().splitlines())+ if row['event'] == 'post'][before:]+ assert len(posts) == 2 and posts[0]['sha256'] == posts[1]['sha256'], (mode, posts)+ assert all(row['protocol'] == 'h2' for row in posts), (mode, posts)+ trace = [row for path in (work/'.graff/traces').glob('*.jsonl')+ for row in map(json.loads, path.read_text().splitlines())]+ assert any(row.get('ev') == 'retry' for row in trace), (mode, 'retry breadcrumb missing')+ print(f'PASS {mode} retry chrome: two matching POSTs, clean answer, raw retry trace retained')+ def retries():+ count = 0+ for path in (case/'.graff/traces').glob('*.jsonl'):+ for line in path.read_text().splitlines(keepends=True):+ # A trace append may be in flight; only parse full rows.+ if line.endswith('\n') and json.loads(line).get('ev') == 'retry':+ count += 1+ return count++ before = len([row for row in map(json.loads, log.read_text().splitlines())+ if row['event'] == 'post'])+ retry_count = retries()+ worker.stdin.write(json.dumps({'jsonrpc':'2.0', 'id':7, 'method':'session/prompt',+ 'params':{'sessionId':sid, 'prompt':[{'type':'text', 'text':'Exercise the cancelled transport fixture.'}]}}) + '\n')+ worker.stdin.flush()+ deadline = time.monotonic() + 15+ # Wait for the fourth retry breadcrumb: its two-second backoff is+ # now in progress, rather than guessing when a request failed.+ while retries() < retry_count + 4:+ assert time.monotonic() < deadline, 'Transport backoff was not reached'+ time.sleep(.01)+ started = time.monotonic()+ worker.stdin.write(json.dumps({'jsonrpc':'2.0', 'method':'session/cancel',+ 'params':{'sessionId':sid}}) + '\n')+ worker.stdin.flush()+ cancelled_updates = []+ while True:+ event = events.get(timeout=max(.01, started + 5 - time.monotonic()))+ assert isinstance(event, dict), 'ACP exited during retry cancellation'+ if event.get('id') == 7:+ assert event.get('result', {}).get('stopReason') == 'cancelled', event+ break+ cancelled_updates.append(event)+ assert time.monotonic() - started < 2, 'Cancellation waited out the retry backoff'+ assert chunks(cancelled_updates) == [], cancelled_updates+ cancelled_posts = [row for row in map(json.loads, log.read_text().splitlines())+ if row['event'] == 'post'][before:]+ assert len(cancelled_posts) == 4, cancelled_posts+ assert len({row['sha256'] for row in cancelled_posts}) == 1, cancelled_posts+ after_cancel, after_cancel_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Try once more explicitly.'}]}, 8)+ assert after_cancel.get('result', {}).get('stopReason') == 'end_turn', after_cancel+ assert chunks(after_cancel_updates) == ['OK'], after_cancel_updates+ print('PASS ACP retry cancellation: interrupts backoff, no extra POST or answer, same session remains usable')+ print('PASS ACP HTTP/2 retry chrome: recovery, six-attempt exhaustion, explicit retry; no retry text in answers')finally:fixture.terminate(worker)fixture.terminate(server)diff --git a/scripts/test-retry-cancellation.py b/scripts/test-retry-cancellation.pynew file mode 100644index 00000000..5812d706--- /dev/null+++ b/scripts/test-retry-cancellation.py@@ -0,0 +1,301 @@+#!/usr/bin/env python3+"""Offline retry/cancel races: ACP v1/v2 and JSON, transport/429/503 failures.++Uses ephemeral loopback ports and isolated homes. No real credentials or model+calls. A retry trace row synchronizes cancellation with the actual backoff.+"""+import hashlib+import http.server+import json+import os+from pathlib import Path+import queue+import signal+import socket+import subprocess+import sys+import tempfile+import threading+import time++class Model:+ def __init__(self, kind, failures=-1, delay_ms=30000):+ self.kind, self.failures, self.delay_ms = kind, failures, delay_ms+ self.lock = threading.Lock()+ self.posts = []+ model = self++ class Handler(http.server.BaseHTTPRequestHandler):+ protocol_version = 'HTTP/1.1'++ def log_message(self, *_args):+ pass++ def do_GET(self):+ self.respond(200, b'{"data":[{"id":"fixture"}]}', 'application/json')++ def respond(self, status, body, content_type):+ self.send_response(status)+ self.send_header('content-type', content_type)+ self.send_header('content-length', str(len(body)))+ self.send_header('connection', 'close')+ if status in (429, 503):+ self.send_header('retry-after-ms', str(model.delay_ms))+ self.end_headers()+ try:+ self.wfile.write(body)+ self.wfile.flush()+ except (BrokenPipeError, ConnectionResetError):+ pass+ self.close_connection = True++ def do_POST(self):+ body = self.rfile.read(int(self.headers['content-length']))+ with model.lock:+ model.posts.append((hashlib.sha256(body).hexdigest(), time.monotonic()))+ fail = model.failures != 0+ if model.failures > 0:+ model.failures -= 1+ if fail and model.kind == 'transport':+ self.close_connection = True+ self.connection.shutdown(socket.SHUT_RDWR)+ self.connection.close()+ elif fail:+ self.respond(int(model.kind), b'{"error":{"message":"fixture transient failure"}}', 'application/json')+ else:+ reply = ('data: {"choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":null}]}\n\n'+ 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n'+ 'data: [DONE]\n\n').encode()+ self.respond(200, reply, 'text/event-stream')++ self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler)+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)+ self.thread.start()++ def recover(self):+ with self.lock:+ self.failures = 0++ def requests(self):+ with self.lock:+ return list(self.posts)++ def close(self):+ self.server.shutdown()+ self.server.server_close()+ self.thread.join(timeout=5)+++class Client:+ def __init__(self, binary, root, model, mode):+ self.root, self.mode = root, mode+ self.seen, self.events = [], queue.Queue()+ self.counter = 0+ self.sid = None+ (root/'mcp.json').write_text('{"mcpServers":{}}')+ shim = root/'bin'+ shim.mkdir()+ (shim/'security').write_text('#!/bin/sh\nexit 44\n')+ (shim/'security').chmod(0o700)+ env = dict(HOME=str(root), USERPROFILE=str(root),+ PATH=str(shim)+os.pathsep+os.defpath, TERM='dumb',+ AI_GATEWAY_API_KEY='fixture', GRAFF_NO_TELEMETRY='1',+ GRAFF_VERCEL_URL=f'http://127.0.0.1:{model.server.server_port}/v1/chat/completions',+ GRAFF_MCP_CONFIG=str(root/'mcp.json'), GRAFF_FLEET='off',+ GRAFF_NO_ADOPT='1', GRAFF_NO_SMOLIFY='1', GRAFF_NO_CODEDB_GUARD='1')+ if 'SystemRoot' in os.environ:+ env['SystemRoot'] = os.environ['SystemRoot']+ if mode == 'acp2':+ env['GRAFF_ACP_V2'] = '1'+ args = [str(binary), '--json'] if mode == 'json' else [str(binary), 'acp']+ self.stderr = (root/'stderr').open('w')+ self.proc = subprocess.Popen(args + ['--model', 'vercel', '--yolo'],+ cwd=root, env=env, stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=self.stderr,+ text=True, start_new_session=os.name == 'posix')++ def read():+ try:+ for line in self.proc.stdout:+ self.events.put(json.loads(line))+ except Exception as error:+ self.events.put(error)+ finally:+ self.events.put(None)+ self.reader = threading.Thread(target=read, daemon=True)+ self.reader.start()+ try:+ if mode != 'json':+ init = self.call('initialize', {'protocolVersion':2 if mode == 'acp2' else 1})+ assert init['result']['protocolVersion'] == (2 if mode == 'acp2' else 1), init+ self.sid = self.call('session/new', {'cwd':str(root), 'mcpServers':[]})['result']['sessionId']+ except BaseException:+ self.close()+ raise++ def send(self, value):+ self.proc.stdin.write(json.dumps(value) + '\n')+ self.proc.stdin.flush()++ def request(self, method, params):+ self.counter += 1+ self.send(dict(jsonrpc='2.0', id=self.counter, method=method, params=params))+ return self.counter++ def until(self, predicate, timeout=8):+ deadline = time.monotonic() + timeout+ while True:+ value = self.events.get(timeout=max(.01, deadline-time.monotonic()))+ assert isinstance(value, dict), ('invalid/closed protocol stream', value)+ self.seen.append(value)+ if predicate(value):+ return value+ assert time.monotonic() < deadline, ('protocol deadline', self.seen[-3:])++ def call(self, method, params):+ identifier = self.request(method, params)+ return self.until(lambda value: value.get('id') == identifier)++ def trace(self):+ return [json.loads(line) for path in (self.root/'.graff/traces').glob('*.jsonl')+ for line in path.read_text().splitlines(keepends=True) if line.endswith('\n')]++ def begin(self):+ self.start = len(self.seen)+ self.retry_count = sum(row.get('ev') == 'retry' for row in self.trace())+ if self.mode == 'json':+ self.send({'type':'user', 'text':'Say OK.'})+ return None+ return self.request('session/prompt', {'sessionId':self.sid, 'prompt':[{'type':'text', 'text':'Say OK.'}]})++ def wait_retry(self):+ deadline = time.monotonic() + 8+ while sum(row.get('ev') == 'retry' for row in self.trace()) <= self.retry_count:+ assert self.proc.poll() is None, 'worker exited before retry'+ assert time.monotonic() < deadline, 'retry backoff not reached'+ time.sleep(.01)++ def cancel(self, wrong=False, client=False):+ if self.mode == 'json':+ self.send({'type':'cancel'})+ else:+ params = {'sessionId':'unrelated-session' if wrong else self.sid}+ if client:+ params['_meta'] = {'graff/cancelSource':'client'}+ self.send(dict(jsonrpc='2.0', method='session/cancel', params=params))++ def finish(self, identifier, cancelled=False):+ if self.mode == 'json':+ value = self.until(lambda event: event.get('type') in ('error', 'turn'))+ assert value.get('type') == ('error' if cancelled else 'turn'), value+ assert value.get('message' if cancelled else 'text') == ('turn cancelled' if cancelled else 'OK'), value+ elif self.mode == 'acp2':+ value = self.until(lambda event: event.get('params', {}).get('update', {}).get('state') == 'idle')+ assert value['params']['update']['stopReason'] == ('cancelled' if cancelled else 'end_turn'), value+ acks = [event for event in self.seen[self.start:] if event.get('id') == identifier]+ assert len(acks) == 1 and acks[0].get('result', {}).get('messageId'), acks+ else:+ value = self.until(lambda event: event.get('id') == identifier)+ assert value.get('result', {}).get('stopReason') == ('cancelled' if cancelled else 'end_turn'), value+ wire = self.seen[self.start:]+ assert not any(token in json.dumps(wire) for token in ('network error:', 'retrying in', '· retry')), wire+ if self.mode == 'json':+ sequences = [event['seq'] for event in self.seen]+ assert sequences == list(range(sequences[0], sequences[0] + len(sequences))), sequences+ return value++ def saved(self):+ return [message for path in (self.root/'.graff/sessions').glob('*.session.json')+ for message in json.loads(path.read_text()).get('messages', [])]++ def close(self):+ if self.proc.poll() is None:+ try:+ if os.name == 'posix':+ os.killpg(self.proc.pid, signal.SIGKILL)+ else:+ self.proc.kill()+ except ProcessLookupError:+ pass+ self.proc.wait(timeout=8)+ self.reader.join(timeout=5)+ assert not self.reader.is_alive(), 'protocol reader did not stop'+ for stream in (self.proc.stdin, self.proc.stdout):+ stream.close()+ self.stderr.close()+++def exercise(binary, mode, kind, scenario):+ with tempfile.TemporaryDirectory(prefix='graff-retry-cancel-') as temp:+ model = Model(kind, failures=1 if scenario == 'wrong' else -1,+ delay_ms=500 if scenario == 'wrong' else 30000)+ client = None+ try:+ client = Client(binary, Path(temp), model, mode)+ marker = None+ identifier = client.begin()+ if scenario == 'immediate':+ client.cancel()+ else:+ client.wait_retry()+ assert len(model.requests()) == 1, 'unexpected retry before cancellation'+ client.cancel(wrong=scenario == 'wrong', client=scenario == 'client')+ client.cancel(wrong=scenario == 'wrong', client=scenario == 'client')+ started = time.monotonic()+ client.finish(identifier, cancelled=scenario != 'wrong')+ elapsed = time.monotonic() - started+ if scenario == 'wrong':+ posts = model.requests()+ assert len(posts) == 2 and posts[0][0] == posts[1][0], posts+ if kind != 'transport':+ assert posts[1][1] - posts[0][1] >= .45, 'Retry-After bypassed'+ else:+ assert elapsed < 2, ('cancel waited out backoff', elapsed)+ assert len(model.requests()) <= 1, model.requests()+ source = 'json_cancel' if mode == 'json' else 'acp_client' if scenario == 'client' else 'acp_cancel'+ if scenario != 'immediate':+ assert f'interrupted source={source}' in json.dumps(client.trace()), source+ marker = 'stopped by the client, not the user' if scenario == 'client' else 'interrupted by user'+ if scenario == 'eof':+ # The documented checkpoint boundary is graceful exit: JSON may+ # publish its error before the synchronous save has returned.+ client.proc.stdin.close()+ assert client.proc.wait(timeout=8) == 0, 'worker did not exit cleanly'+ assert marker in json.dumps(client.saved()), ('cancelled history lost at EOF', client.saved())+ assert len(model.requests()) == 1, 'EOF dispatched another retry'+ print(f'PASS {mode}/{kind}/eof: graceful exit drains the cancelled checkpoint', flush=True)+ return+ # Duplicate/late cancels while idle must not poison the next turn.+ client.cancel()+ client.cancel()+ model.recover()+ before = len(model.requests())+ followup = client.begin()+ client.finish(followup)+ assert len(model.requests()) == before + 1, 'follow-up was lost or replayed'+ # JSON emits its error before persisting history. The next completed+ # turn is a barrier proving the prior interrupted turn was saved.+ if marker:+ assert marker in json.dumps(client.saved()), (scenario, client.saved())+ print(f'PASS {mode}/{kind}/{scenario}: correct scope, clean wire, same-worker continuation', flush=True)+ finally:+ if client:+ client.close()+ model.close()+++def main():+ binary = Path(sys.argv[1] if len(sys.argv) > 1 else 'zig-out/bin/graff').resolve()+ cases = [(mode, kind, 'cancel') for mode in ('acp1', 'acp2', 'json')+ for kind in ('transport', '429', '503')]+ cases += [(mode, kind, 'wrong') for mode in ('acp1', 'acp2') for kind in ('transport', '429')]+ cases += [(mode, '503', 'immediate') for mode in ('acp1', 'acp2', 'json')]+ cases += [(mode, '429', 'client') for mode in ('acp1', 'acp2')]+ cases += [(mode, '503', 'eof') for mode in ('acp1', 'acp2', 'json')]+ for case in cases:+ exercise(binary, *case)+ print(f'PASS retry cancellation matrix: {len(cases)} scenarios', flush=True)+++if __name__ == '__main__':+ main()diff --git a/src/agent_interrupt.zig b/src/agent_interrupt.zigindex 900d08f1..77a0e68a 100644--- a/src/agent_interrupt.zig+++ b/src/agent_interrupt.zig@@ -91,18 +91,19 @@ pub fn restoreStdin(orig: tty.RawState) void {tty.restore(orig);}-/// Sleep `ms` watching stdin for Esc (when the root is on a TTY), so the-/// user can cancel a retry backoff instead of waiting it out.+/// Sleep `ms` watching both the engine cancel flag (ACP/JSON/hosted clients)+/// and stdin Esc when the root is on a TTY. Never wait out a cancelled backoff.pub fn sleepInterruptible(self: *Agent, ms: u64) error{Interrupted}!void {+ if (Agent.esc_cancel.load(.acquire)) return error.Interrupted;const watch = !self.sub and self.in != null and main_mod.use_color and !main_mod.json_mode;const orig_tio: ?tty.RawState = if (watch) rawNonblockStdin() else null;defer if (orig_tio) |o| tty.restore(o);var left = ms;while (left > 0) {const chunk = @min(left, 100);- self.io.sleep(.fromMilliseconds(@intCast(chunk)), .awake) catch {};+ self.io.sleep(.fromMilliseconds(@intCast(chunk)), .awake) catch return error.Interrupted;left -= chunk;- if (orig_tio != null and escPressed(true)) return error.Interrupted;+ if (Agent.esc_cancel.load(.acquire) or (orig_tio != null and escPressed(true))) return error.Interrupted;}}@@ -122,6 +123,92 @@ pub fn sseIndex(obj: std.json.ObjectMap) ?usize {return @intCast(ix.integer);}+const BackoffSleepProbe = struct {+ calls: u64 = 0,+ total_ms: u64 = 0,+ max_ms: u64 = 0,+ cancel_at: u64 = 0,+ cancel_io: bool = false,++ fn sleep(ctx: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {+ const self: *@This() = @ptrCast(@alignCast(ctx.?));+ const ms: u64 = @intCast(timeout.duration.raw.toMilliseconds());+ std.debug.assert(timeout.duration.clock == .awake);+ self.calls += 1;+ self.total_ms += ms;+ self.max_ms = @max(self.max_ms, ms);+ if (self.cancel_io) return error.Canceled;+ if (self.calls == self.cancel_at) Agent.esc_cancel.store(true, .release);+ }+};++test "retry backoff preserves its full duration in bounded sleep chunks" {+ const previous = Agent.esc_cancel.swap(false, .acq_rel);+ defer Agent.esc_cancel.store(previous, .release);+ var vtable = std.testing.io.vtable.*;+ vtable.sleep = BackoffSleepProbe.sleep;+ var agent: Agent = undefined;+ agent.in = null;+ agent.sub = false;+ for ([_]u64{ 0, 1, 99, 100, 101, 250, 65_535 }) |ms| {+ var probe: BackoffSleepProbe = .{};+ agent.io = .{ .userdata = &probe, .vtable = &vtable };+ try sleepInterruptible(&agent, ms);+ try std.testing.expectEqual(ms, probe.total_ms);+ try std.testing.expectEqual((ms + 99) / 100, probe.calls);+ try std.testing.expect(probe.max_ms <= 100);+ }+}++test "retry backoff stops when cancellation arrives during a sleep chunk" {+ const previous = Agent.esc_cancel.load(.acquire);+ defer Agent.esc_cancel.store(previous, .release);+ var vtable = std.testing.io.vtable.*;+ vtable.sleep = BackoffSleepProbe.sleep;+ var agent: Agent = undefined;+ agent.in = null;+ for ([_]bool{ false, true }) |sub| {+ agent.sub = sub;+ for ([_]u64{ 1, 2 }) |cancel_at| {+ Agent.esc_cancel.store(false, .release);+ var probe: BackoffSleepProbe = .{ .cancel_at = cancel_at };+ agent.io = .{ .userdata = &probe, .vtable = &vtable };+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 10_000));+ try std.testing.expectEqual(cancel_at, probe.calls);+ try std.testing.expectEqual(cancel_at * 100, probe.total_ms);+ }+ }+}++test "retry backoff propagates I/O cancellation without another sleep" {+ const previous = Agent.esc_cancel.swap(false, .acq_rel);+ defer Agent.esc_cancel.store(previous, .release);+ var vtable = std.testing.io.vtable.*;+ vtable.sleep = BackoffSleepProbe.sleep;+ var agent: Agent = undefined;+ agent.in = null;+ agent.sub = false;+ var probe: BackoffSleepProbe = .{ .cancel_io = true };+ agent.io = .{ .userdata = &probe, .vtable = &vtable };+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 10_000));+ try std.testing.expectEqual(@as(u64, 1), probe.calls);+ try std.testing.expect(!Agent.esc_cancel.load(.acquire));+}++test "retry backoff respects a pending non-TTY cancellation without sleeping" {+ const previous = Agent.esc_cancel.swap(true, .acq_rel);+ defer Agent.esc_cancel.store(previous, .release);+ var agent: Agent = undefined;+ agent.in = null;+ for ([_]bool{ false, true }) |sub| {+ agent.sub = sub;+ // io is deliberately undefined: even a long wait must return before+ // touching it, including for a child or a zero-duration backoff.+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 60_000));+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 0));+ }+}+test { // #728: cancel_source has no other path into the test root_ = cancel_source;_ = @import("agent_interrupt_repl_tests.zig");diff --git a/src/agent_request.zig b/src/agent_request.zigindex cf7415f3..5a41a098 100644--- a/src/agent_request.zig+++ b/src/agent_request.zig@@ -374,9 +374,8 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {// Transport flakes back off; rapid retries against a// just-closed keep-alive re-fail (#86). Cap: 4s/6 tries.const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(throttled, attempt));- @import("turn_chrome.zig").emitRetryNotice(self.io, @errorName(err), attempt + 1, max_attempts);if (scratch.showRecoveredTransportRetry(self.call_kind))- try scratch.announceNetworkRetry(self, err, delay_ms, attempt + 1, max_attempts);+ @import("turn_chrome.zig").emitRetryNotice(self.io, @errorName(err), attempt + 1, max_attempts);// Same trace breadcrumb the 429/5xx branch leaves: a// transport-flake retry is otherwise invisible in the// session trace, hiding how flaky a provider really is.diff --git a/src/agent_request_scratch.zig b/src/agent_request_scratch.zigindex daa8b5c8..0f0ceb98 100644--- a/src/agent_request_scratch.zig+++ b/src/agent_request_scratch.zig@@ -29,16 +29,6 @@ pub fn showRecoveredTransportRetry(kind: run_budget_mod.CallKind) bool {return kind != .recap;}-/// Preserve the existing notice in JSON/ACP, where say() is deliberately silent.-pub fn announceNetworkRetry(self: *Agent, err: anyerror, delay_ms: u64, attempt: usize, limit: usize) !void {- try self.say("[network error: {t} — retrying in {d}ms ({d}/{d})]\n", .{ err, delay_ms, attempt, limit });- if (@import("main.zig").json_mode and !self.sub) {- var buf: [192]u8 = undefined;- const text = try std.fmt.bufPrint(&buf, "[network error: {t} — retrying in {d}ms ({d}/{d})]", .{ err, delay_ms, attempt, limit });- self.emit(.{ .type = "text", .text = text });- }-}-/// A response body of only SSE comment lines (`: OPENROUTER PROCESSING` …)/// means the gateway queued us and never produced tokens — back off and re-ask/// like a 5xx instead of dying on an "unparseable" JSON parse.diff --git a/src/tool_pulse.zig b/src/tool_pulse.zigindex 7178af4a..5472d58d 100644--- a/src/tool_pulse.zig+++ b/src/tool_pulse.zig@@ -70,7 +70,8 @@ pub fn emitNotice(io: Io, comptime fmt: []const u8, args: anytype) void {w.flush() catch {};}-/// Line-REPL only. Hosted TUI/ACP still get the pulse via `hostedSink`.+/// Line-REPL only. Hosted frontends get the pulse via a bound `hostedSink`.+/// ACP has no bound chrome sink, so --json keeps retries off its answer stream./// `-p` / `--json` keep stdout as the answer (in-house atomic-symlink-write/// captured `· turn still going ·` when the pulse rode `g_out`).pub fn chromeGoesToStdout() bool {diff --git a/src/turn_chrome.zig b/src/turn_chrome.zigindex b2993c2c..37ff3626 100644--- a/src/turn_chrome.zig+++ b/src/turn_chrome.zig@@ -88,6 +88,69 @@ test "formatRetryNotice names HungRequest and UnknownHostName with attempt n/N"try std.testing.expectEqualStrings("· retry 1/6 · UnknownHostName", formatRetryNotice(&buf, "UnknownHostName", 1, 6));}+test "retry chrome stays out of ACP JSON and one-shot answer writers" {+ const main_mod = @import("main.zig");+ const prev_json = main_mod.json_mode;+ const prev_unattended = main_mod.unattended;+ const prev_out = main_mod.g_out;+ defer {+ main_mod.json_mode = prev_json;+ main_mod.unattended = prev_unattended;+ main_mod.g_out = prev_out;+ }+ var output: Io.Writer.Allocating = .init(std.testing.allocator);+ defer output.deinit();+ main_mod.g_out = &output.writer;+ for ([_][2]bool{ .{ true, false }, .{ false, true }, .{ true, true } }) |mode| {+ main_mod.json_mode = mode[0];+ main_mod.unattended = mode[1];+ emitRetryNotice(undefined, "UnknownHostName", 1, 6);+ emitRetryNotice(undefined, "HungRequest", 2, 6);+ try std.testing.expectEqualStrings("", output.written());+ }+ main_mod.json_mode = false;+ main_mod.unattended = false;+ emitRetryNotice(undefined, "UnknownHostName", 1, 6);+ emitRetryNotice(undefined, "HungRequest", 2, 6);+ const text = output.written();+ try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, text, "\n"));+ try std.testing.expect(std.mem.indexOf(u8, text, "· retry 1/6 · UnknownHostName") != null);+ try std.testing.expect(std.mem.indexOf(u8, text, "· retry 2/6 · HungRequest") != null);+ try std.testing.expect(std.mem.indexOf(u8, text, "[network error:") == null);+}++test "hosted retry chrome is one dim presentation notice, not a durable answer" {+ const sinks = @import("engine_sink.zig");+ const events = @import("engine_events.zig");+ const Capture = struct {+ count: usize = 0,+ dim: bool = false,+ durable: bool = false,+ text: [120]u8 = undefined,+ len: usize = 0,+ fn emit(ctx: *anyopaque, stamped: sinks.Stamped) void {+ const self: *@This() = @ptrCast(@alignCast(ctx));+ self.count += 1;+ self.durable = events.durable(stamped.event);+ if (stamped.event == .session_notice) {+ const notice = stamped.event.session_notice;+ self.dim = notice.tone == .dim;+ self.len = notice.text.len;+ @memcpy(self.text[0..self.len], notice.text);+ }+ }+ };+ var capture: Capture = .{};+ const vt: sinks.VTable = .{ .emit = Capture.emit, .durable = false };+ sinks.bindTurnSink(.{ .ctx = &capture, .vt = &vt });+ defer sinks.unbindTurnSink();+ emitRetryNotice(undefined, "HungRequest", 3, 6);+ try std.testing.expectEqual(@as(usize, 1), capture.count);+ try std.testing.expect(capture.dim);+ try std.testing.expect(!capture.durable);+ try std.testing.expectEqualStrings("· retry 3/6 · HungRequest", capture.text[0..capture.len]);+}+test "formatTurnPulse stays off the transcript (ADR 0021)" {var buf: [80]u8 = undefined;try std.testing.expectEqualStrings("", formatTurnPulse(&buf, 3, 0, 11));