Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

fix: keep retries out of answers and interrupt backoff

yxlyx1b9edc8bd82c
diff --git a/build.zig b/build.zig
index aee1d528..8c762e9b 100644
--- a/build.zig
+++ b/build.zig
@@ -224,6 +224,9 @@ pub fn build(b: *std.Build) void {
const acp_stream_drop_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-stream-drop.py" });
acp_stream_drop_test.addArtifactArg(exe);
test_step.dependOn(&acp_stream_drop_test.step);
+ const retry_cancellation_test = b.addSystemCommand(&.{ "python3", "scripts/test-retry-cancellation.py" });
+ retry_cancellation_test.addArtifactArg(exe);
+ test_step.dependOn(&retry_cancellation_test.step);
const run_tool_budget_test = b.addSystemCommand(&.{ "python3", "scripts/test-run-tool-budget.py" });
run_tool_budget_test.addArtifactArg(exe);
test_step.dependOn(&run_tool_budget_test.step);
diff --git a/scripts/test-acp-http2-retry.py b/scripts/test-acp-http2-retry.py
index f80e8025..5a2c4bf3 100644
--- a/scripts/test-acp-http2-retry.py
+++ b/scripts/test-acp-http2-retry.py
@@ -2,7 +2,7 @@
"""Offline ACP regression for an ambiguous HTTP/2 failure after POST flush.
Builds with a verified copy of the pinned http-zig dependency. Only the local
-CA trust source and a one-shot post-flush fault are injected into that copy.
+CA trust source and controlled post-flush faults are injected into that copy.
The production harness and retry policy are used unchanged.
"""
import importlib.util
@@ -48,8 +48,22 @@ def build(root):
site = ' try self.sendBody(sid, req.body);'
position = contents.index(site, start)
assert contents.find(site, position + len(site), contents.index('\n }', start)) == -1
- contents = contents[:position + len(site)] + '\n // Test-only fault after the complete POST flush.\n if (!retry_fault_once.swap(true, .acq_rel)) return error.EndOfStream;' + contents[position + len(site):]
- contents = contents.replace('const std = @import("std");', 'const std = @import("std");\nvar retry_fault_once: std.atomic.Value(bool) = .init(false);', 1)
+ fault = '''
+ // Test-only faults before the engine receives any response bytes:
+ // recover once, exhaust a turn, then fail four times for cancellation.
+ if (std.mem.eql(u8, req.authority, "localhost") and std.mem.eql(u8, req.method, "POST")) {
+ const n = retry_fault_calls.fetchAdd(1, .acq_rel);
+ if (n == 0 or (n >= 2 and n < 8) or (n >= 9 and n < 13)) {
+ // Synchronize with the peer: an immediate close can race delivery
+ // of the flushed POST, making the duplicate-body assertion flaky.
+ var received = try self.readResponse(sid);
+ received.deinit();
+ return error.EndOfStream;
+ }
+ }
+'''
+ contents = contents[:position + len(site)] + fault + contents[position + len(site):]
+ contents = contents.replace('const std = @import("std");', 'const std = @import("std");\nvar retry_fault_calls: std.atomic.Value(usize) = .init(0);', 1)
conn.write_text(contents)
fixture.run(['zig', 'build', '--system', str(packages), '--cache-dir', str(root/'cache'),
'-p', str(root/'install')], cwd=REPO, timeout=240)
@@ -116,11 +130,11 @@ def exercise(root, binary):
result, updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Say OK.'}]}, 3)
assert result.get('result', {}).get('stopReason') == 'end_turn', result
reply_chunks = chunks(updates)
- # #1274: the 250 ms first flake step is jittered upward by at most 25%.
- retry_line = re.fullmatch(
- r'\[network error: EndOfStream — retrying in (\d+)ms \(1/6\)\]', reply_chunks[0]
- ) if len(reply_chunks) == 2 else None
- assert retry_line and 250 <= int(retry_line.group(1)) <= 312 and reply_chunks[1] == 'OK', reply_chunks
+ # #625: retry housekeeping is chrome, never answer/thought content.
+ # ACP has no retry-status channel; do not counterfeit a message.
+ assert reply_chunks == ['OK'], reply_chunks
+ assert not any('network error:' in json.dumps(event) or 'retrying in' in json.dumps(event)
+ for event in updates), updates
rows = [json.loads(line) for line in log.read_text().splitlines()]
posts = [row for row in rows if row['event'] == 'post']
assert len(posts) == 2, posts
@@ -137,7 +151,104 @@ def exercise(root, binary):
assert posts[1]['atMs'] - posts[0]['atMs'] >= 200, 'Second POST bypassed the 250ms request-level retry'
after_cost = [json.loads(line) for line in log.read_text().splitlines()]
assert len([row for row in after_cost if row['event'] == 'post']) == 2, '/cost caused a model POST'
- print('ACP HTTP/2 post-flush retry: 2 matching h2 POSTs, outer retry observed, 1 failed attempt reported')
+ failed, failed_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Exercise the exhausted transport fixture.'}]}, 5)
+ assert failed.get('error', {}).get('code') == -32603, failed
+ assert 'EndOfStream (gave up after 6 attempts)' in failed['error']['message'], failed
+ assert chunks(failed_updates) == [], failed_updates
+ assert not any('network error:' in json.dumps(event) or 'retrying in' in json.dumps(event)
+ for event in failed_updates), failed_updates
+ failed_posts = [json.loads(line) for line in log.read_text().splitlines()
+ if json.loads(line)['event'] == 'post'][2:]
+ assert len(failed_posts) == 6, failed_posts
+ assert len({row['sha256'] for row in failed_posts}) == 1, failed_posts
+ assert all(row['protocol'] == 'h2' for row in failed_posts), failed_posts
+ resumed, resumed_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Try again explicitly.'}]}, 6)
+ assert resumed.get('result', {}).get('stopReason') == 'end_turn', resumed
+ assert chunks(resumed_updates) == ['OK'], resumed_updates
+ for mode in ('json', 'oneshot'):
+ work = case/mode
+ work.mkdir()
+ before = len([row for row in map(json.loads, log.read_text().splitlines())
+ if row['event'] == 'post'])
+ args = [str(binary), '--model', 'vercel', '--yolo']
+ stdin = ''
+ if mode == 'json':
+ args += ['--json']
+ stdin = json.dumps({'type':'user', 'text':'Say OK.'}) + '\n'
+ else:
+ args += ['-p', 'Say OK.']
+ run = subprocess.Popen(args, cwd=work, env=env, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ text=True, start_new_session=True)
+ try:
+ stdout, stderr = run.communicate(stdin, timeout=45)
+ assert run.returncode == 0, (mode, stderr)
+ finally:
+ fixture.terminate(run)
+ assert 'network error:' not in stdout and 'retrying in' not in stdout, stdout
+ assert '· retry' not in stdout, stdout
+ if mode == 'json':
+ wire = [json.loads(line) for line in stdout.splitlines()]
+ turns = [event for event in wire if event.get('type') == 'turn']
+ assert len(turns) == 1 and turns[0]['text'] == 'OK', turns
+ text = ''.join(event.get('text', '') for event in wire if event.get('type') == 'text')
+ assert text == 'OK', text
+ sequence = [event['seq'] for event in wire]
+ assert sequence == list(range(sequence[0], sequence[0] + len(sequence))), sequence
+ else:
+ assert stdout.strip() == 'OK', stdout
+ posts = [row for row in map(json.loads, log.read_text().splitlines())
+ if row['event'] == 'post'][before:]
+ assert len(posts) == 2 and posts[0]['sha256'] == posts[1]['sha256'], (mode, posts)
+ assert all(row['protocol'] == 'h2' for row in posts), (mode, posts)
+ trace = [row for path in (work/'.graff/traces').glob('*.jsonl')
+ for row in map(json.loads, path.read_text().splitlines())]
+ assert any(row.get('ev') == 'retry' for row in trace), (mode, 'retry breadcrumb missing')
+ print(f'PASS {mode} retry chrome: two matching POSTs, clean answer, raw retry trace retained')
+ def retries():
+ count = 0
+ for path in (case/'.graff/traces').glob('*.jsonl'):
+ for line in path.read_text().splitlines(keepends=True):
+ # A trace append may be in flight; only parse full rows.
+ if line.endswith('\n') and json.loads(line).get('ev') == 'retry':
+ count += 1
+ return count
+
+ before = len([row for row in map(json.loads, log.read_text().splitlines())
+ if row['event'] == 'post'])
+ retry_count = retries()
+ worker.stdin.write(json.dumps({'jsonrpc':'2.0', 'id':7, 'method':'session/prompt',
+ 'params':{'sessionId':sid, 'prompt':[{'type':'text', 'text':'Exercise the cancelled transport fixture.'}]}}) + '\n')
+ worker.stdin.flush()
+ deadline = time.monotonic() + 15
+ # Wait for the fourth retry breadcrumb: its two-second backoff is
+ # now in progress, rather than guessing when a request failed.
+ while retries() < retry_count + 4:
+ assert time.monotonic() < deadline, 'Transport backoff was not reached'
+ time.sleep(.01)
+ started = time.monotonic()
+ worker.stdin.write(json.dumps({'jsonrpc':'2.0', 'method':'session/cancel',
+ 'params':{'sessionId':sid}}) + '\n')
+ worker.stdin.flush()
+ cancelled_updates = []
+ while True:
+ event = events.get(timeout=max(.01, started + 5 - time.monotonic()))
+ assert isinstance(event, dict), 'ACP exited during retry cancellation'
+ if event.get('id') == 7:
+ assert event.get('result', {}).get('stopReason') == 'cancelled', event
+ break
+ cancelled_updates.append(event)
+ assert time.monotonic() - started < 2, 'Cancellation waited out the retry backoff'
+ assert chunks(cancelled_updates) == [], cancelled_updates
+ cancelled_posts = [row for row in map(json.loads, log.read_text().splitlines())
+ if row['event'] == 'post'][before:]
+ assert len(cancelled_posts) == 4, cancelled_posts
+ assert len({row['sha256'] for row in cancelled_posts}) == 1, cancelled_posts
+ after_cancel, after_cancel_updates = call('session/prompt', {'sessionId':sid, 'prompt':[{'type':'text', 'text':'Try once more explicitly.'}]}, 8)
+ assert after_cancel.get('result', {}).get('stopReason') == 'end_turn', after_cancel
+ assert chunks(after_cancel_updates) == ['OK'], after_cancel_updates
+ print('PASS ACP retry cancellation: interrupts backoff, no extra POST or answer, same session remains usable')
+ print('PASS ACP HTTP/2 retry chrome: recovery, six-attempt exhaustion, explicit retry; no retry text in answers')
finally:
fixture.terminate(worker)
fixture.terminate(server)
diff --git a/scripts/test-retry-cancellation.py b/scripts/test-retry-cancellation.py
new file mode 100644
index 00000000..5812d706
--- /dev/null
+++ b/scripts/test-retry-cancellation.py
@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""Offline retry/cancel races: ACP v1/v2 and JSON, transport/429/503 failures.
+
+Uses ephemeral loopback ports and isolated homes. No real credentials or model
+calls. A retry trace row synchronizes cancellation with the actual backoff.
+"""
+import hashlib
+import http.server
+import json
+import os
+from pathlib import Path
+import queue
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+class Model:
+ def __init__(self, kind, failures=-1, delay_ms=30000):
+ self.kind, self.failures, self.delay_ms = kind, failures, delay_ms
+ self.lock = threading.Lock()
+ self.posts = []
+ model = self
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ protocol_version = 'HTTP/1.1'
+
+ def log_message(self, *_args):
+ pass
+
+ def do_GET(self):
+ self.respond(200, b'{"data":[{"id":"fixture"}]}', 'application/json')
+
+ def respond(self, status, body, content_type):
+ self.send_response(status)
+ self.send_header('content-type', content_type)
+ self.send_header('content-length', str(len(body)))
+ self.send_header('connection', 'close')
+ if status in (429, 503):
+ self.send_header('retry-after-ms', str(model.delay_ms))
+ self.end_headers()
+ try:
+ self.wfile.write(body)
+ self.wfile.flush()
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+ self.close_connection = True
+
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers['content-length']))
+ with model.lock:
+ model.posts.append((hashlib.sha256(body).hexdigest(), time.monotonic()))
+ fail = model.failures != 0
+ if model.failures > 0:
+ model.failures -= 1
+ if fail and model.kind == 'transport':
+ self.close_connection = True
+ self.connection.shutdown(socket.SHUT_RDWR)
+ self.connection.close()
+ elif fail:
+ self.respond(int(model.kind), b'{"error":{"message":"fixture transient failure"}}', 'application/json')
+ else:
+ reply = ('data: {"choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":null}]}\n\n'
+ 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n'
+ 'data: [DONE]\n\n').encode()
+ self.respond(200, reply, 'text/event-stream')
+
+ self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler)
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+
+ def recover(self):
+ with self.lock:
+ self.failures = 0
+
+ def requests(self):
+ with self.lock:
+ return list(self.posts)
+
+ def close(self):
+ self.server.shutdown()
+ self.server.server_close()
+ self.thread.join(timeout=5)
+
+
+class Client:
+ def __init__(self, binary, root, model, mode):
+ self.root, self.mode = root, mode
+ self.seen, self.events = [], queue.Queue()
+ self.counter = 0
+ self.sid = None
+ (root/'mcp.json').write_text('{"mcpServers":{}}')
+ shim = root/'bin'
+ shim.mkdir()
+ (shim/'security').write_text('#!/bin/sh\nexit 44\n')
+ (shim/'security').chmod(0o700)
+ env = dict(HOME=str(root), USERPROFILE=str(root),
+ PATH=str(shim)+os.pathsep+os.defpath, TERM='dumb',
+ AI_GATEWAY_API_KEY='fixture', GRAFF_NO_TELEMETRY='1',
+ GRAFF_VERCEL_URL=f'http://127.0.0.1:{model.server.server_port}/v1/chat/completions',
+ GRAFF_MCP_CONFIG=str(root/'mcp.json'), GRAFF_FLEET='off',
+ GRAFF_NO_ADOPT='1', GRAFF_NO_SMOLIFY='1', GRAFF_NO_CODEDB_GUARD='1')
+ if 'SystemRoot' in os.environ:
+ env['SystemRoot'] = os.environ['SystemRoot']
+ if mode == 'acp2':
+ env['GRAFF_ACP_V2'] = '1'
+ args = [str(binary), '--json'] if mode == 'json' else [str(binary), 'acp']
+ self.stderr = (root/'stderr').open('w')
+ self.proc = subprocess.Popen(args + ['--model', 'vercel', '--yolo'],
+ cwd=root, env=env, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=self.stderr,
+ text=True, start_new_session=os.name == 'posix')
+
+ def read():
+ try:
+ for line in self.proc.stdout:
+ self.events.put(json.loads(line))
+ except Exception as error:
+ self.events.put(error)
+ finally:
+ self.events.put(None)
+ self.reader = threading.Thread(target=read, daemon=True)
+ self.reader.start()
+ try:
+ if mode != 'json':
+ init = self.call('initialize', {'protocolVersion':2 if mode == 'acp2' else 1})
+ assert init['result']['protocolVersion'] == (2 if mode == 'acp2' else 1), init
+ self.sid = self.call('session/new', {'cwd':str(root), 'mcpServers':[]})['result']['sessionId']
+ except BaseException:
+ self.close()
+ raise
+
+ def send(self, value):
+ self.proc.stdin.write(json.dumps(value) + '\n')
+ self.proc.stdin.flush()
+
+ def request(self, method, params):
+ self.counter += 1
+ self.send(dict(jsonrpc='2.0', id=self.counter, method=method, params=params))
+ return self.counter
+
+ def until(self, predicate, timeout=8):
+ deadline = time.monotonic() + timeout
+ while True:
+ value = self.events.get(timeout=max(.01, deadline-time.monotonic()))
+ assert isinstance(value, dict), ('invalid/closed protocol stream', value)
+ self.seen.append(value)
+ if predicate(value):
+ return value
+ assert time.monotonic() < deadline, ('protocol deadline', self.seen[-3:])
+
+ def call(self, method, params):
+ identifier = self.request(method, params)
+ return self.until(lambda value: value.get('id') == identifier)
+
+ def trace(self):
+ return [json.loads(line) for path in (self.root/'.graff/traces').glob('*.jsonl')
+ for line in path.read_text().splitlines(keepends=True) if line.endswith('\n')]
+
+ def begin(self):
+ self.start = len(self.seen)
+ self.retry_count = sum(row.get('ev') == 'retry' for row in self.trace())
+ if self.mode == 'json':
+ self.send({'type':'user', 'text':'Say OK.'})
+ return None
+ return self.request('session/prompt', {'sessionId':self.sid, 'prompt':[{'type':'text', 'text':'Say OK.'}]})
+
+ def wait_retry(self):
+ deadline = time.monotonic() + 8
+ while sum(row.get('ev') == 'retry' for row in self.trace()) <= self.retry_count:
+ assert self.proc.poll() is None, 'worker exited before retry'
+ assert time.monotonic() < deadline, 'retry backoff not reached'
+ time.sleep(.01)
+
+ def cancel(self, wrong=False, client=False):
+ if self.mode == 'json':
+ self.send({'type':'cancel'})
+ else:
+ params = {'sessionId':'unrelated-session' if wrong else self.sid}
+ if client:
+ params['_meta'] = {'graff/cancelSource':'client'}
+ self.send(dict(jsonrpc='2.0', method='session/cancel', params=params))
+
+ def finish(self, identifier, cancelled=False):
+ if self.mode == 'json':
+ value = self.until(lambda event: event.get('type') in ('error', 'turn'))
+ assert value.get('type') == ('error' if cancelled else 'turn'), value
+ assert value.get('message' if cancelled else 'text') == ('turn cancelled' if cancelled else 'OK'), value
+ elif self.mode == 'acp2':
+ value = self.until(lambda event: event.get('params', {}).get('update', {}).get('state') == 'idle')
+ assert value['params']['update']['stopReason'] == ('cancelled' if cancelled else 'end_turn'), value
+ acks = [event for event in self.seen[self.start:] if event.get('id') == identifier]
+ assert len(acks) == 1 and acks[0].get('result', {}).get('messageId'), acks
+ else:
+ value = self.until(lambda event: event.get('id') == identifier)
+ assert value.get('result', {}).get('stopReason') == ('cancelled' if cancelled else 'end_turn'), value
+ wire = self.seen[self.start:]
+ assert not any(token in json.dumps(wire) for token in ('network error:', 'retrying in', '· retry')), wire
+ if self.mode == 'json':
+ sequences = [event['seq'] for event in self.seen]
+ assert sequences == list(range(sequences[0], sequences[0] + len(sequences))), sequences
+ return value
+
+ def saved(self):
+ return [message for path in (self.root/'.graff/sessions').glob('*.session.json')
+ for message in json.loads(path.read_text()).get('messages', [])]
+
+ def close(self):
+ if self.proc.poll() is None:
+ try:
+ if os.name == 'posix':
+ os.killpg(self.proc.pid, signal.SIGKILL)
+ else:
+ self.proc.kill()
+ except ProcessLookupError:
+ pass
+ self.proc.wait(timeout=8)
+ self.reader.join(timeout=5)
+ assert not self.reader.is_alive(), 'protocol reader did not stop'
+ for stream in (self.proc.stdin, self.proc.stdout):
+ stream.close()
+ self.stderr.close()
+
+
+def exercise(binary, mode, kind, scenario):
+ with tempfile.TemporaryDirectory(prefix='graff-retry-cancel-') as temp:
+ model = Model(kind, failures=1 if scenario == 'wrong' else -1,
+ delay_ms=500 if scenario == 'wrong' else 30000)
+ client = None
+ try:
+ client = Client(binary, Path(temp), model, mode)
+ marker = None
+ identifier = client.begin()
+ if scenario == 'immediate':
+ client.cancel()
+ else:
+ client.wait_retry()
+ assert len(model.requests()) == 1, 'unexpected retry before cancellation'
+ client.cancel(wrong=scenario == 'wrong', client=scenario == 'client')
+ client.cancel(wrong=scenario == 'wrong', client=scenario == 'client')
+ started = time.monotonic()
+ client.finish(identifier, cancelled=scenario != 'wrong')
+ elapsed = time.monotonic() - started
+ if scenario == 'wrong':
+ posts = model.requests()
+ assert len(posts) == 2 and posts[0][0] == posts[1][0], posts
+ if kind != 'transport':
+ assert posts[1][1] - posts[0][1] >= .45, 'Retry-After bypassed'
+ else:
+ assert elapsed < 2, ('cancel waited out backoff', elapsed)
+ assert len(model.requests()) <= 1, model.requests()
+ source = 'json_cancel' if mode == 'json' else 'acp_client' if scenario == 'client' else 'acp_cancel'
+ if scenario != 'immediate':
+ assert f'interrupted source={source}' in json.dumps(client.trace()), source
+ marker = 'stopped by the client, not the user' if scenario == 'client' else 'interrupted by user'
+ if scenario == 'eof':
+ # The documented checkpoint boundary is graceful exit: JSON may
+ # publish its error before the synchronous save has returned.
+ client.proc.stdin.close()
+ assert client.proc.wait(timeout=8) == 0, 'worker did not exit cleanly'
+ assert marker in json.dumps(client.saved()), ('cancelled history lost at EOF', client.saved())
+ assert len(model.requests()) == 1, 'EOF dispatched another retry'
+ print(f'PASS {mode}/{kind}/eof: graceful exit drains the cancelled checkpoint', flush=True)
+ return
+ # Duplicate/late cancels while idle must not poison the next turn.
+ client.cancel()
+ client.cancel()
+ model.recover()
+ before = len(model.requests())
+ followup = client.begin()
+ client.finish(followup)
+ assert len(model.requests()) == before + 1, 'follow-up was lost or replayed'
+ # JSON emits its error before persisting history. The next completed
+ # turn is a barrier proving the prior interrupted turn was saved.
+ if marker:
+ assert marker in json.dumps(client.saved()), (scenario, client.saved())
+ print(f'PASS {mode}/{kind}/{scenario}: correct scope, clean wire, same-worker continuation', flush=True)
+ finally:
+ if client:
+ client.close()
+ model.close()
+
+
+def main():
+ binary = Path(sys.argv[1] if len(sys.argv) > 1 else 'zig-out/bin/graff').resolve()
+ cases = [(mode, kind, 'cancel') for mode in ('acp1', 'acp2', 'json')
+ for kind in ('transport', '429', '503')]
+ cases += [(mode, kind, 'wrong') for mode in ('acp1', 'acp2') for kind in ('transport', '429')]
+ cases += [(mode, '503', 'immediate') for mode in ('acp1', 'acp2', 'json')]
+ cases += [(mode, '429', 'client') for mode in ('acp1', 'acp2')]
+ cases += [(mode, '503', 'eof') for mode in ('acp1', 'acp2', 'json')]
+ for case in cases:
+ exercise(binary, *case)
+ print(f'PASS retry cancellation matrix: {len(cases)} scenarios', flush=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/agent_interrupt.zig b/src/agent_interrupt.zig
index 900d08f1..77a0e68a 100644
--- a/src/agent_interrupt.zig
+++ b/src/agent_interrupt.zig
@@ -91,18 +91,19 @@ pub fn restoreStdin(orig: tty.RawState) void {
tty.restore(orig);
}
-/// Sleep `ms` watching stdin for Esc (when the root is on a TTY), so the
-/// user can cancel a retry backoff instead of waiting it out.
+/// Sleep `ms` watching both the engine cancel flag (ACP/JSON/hosted clients)
+/// and stdin Esc when the root is on a TTY. Never wait out a cancelled backoff.
pub fn sleepInterruptible(self: *Agent, ms: u64) error{Interrupted}!void {
+ if (Agent.esc_cancel.load(.acquire)) return error.Interrupted;
const watch = !self.sub and self.in != null and main_mod.use_color and !main_mod.json_mode;
const orig_tio: ?tty.RawState = if (watch) rawNonblockStdin() else null;
defer if (orig_tio) |o| tty.restore(o);
var left = ms;
while (left > 0) {
const chunk = @min(left, 100);
- self.io.sleep(.fromMilliseconds(@intCast(chunk)), .awake) catch {};
+ self.io.sleep(.fromMilliseconds(@intCast(chunk)), .awake) catch return error.Interrupted;
left -= chunk;
- if (orig_tio != null and escPressed(true)) return error.Interrupted;
+ if (Agent.esc_cancel.load(.acquire) or (orig_tio != null and escPressed(true))) return error.Interrupted;
}
}
@@ -122,6 +123,92 @@ pub fn sseIndex(obj: std.json.ObjectMap) ?usize {
return @intCast(ix.integer);
}
+const BackoffSleepProbe = struct {
+ calls: u64 = 0,
+ total_ms: u64 = 0,
+ max_ms: u64 = 0,
+ cancel_at: u64 = 0,
+ cancel_io: bool = false,
+
+ fn sleep(ctx: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
+ const self: *@This() = @ptrCast(@alignCast(ctx.?));
+ const ms: u64 = @intCast(timeout.duration.raw.toMilliseconds());
+ std.debug.assert(timeout.duration.clock == .awake);
+ self.calls += 1;
+ self.total_ms += ms;
+ self.max_ms = @max(self.max_ms, ms);
+ if (self.cancel_io) return error.Canceled;
+ if (self.calls == self.cancel_at) Agent.esc_cancel.store(true, .release);
+ }
+};
+
+test "retry backoff preserves its full duration in bounded sleep chunks" {
+ const previous = Agent.esc_cancel.swap(false, .acq_rel);
+ defer Agent.esc_cancel.store(previous, .release);
+ var vtable = std.testing.io.vtable.*;
+ vtable.sleep = BackoffSleepProbe.sleep;
+ var agent: Agent = undefined;
+ agent.in = null;
+ agent.sub = false;
+ for ([_]u64{ 0, 1, 99, 100, 101, 250, 65_535 }) |ms| {
+ var probe: BackoffSleepProbe = .{};
+ agent.io = .{ .userdata = &probe, .vtable = &vtable };
+ try sleepInterruptible(&agent, ms);
+ try std.testing.expectEqual(ms, probe.total_ms);
+ try std.testing.expectEqual((ms + 99) / 100, probe.calls);
+ try std.testing.expect(probe.max_ms <= 100);
+ }
+}
+
+test "retry backoff stops when cancellation arrives during a sleep chunk" {
+ const previous = Agent.esc_cancel.load(.acquire);
+ defer Agent.esc_cancel.store(previous, .release);
+ var vtable = std.testing.io.vtable.*;
+ vtable.sleep = BackoffSleepProbe.sleep;
+ var agent: Agent = undefined;
+ agent.in = null;
+ for ([_]bool{ false, true }) |sub| {
+ agent.sub = sub;
+ for ([_]u64{ 1, 2 }) |cancel_at| {
+ Agent.esc_cancel.store(false, .release);
+ var probe: BackoffSleepProbe = .{ .cancel_at = cancel_at };
+ agent.io = .{ .userdata = &probe, .vtable = &vtable };
+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 10_000));
+ try std.testing.expectEqual(cancel_at, probe.calls);
+ try std.testing.expectEqual(cancel_at * 100, probe.total_ms);
+ }
+ }
+}
+
+test "retry backoff propagates I/O cancellation without another sleep" {
+ const previous = Agent.esc_cancel.swap(false, .acq_rel);
+ defer Agent.esc_cancel.store(previous, .release);
+ var vtable = std.testing.io.vtable.*;
+ vtable.sleep = BackoffSleepProbe.sleep;
+ var agent: Agent = undefined;
+ agent.in = null;
+ agent.sub = false;
+ var probe: BackoffSleepProbe = .{ .cancel_io = true };
+ agent.io = .{ .userdata = &probe, .vtable = &vtable };
+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 10_000));
+ try std.testing.expectEqual(@as(u64, 1), probe.calls);
+ try std.testing.expect(!Agent.esc_cancel.load(.acquire));
+}
+
+test "retry backoff respects a pending non-TTY cancellation without sleeping" {
+ const previous = Agent.esc_cancel.swap(true, .acq_rel);
+ defer Agent.esc_cancel.store(previous, .release);
+ var agent: Agent = undefined;
+ agent.in = null;
+ for ([_]bool{ false, true }) |sub| {
+ agent.sub = sub;
+ // io is deliberately undefined: even a long wait must return before
+ // touching it, including for a child or a zero-duration backoff.
+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 60_000));
+ try std.testing.expectError(error.Interrupted, sleepInterruptible(&agent, 0));
+ }
+}
+
test { // #728: cancel_source has no other path into the test root
_ = cancel_source;
_ = @import("agent_interrupt_repl_tests.zig");
diff --git a/src/agent_request.zig b/src/agent_request.zig
index cf7415f3..5a41a098 100644
--- a/src/agent_request.zig
+++ b/src/agent_request.zig
@@ -374,9 +374,8 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
// Transport flakes back off; rapid retries against a
// just-closed keep-alive re-fail (#86). Cap: 4s/6 tries.
const delay_ms = jitter.ms(self.io, RetryPlan.delayMs(throttled, attempt));
- @import("turn_chrome.zig").emitRetryNotice(self.io, @errorName(err), attempt + 1, max_attempts);
if (scratch.showRecoveredTransportRetry(self.call_kind))
- try scratch.announceNetworkRetry(self, err, delay_ms, attempt + 1, max_attempts);
+ @import("turn_chrome.zig").emitRetryNotice(self.io, @errorName(err), attempt + 1, max_attempts);
// Same trace breadcrumb the 429/5xx branch leaves: a
// transport-flake retry is otherwise invisible in the
// session trace, hiding how flaky a provider really is.
diff --git a/src/agent_request_scratch.zig b/src/agent_request_scratch.zig
index daa8b5c8..0f0ceb98 100644
--- a/src/agent_request_scratch.zig
+++ b/src/agent_request_scratch.zig
@@ -29,16 +29,6 @@ pub fn showRecoveredTransportRetry(kind: run_budget_mod.CallKind) bool {
return kind != .recap;
}
-/// Preserve the existing notice in JSON/ACP, where say() is deliberately silent.
-pub fn announceNetworkRetry(self: *Agent, err: anyerror, delay_ms: u64, attempt: usize, limit: usize) !void {
- try self.say("[network error: {t} — retrying in {d}ms ({d}/{d})]\n", .{ err, delay_ms, attempt, limit });
- if (@import("main.zig").json_mode and !self.sub) {
- var buf: [192]u8 = undefined;
- const text = try std.fmt.bufPrint(&buf, "[network error: {t} — retrying in {d}ms ({d}/{d})]", .{ err, delay_ms, attempt, limit });
- self.emit(.{ .type = "text", .text = text });
- }
-}
-
/// A response body of only SSE comment lines (`: OPENROUTER PROCESSING` …)
/// means the gateway queued us and never produced tokens — back off and re-ask
/// like a 5xx instead of dying on an "unparseable" JSON parse.
diff --git a/src/tool_pulse.zig b/src/tool_pulse.zig
index 7178af4a..5472d58d 100644
--- a/src/tool_pulse.zig
+++ b/src/tool_pulse.zig
@@ -70,7 +70,8 @@ pub fn emitNotice(io: Io, comptime fmt: []const u8, args: anytype) void {
w.flush() catch {};
}
-/// Line-REPL only. Hosted TUI/ACP still get the pulse via `hostedSink`.
+/// Line-REPL only. Hosted frontends get the pulse via a bound `hostedSink`.
+/// ACP has no bound chrome sink, so --json keeps retries off its answer stream.
/// `-p` / `--json` keep stdout as the answer (in-house atomic-symlink-write
/// captured `· turn still going ·` when the pulse rode `g_out`).
pub fn chromeGoesToStdout() bool {
diff --git a/src/turn_chrome.zig b/src/turn_chrome.zig
index b2993c2c..37ff3626 100644
--- a/src/turn_chrome.zig
+++ b/src/turn_chrome.zig
@@ -88,6 +88,69 @@ test "formatRetryNotice names HungRequest and UnknownHostName with attempt n/N"
try std.testing.expectEqualStrings("· retry 1/6 · UnknownHostName", formatRetryNotice(&buf, "UnknownHostName", 1, 6));
}
+test "retry chrome stays out of ACP JSON and one-shot answer writers" {
+ const main_mod = @import("main.zig");
+ const prev_json = main_mod.json_mode;
+ const prev_unattended = main_mod.unattended;
+ const prev_out = main_mod.g_out;
+ defer {
+ main_mod.json_mode = prev_json;
+ main_mod.unattended = prev_unattended;
+ main_mod.g_out = prev_out;
+ }
+ var output: Io.Writer.Allocating = .init(std.testing.allocator);
+ defer output.deinit();
+ main_mod.g_out = &output.writer;
+ for ([_][2]bool{ .{ true, false }, .{ false, true }, .{ true, true } }) |mode| {
+ main_mod.json_mode = mode[0];
+ main_mod.unattended = mode[1];
+ emitRetryNotice(undefined, "UnknownHostName", 1, 6);
+ emitRetryNotice(undefined, "HungRequest", 2, 6);
+ try std.testing.expectEqualStrings("", output.written());
+ }
+ main_mod.json_mode = false;
+ main_mod.unattended = false;
+ emitRetryNotice(undefined, "UnknownHostName", 1, 6);
+ emitRetryNotice(undefined, "HungRequest", 2, 6);
+ const text = output.written();
+ try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, text, "\n"));
+ try std.testing.expect(std.mem.indexOf(u8, text, "· retry 1/6 · UnknownHostName") != null);
+ try std.testing.expect(std.mem.indexOf(u8, text, "· retry 2/6 · HungRequest") != null);
+ try std.testing.expect(std.mem.indexOf(u8, text, "[network error:") == null);
+}
+
+test "hosted retry chrome is one dim presentation notice, not a durable answer" {
+ const sinks = @import("engine_sink.zig");
+ const events = @import("engine_events.zig");
+ const Capture = struct {
+ count: usize = 0,
+ dim: bool = false,
+ durable: bool = false,
+ text: [120]u8 = undefined,
+ len: usize = 0,
+ fn emit(ctx: *anyopaque, stamped: sinks.Stamped) void {
+ const self: *@This() = @ptrCast(@alignCast(ctx));
+ self.count += 1;
+ self.durable = events.durable(stamped.event);
+ if (stamped.event == .session_notice) {
+ const notice = stamped.event.session_notice;
+ self.dim = notice.tone == .dim;
+ self.len = notice.text.len;
+ @memcpy(self.text[0..self.len], notice.text);
+ }
+ }
+ };
+ var capture: Capture = .{};
+ const vt: sinks.VTable = .{ .emit = Capture.emit, .durable = false };
+ sinks.bindTurnSink(.{ .ctx = &capture, .vt = &vt });
+ defer sinks.unbindTurnSink();
+ emitRetryNotice(undefined, "HungRequest", 3, 6);
+ try std.testing.expectEqual(@as(usize, 1), capture.count);
+ try std.testing.expect(capture.dim);
+ try std.testing.expect(!capture.durable);
+ try std.testing.expectEqualStrings("· retry 3/6 · HungRequest", capture.text[0..capture.len]);
+}
+
test "formatTurnPulse stays off the transcript (ADR 0021)" {
var buf: [80]u8 = undefined;
try std.testing.expectEqualStrings("", formatTurnPulse(&buf, 3, 0, 11));