graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Streams: ride out repeated connection drops; ACP keeps the cut reply
justrach68cf7c9d63b5
diff --git a/build.zig b/build.zigindex c55644de..87a0c2ff 100644--- a/build.zig+++ b/build.zig@@ -221,6 +221,9 @@ pub fn build(b: *std.Build) void {const acp_compaction_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-compaction.py" });acp_compaction_test.addArtifactArg(exe);test_step.dependOn(&acp_compaction_test.step);+ const acp_stream_drop_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-stream-drop.py" });+ acp_stream_drop_test.addArtifactArg(exe);+ test_step.dependOn(&acp_stream_drop_test.step);const run_tool_budget_test = b.addSystemCommand(&.{ "python3", "scripts/test-run-tool-budget.py" });run_tool_budget_test.addArtifactArg(exe);test_step.dependOn(&run_tool_budget_test.step);diff --git a/scripts/test-acp-stream-drop.py b/scripts/test-acp-stream-drop.pynew file mode 100644index 00000000..1031ca05--- /dev/null+++ b/scripts/test-acp-stream-drop.py@@ -0,0 +1,82 @@+#!/usr/bin/env python3+"""#1639 over ACP: a turn whose stream keeps dropping ends with the same saved+row as the REPL ("[response ended early: connection dropped]"), not with no+assistant reply at all. GRAFF_FORCE_DROP_ALWAYS makes every live attempt return+error.StreamDropped before any network call, so no provider or key is used.+"""+import json+import os+from pathlib import Path+import queue+import signal+import subprocess+import sys+import tempfile+import threading+import time++MARKER = '[response ended early: connection dropped]'+++def main():+ binary = str(Path(sys.argv[1] if len(sys.argv) > 1 else 'zig-out/bin/graff').resolve())+ with tempfile.TemporaryDirectory(prefix='graff-acp-drop-') as temp:+ env = {k: v for k, v in os.environ.items() if not k.endswith('_API_KEY')}+ config = Path(temp) / 'mcp.json'+ config.write_text('{"mcpServers":{}}')+ env.update(HOME=temp, CODEGRAFF_API_KEY='fake-test-key', GRAFF_MCP_CONFIG=str(config),+ GRAFF_NO_TELEMETRY='1', GRAFF_FLEET='off', GRAFF_NO_SMOLIFY='1',+ GRAFF_NO_CODEDB_GUARD='1', GRAFF_FORCE_DROP_ALWAYS='1')+ proc = None+ try:+ proc = subprocess.Popen([binary, 'acp', '--yolo'], cwd=temp, env=env,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL, text=True, start_new_session=True)+ events = queue.Queue()++ def reader():+ for line in proc.stdout:+ try:+ events.put(json.loads(line))+ except ValueError:+ pass+ events.put(None)+ threading.Thread(target=reader, daemon=True).start()++ def call(method, params, id, budget=25):+ proc.stdin.write(json.dumps(dict(jsonrpc='2.0', method=method, params=params, id=id)) + '\n')+ proc.stdin.flush()+ deadline = time.monotonic() + budget+ while True:+ value = events.get(timeout=max(.01, deadline - time.monotonic()))+ assert value is not None, 'ACP worker exited before replying'+ if value.get('id') == id:+ return value++ call('initialize', {'protocolVersion': 1}, 1)+ sid = call('session/new', {}, 2)['result']['sessionId']+ # Every reconnect drops too, so the whole drop budget (with its+ # growing pauses) runs before the turn ends.+ failed = call('session/prompt', {'sessionId': sid, 'prompt': [{'type': 'text', 'text': 'hello'}]}, 3, budget=90)+ assert 'StreamDropped' in json.dumps(failed.get('error')), failed+ saved = list((Path(temp) / '.graff/sessions').glob('*.session.json'))+ assert saved, 'the failed turn was not saved'+ rows = [m for p in saved for m in json.loads(p.read_text()).get('messages', [])]+ texts = [json.dumps(m) for m in rows if isinstance(m, dict) and m.get('role') == 'assistant']+ assert any(MARKER in t for t in texts), f'no end marker saved: {texts}'+ assert not any('interrupted by user' in t for t in texts), texts+ print(f'PASS ACP stream drop: turn saved as {MARKER!r}, never a user interrupt')+ finally:+ if proc:+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except ProcessLookupError:+ pass+ proc.wait(timeout=2)+ for pipe in (proc.stdin, proc.stdout):+ if pipe:+ pipe.close()+++if __name__ == '__main__':+ main()diff --git a/src/acp_live_turn.zig b/src/acp_live_turn.zigindex 93b2afb5..519298b1 100644--- a/src/acp_live_turn.zig+++ b/src/acp_live_turn.zig@@ -182,6 +182,13 @@ pub const LiveTurn = struct {cancel_source.marker(self.cancelled_source.?),));}+ // #1639: like the REPL, a turn the transport ended keeps its+ // partial reply and says why, so the next prompt (and a resume)+ // sees where the answer stopped instead of no reply at all.+ if (!isolated and (err == error.StreamDropped or err == error.StreamStalled)) {+ const ended = try @import("stream_reconnect.zig").endedEarlyText(arena, self.root.partial_text.items, err == error.StreamStalled);+ try self.root.messages.append(try messages.textMessage(arena, "assistant", ended));+ }if (isolated) {const marker = if (err == error.Interrupted or err == error.Canceled)cancel_source.marker(self.cancelled_source.?)diff --git a/src/agent_request.zig b/src/agent_request.zigindex 2beda2e0..3c739e8e 100644--- a/src/agent_request.zig+++ b/src/agent_request.zig@@ -207,8 +207,8 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {// #56: bounded stream-stall / drop reconnect budget (codex's stream_max_retries analog), outside the rebuild loop so a WS reanchor can't reset it.self.stall = .{}; // #680: a fresh widen ladder + tripped budget per requestvar stall_retries: usize = 0;- const max_stall_retries: usize = 2;- const stall_reconnect_backoff_ms = 750; // brief pause before reconnecting on a fresh stream+ var drop_retries: usize = 0; // #1639: a peer close earns its own, larger budget (stream_reconnect.zig)+ const reconnect = @import("stream_reconnect.zig");var server_retries: usize = 0; // #opencode-parity: bounded retries for a transient in-stream server overloadvar openai404_retries: usize = 0; // #opencode-parity: bounded retries for OpenAI's spurious model 404sconst max_openai404_retries: usize = 2;@@ -282,24 +282,25 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {if (err == error.StreamStalled or err == error.StreamDropped) {const stalled = err == error.StreamStalled;const what: []const u8 = if (stalled) "stream stalled" else "stream dropped";- if (stall_retries < max_stall_retries) {- stall_retries += 1;+ const retries = if (stalled) &stall_retries else &drop_retries;+ if (retries.* < reconnect.maxRetries(stalled)) {+ retries.* += 1;// #56: give a fresh WS one reconnect; if it stalls again,// fall back to the reliable SSE transport for the rest of// the session — codex's WS→SSE fallback, and consistent// with how a WS transport error is already handled (ws_off).- if (stall_retries > 1) self.ws_off = true;+ if (stall_retries + drop_retries > 1) self.ws_off = true;if (self.tracer) |tr| tr.note("stream_retry", what);if (telemetry.g_telem) |t| t.errorEvent("stream_retry", what);- @import("agent_stream.zig").noteStallRetry(self, stall_retries); // #680: clears the partial (fresh stream, no concat) + widens the between-lines budget+ @import("agent_stream.zig").noteStallRetry(self, stall_retries); // #680: clears the partial (fresh stream, no concat) + widens the between-lines budget (stalls only)self.closeCodexWs(); // tear down the dead WS + null codex_prev_id for a full re-send (no-op off codex WS)- self.sleepInterruptible(jitter.ms(self.io, stall_reconnect_backoff_ms)) catch return error.Interrupted; // #1274: desync parallel reconnects+ self.sleepInterruptible(jitter.ms(self.io, reconnect.backoffMs(stalled, retries.*))) catch return error.Interrupted; // #1274: desync parallel reconnectscontinue :rebuild;}// Budget gone: the turn is ending. Mid-turn cuts stayed silent.@import("engine_sink.zig").forAgent(self).emit(self.io, .{ .transport_aborted = .{ .reason = if (stalled) .stalled else .dropped, .turn_ending = true } });if (stalled) {- self.last_api_error = @import("agent_stream.zig").stallGiveUpMessage(self, max_stall_retries); // #680: reports the wait that tripped, not the configured total+ self.last_api_error = @import("agent_stream.zig").stallGiveUpMessage(self, reconnect.max_stall_retries); // #680: reports the wait that tripped, not the configured totalif (telemetry.g_telem) |t| t.errorEvent("stream_stall", self.last_api_error orelse "stream stalled");if (self.tracer) |tr| tr.note("stream_stall", self.last_api_error orelse "stream stalled");return error.StreamStalled;diff --git a/src/main.zig b/src/main.zigindex a2b160f7..2efddacb 100644--- a/src/main.zig+++ b/src/main.zig@@ -566,7 +566,7 @@ test { // ── Unit tests (`zig build test`): pull in tests from imported modu_ = .{ @import("prompt_astra.zig"), @import("prompt_guidance.zig"), @import("oauth_chatgpt.zig"), @import("chatgpt_reauth.zig") }; // per-model guidance (Astra, GPT-5.6); ChatGPT plan sign-in (ADR 0253)_ = .{ @import("agent_empty_completion.zig"), @import("agent_model_loop.zig"), @import("publication_policy_tests.zig"), @import("jobs_completion_tests.zig") };_ = .{ @import("provider_routing_tests.zig"), @import("codegraff_messages_tests.zig"), @import("tool_result_empty_tests.zig"), @import("history_wire.zig"), @import("path_lookup.zig"), @import("agents_prepare.zig"), @import("catalog_price.zig"), @import("readline_paste_number_tests.zig"), @import("test_hooks.zig"), @import("peer_wake_loop.zig") };- _ = .{ @import("list_dir_nearmiss.zig"), @import("tool_preview_tests.zig") };+ _ = .{ @import("list_dir_nearmiss.zig"), @import("tool_preview_tests.zig"), @import("stream_reconnect.zig") };_ = @import("repo_map.zig");_ = @import("agent_overflow_tests.zig"); // #414: and, through it, agent_overflow.zig's table tests_ = @import("agent_gateway_retry.zig"); // #1019: flake vs overflow classification must stay reachablediff --git a/src/mainloop.zig b/src/mainloop.zigindex 3daadc97..3f64a6d9 100644--- a/src/mainloop.zig+++ b/src/mainloop.zig@@ -446,10 +446,7 @@ pub fn run(ctx: *Ctx) !void {// harness gave up, NOT the user (#134) — keep the partial but// tag it a stall, and emit the structured error for --json.const partial = std.mem.trim(u8, ctx.root.partial_text.items, " \t\r\n");- const marker: []const u8 = if (partial.len > 0)- try std.fmt.allocPrint(ctx.arena, "{s}\n\n[response ended early: stream stalled]", .{partial})- else- "[response ended early: stream stalled]";+ const marker = try @import("stream_reconnect.zig").endedEarlyText(ctx.arena, partial, true);try ctx.root.messages.append(try messages.textMessage(ctx.arena, "assistant", marker));if (main_mod.json_mode) {ctx.root.emit(.{ .type = "error", .message = ctx.root.last_api_error orelse "stream stalled — ending turn" });@@ -468,10 +465,7 @@ pub fn run(ctx: *Ctx) !void {// the user (#133). Keep the partial, tag it a drop (never// "[response interrupted by user]"), and emit the --json error.const partial = std.mem.trim(u8, ctx.root.partial_text.items, " \t\r\n");- const marker: []const u8 = if (partial.len > 0)- try std.fmt.allocPrint(ctx.arena, "{s}\n\n[response ended early: connection dropped]", .{partial})- else- "[response ended early: connection dropped]";+ const marker = try @import("stream_reconnect.zig").endedEarlyText(ctx.arena, partial, false);try ctx.root.messages.append(try messages.textMessage(ctx.arena, "assistant", marker));if (main_mod.json_mode) {ctx.root.emit(.{ .type = "error", .message = ctx.root.last_api_error orelse "stream dropped — ending turn" });diff --git a/src/stream_reconnect.zig b/src/stream_reconnect.zignew file mode 100644index 00000000..62149fdf--- /dev/null+++ b/src/stream_reconnect.zig@@ -0,0 +1,69 @@+//! #1639: reconnecting a response stream that died before its terminal event.+//!+//! A stall (no bytes for the between-lines budget) and a drop (the peer closed+//! or reset the connection) each re-send the whole request on a fresh stream.+//! They get separate budgets: a drop says nothing about the model, and a link+//! that kills long-lived streams intermittently would otherwise end a healthy+//! turn after three cuts. Drops therefore get more attempts and a pause that+//! doubles, so a flapping connection has time to recover.+//!+//! Whichever budget runs out, the turn keeps its partial reply tagged with why+//! it ended — the same row on the REPL and over ACP.++const std = @import("std");++pub const max_stall_retries: usize = 2;+pub const max_drop_retries: usize = 5;++const base_backoff_ms: u64 = 750;+const max_backoff_ms: u64 = 8_000;++/// Pause before reconnect `attempt` (1-based, counted per kind). Stalls keep+/// the flat pause; drops double it per attempt up to `max_backoff_ms`.+pub fn backoffMs(stalled: bool, attempt: usize) u64 {+ if (stalled or attempt <= 1) return base_backoff_ms;+ const shift: u6 = @intCast(@min(attempt - 1, 4));+ return @min(base_backoff_ms << shift, max_backoff_ms);+}++pub fn maxRetries(stalled: bool) usize {+ return if (stalled) max_stall_retries else max_drop_retries;+}++/// The assistant row for a turn the transport ended: the partial reply (if+/// any), then a marker saying it was cut short and why — never a user cancel.+pub fn endedEarlyText(arena: std.mem.Allocator, partial_raw: []const u8, stalled: bool) ![]const u8 {+ const marker = if (stalled)+ "[response ended early: stream stalled]"+ else+ "[response ended early: connection dropped]";+ const partial = std.mem.trim(u8, partial_raw, " \t\r\n");+ if (partial.len == 0) return marker;+ return std.fmt.allocPrint(arena, "{s}\n\n{s}", .{ partial, marker });+}++test "drops get a larger, growing reconnect budget than stalls" {+ try std.testing.expect(max_drop_retries > max_stall_retries);+ try std.testing.expectEqual(@as(u64, 750), backoffMs(true, 1));+ try std.testing.expectEqual(@as(u64, 750), backoffMs(true, 2));+ try std.testing.expectEqual(@as(u64, 750), backoffMs(false, 1));+ try std.testing.expectEqual(@as(u64, 1500), backoffMs(false, 2));+ try std.testing.expectEqual(@as(u64, 3000), backoffMs(false, 3));+ try std.testing.expectEqual(@as(u64, 6000), backoffMs(false, 4));+ try std.testing.expectEqual(@as(u64, 8000), backoffMs(false, 5));+ try std.testing.expectEqual(@as(u64, 8000), backoffMs(false, 50));+}++test "a transport-ended turn keeps its partial and says why it ended" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ try std.testing.expectEqualStrings(+ "half a reply\n\n[response ended early: connection dropped]",+ try endedEarlyText(arena, " half a reply \n", false),+ );+ try std.testing.expectEqualStrings(+ "[response ended early: stream stalled]",+ try endedEarlyText(arena, " \n", true),+ );+}