Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Streams: ride out repeated connection drops; ACP keeps the cut reply

justrach68cf7c9d63b5
diff --git a/build.zig b/build.zig
index c55644de..87a0c2ff 100644
--- a/build.zig
+++ b/build.zig
@@ -221,6 +221,9 @@ pub fn build(b: *std.Build) void {
const acp_compaction_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-compaction.py" });
acp_compaction_test.addArtifactArg(exe);
test_step.dependOn(&acp_compaction_test.step);
+ const acp_stream_drop_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-stream-drop.py" });
+ acp_stream_drop_test.addArtifactArg(exe);
+ test_step.dependOn(&acp_stream_drop_test.step);
const run_tool_budget_test = b.addSystemCommand(&.{ "python3", "scripts/test-run-tool-budget.py" });
run_tool_budget_test.addArtifactArg(exe);
test_step.dependOn(&run_tool_budget_test.step);
diff --git a/scripts/test-acp-stream-drop.py b/scripts/test-acp-stream-drop.py
new file mode 100644
index 00000000..1031ca05
--- /dev/null
+++ b/scripts/test-acp-stream-drop.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""#1639 over ACP: a turn whose stream keeps dropping ends with the same saved
+row as the REPL ("[response ended early: connection dropped]"), not with no
+assistant reply at all. GRAFF_FORCE_DROP_ALWAYS makes every live attempt return
+error.StreamDropped before any network call, so no provider or key is used.
+"""
+import json
+import os
+from pathlib import Path
+import queue
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+MARKER = '[response ended early: connection dropped]'
+
+
+def main():
+ binary = str(Path(sys.argv[1] if len(sys.argv) > 1 else 'zig-out/bin/graff').resolve())
+ with tempfile.TemporaryDirectory(prefix='graff-acp-drop-') as temp:
+ env = {k: v for k, v in os.environ.items() if not k.endswith('_API_KEY')}
+ config = Path(temp) / 'mcp.json'
+ config.write_text('{"mcpServers":{}}')
+ env.update(HOME=temp, CODEGRAFF_API_KEY='fake-test-key', GRAFF_MCP_CONFIG=str(config),
+ GRAFF_NO_TELEMETRY='1', GRAFF_FLEET='off', GRAFF_NO_SMOLIFY='1',
+ GRAFF_NO_CODEDB_GUARD='1', GRAFF_FORCE_DROP_ALWAYS='1')
+ proc = None
+ try:
+ proc = subprocess.Popen([binary, 'acp', '--yolo'], cwd=temp, env=env,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, text=True, start_new_session=True)
+ events = queue.Queue()
+
+ def reader():
+ for line in proc.stdout:
+ try:
+ events.put(json.loads(line))
+ except ValueError:
+ pass
+ events.put(None)
+ threading.Thread(target=reader, daemon=True).start()
+
+ def call(method, params, id, budget=25):
+ proc.stdin.write(json.dumps(dict(jsonrpc='2.0', method=method, params=params, id=id)) + '\n')
+ proc.stdin.flush()
+ deadline = time.monotonic() + budget
+ while True:
+ value = events.get(timeout=max(.01, deadline - time.monotonic()))
+ assert value is not None, 'ACP worker exited before replying'
+ if value.get('id') == id:
+ return value
+
+ call('initialize', {'protocolVersion': 1}, 1)
+ sid = call('session/new', {}, 2)['result']['sessionId']
+ # Every reconnect drops too, so the whole drop budget (with its
+ # growing pauses) runs before the turn ends.
+ failed = call('session/prompt', {'sessionId': sid, 'prompt': [{'type': 'text', 'text': 'hello'}]}, 3, budget=90)
+ assert 'StreamDropped' in json.dumps(failed.get('error')), failed
+ saved = list((Path(temp) / '.graff/sessions').glob('*.session.json'))
+ assert saved, 'the failed turn was not saved'
+ rows = [m for p in saved for m in json.loads(p.read_text()).get('messages', [])]
+ texts = [json.dumps(m) for m in rows if isinstance(m, dict) and m.get('role') == 'assistant']
+ assert any(MARKER in t for t in texts), f'no end marker saved: {texts}'
+ assert not any('interrupted by user' in t for t in texts), texts
+ print(f'PASS ACP stream drop: turn saved as {MARKER!r}, never a user interrupt')
+ finally:
+ if proc:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ proc.wait(timeout=2)
+ for pipe in (proc.stdin, proc.stdout):
+ if pipe:
+ pipe.close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/acp_live_turn.zig b/src/acp_live_turn.zig
index 93b2afb5..519298b1 100644
--- a/src/acp_live_turn.zig
+++ b/src/acp_live_turn.zig
@@ -182,6 +182,13 @@ pub const LiveTurn = struct {
cancel_source.marker(self.cancelled_source.?),
));
}
+ // #1639: like the REPL, a turn the transport ended keeps its
+ // partial reply and says why, so the next prompt (and a resume)
+ // sees where the answer stopped instead of no reply at all.
+ if (!isolated and (err == error.StreamDropped or err == error.StreamStalled)) {
+ const ended = try @import("stream_reconnect.zig").endedEarlyText(arena, self.root.partial_text.items, err == error.StreamStalled);
+ try self.root.messages.append(try messages.textMessage(arena, "assistant", ended));
+ }
if (isolated) {
const marker = if (err == error.Interrupted or err == error.Canceled)
cancel_source.marker(self.cancelled_source.?)
diff --git a/src/agent_request.zig b/src/agent_request.zig
index 2beda2e0..3c739e8e 100644
--- a/src/agent_request.zig
+++ b/src/agent_request.zig
@@ -207,8 +207,8 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
// #56: bounded stream-stall / drop reconnect budget (codex's stream_max_retries analog), outside the rebuild loop so a WS reanchor can't reset it.
self.stall = .{}; // #680: a fresh widen ladder + tripped budget per request
var stall_retries: usize = 0;
- const max_stall_retries: usize = 2;
- const stall_reconnect_backoff_ms = 750; // brief pause before reconnecting on a fresh stream
+ var drop_retries: usize = 0; // #1639: a peer close earns its own, larger budget (stream_reconnect.zig)
+ const reconnect = @import("stream_reconnect.zig");
var server_retries: usize = 0; // #opencode-parity: bounded retries for a transient in-stream server overload
var openai404_retries: usize = 0; // #opencode-parity: bounded retries for OpenAI's spurious model 404s
const max_openai404_retries: usize = 2;
@@ -282,24 +282,25 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
if (err == error.StreamStalled or err == error.StreamDropped) {
const stalled = err == error.StreamStalled;
const what: []const u8 = if (stalled) "stream stalled" else "stream dropped";
- if (stall_retries < max_stall_retries) {
- stall_retries += 1;
+ const retries = if (stalled) &stall_retries else &drop_retries;
+ if (retries.* < reconnect.maxRetries(stalled)) {
+ retries.* += 1;
// #56: give a fresh WS one reconnect; if it stalls again,
// fall back to the reliable SSE transport for the rest of
// the session — codex's WS→SSE fallback, and consistent
// with how a WS transport error is already handled (ws_off).
- if (stall_retries > 1) self.ws_off = true;
+ if (stall_retries + drop_retries > 1) self.ws_off = true;
if (self.tracer) |tr| tr.note("stream_retry", what);
if (telemetry.g_telem) |t| t.errorEvent("stream_retry", what);
- @import("agent_stream.zig").noteStallRetry(self, stall_retries); // #680: clears the partial (fresh stream, no concat) + widens the between-lines budget
+ @import("agent_stream.zig").noteStallRetry(self, stall_retries); // #680: clears the partial (fresh stream, no concat) + widens the between-lines budget (stalls only)
self.closeCodexWs(); // tear down the dead WS + null codex_prev_id for a full re-send (no-op off codex WS)
- self.sleepInterruptible(jitter.ms(self.io, stall_reconnect_backoff_ms)) catch return error.Interrupted; // #1274: desync parallel reconnects
+ self.sleepInterruptible(jitter.ms(self.io, reconnect.backoffMs(stalled, retries.*))) catch return error.Interrupted; // #1274: desync parallel reconnects
continue :rebuild;
}
// Budget gone: the turn is ending. Mid-turn cuts stayed silent.
@import("engine_sink.zig").forAgent(self).emit(self.io, .{ .transport_aborted = .{ .reason = if (stalled) .stalled else .dropped, .turn_ending = true } });
if (stalled) {
- self.last_api_error = @import("agent_stream.zig").stallGiveUpMessage(self, max_stall_retries); // #680: reports the wait that tripped, not the configured total
+ self.last_api_error = @import("agent_stream.zig").stallGiveUpMessage(self, reconnect.max_stall_retries); // #680: reports the wait that tripped, not the configured total
if (telemetry.g_telem) |t| t.errorEvent("stream_stall", self.last_api_error orelse "stream stalled");
if (self.tracer) |tr| tr.note("stream_stall", self.last_api_error orelse "stream stalled");
return error.StreamStalled;
diff --git a/src/main.zig b/src/main.zig
index a2b160f7..2efddacb 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -566,7 +566,7 @@ test { // ── Unit tests (`zig build test`): pull in tests from imported modu
_ = .{ @import("prompt_astra.zig"), @import("prompt_guidance.zig"), @import("oauth_chatgpt.zig"), @import("chatgpt_reauth.zig") }; // per-model guidance (Astra, GPT-5.6); ChatGPT plan sign-in (ADR 0253)
_ = .{ @import("agent_empty_completion.zig"), @import("agent_model_loop.zig"), @import("publication_policy_tests.zig"), @import("jobs_completion_tests.zig") };
_ = .{ @import("provider_routing_tests.zig"), @import("codegraff_messages_tests.zig"), @import("tool_result_empty_tests.zig"), @import("history_wire.zig"), @import("path_lookup.zig"), @import("agents_prepare.zig"), @import("catalog_price.zig"), @import("readline_paste_number_tests.zig"), @import("test_hooks.zig"), @import("peer_wake_loop.zig") };
- _ = .{ @import("list_dir_nearmiss.zig"), @import("tool_preview_tests.zig") };
+ _ = .{ @import("list_dir_nearmiss.zig"), @import("tool_preview_tests.zig"), @import("stream_reconnect.zig") };
_ = @import("repo_map.zig");
_ = @import("agent_overflow_tests.zig"); // #414: and, through it, agent_overflow.zig's table tests
_ = @import("agent_gateway_retry.zig"); // #1019: flake vs overflow classification must stay reachable
diff --git a/src/mainloop.zig b/src/mainloop.zig
index 3daadc97..3f64a6d9 100644
--- a/src/mainloop.zig
+++ b/src/mainloop.zig
@@ -446,10 +446,7 @@ pub fn run(ctx: *Ctx) !void {
// harness gave up, NOT the user (#134) — keep the partial but
// tag it a stall, and emit the structured error for --json.
const partial = std.mem.trim(u8, ctx.root.partial_text.items, " \t\r\n");
- const marker: []const u8 = if (partial.len > 0)
- try std.fmt.allocPrint(ctx.arena, "{s}\n\n[response ended early: stream stalled]", .{partial})
- else
- "[response ended early: stream stalled]";
+ const marker = try @import("stream_reconnect.zig").endedEarlyText(ctx.arena, partial, true);
try ctx.root.messages.append(try messages.textMessage(ctx.arena, "assistant", marker));
if (main_mod.json_mode) {
ctx.root.emit(.{ .type = "error", .message = ctx.root.last_api_error orelse "stream stalled — ending turn" });
@@ -468,10 +465,7 @@ pub fn run(ctx: *Ctx) !void {
// the user (#133). Keep the partial, tag it a drop (never
// "[response interrupted by user]"), and emit the --json error.
const partial = std.mem.trim(u8, ctx.root.partial_text.items, " \t\r\n");
- const marker: []const u8 = if (partial.len > 0)
- try std.fmt.allocPrint(ctx.arena, "{s}\n\n[response ended early: connection dropped]", .{partial})
- else
- "[response ended early: connection dropped]";
+ const marker = try @import("stream_reconnect.zig").endedEarlyText(ctx.arena, partial, false);
try ctx.root.messages.append(try messages.textMessage(ctx.arena, "assistant", marker));
if (main_mod.json_mode) {
ctx.root.emit(.{ .type = "error", .message = ctx.root.last_api_error orelse "stream dropped — ending turn" });
diff --git a/src/stream_reconnect.zig b/src/stream_reconnect.zig
new file mode 100644
index 00000000..62149fdf
--- /dev/null
+++ b/src/stream_reconnect.zig
@@ -0,0 +1,69 @@
+//! #1639: reconnecting a response stream that died before its terminal event.
+//!
+//! A stall (no bytes for the between-lines budget) and a drop (the peer closed
+//! or reset the connection) each re-send the whole request on a fresh stream.
+//! They get separate budgets: a drop says nothing about the model, and a link
+//! that kills long-lived streams intermittently would otherwise end a healthy
+//! turn after three cuts. Drops therefore get more attempts and a pause that
+//! doubles, so a flapping connection has time to recover.
+//!
+//! Whichever budget runs out, the turn keeps its partial reply tagged with why
+//! it ended — the same row on the REPL and over ACP.
+
+const std = @import("std");
+
+pub const max_stall_retries: usize = 2;
+pub const max_drop_retries: usize = 5;
+
+const base_backoff_ms: u64 = 750;
+const max_backoff_ms: u64 = 8_000;
+
+/// Pause before reconnect `attempt` (1-based, counted per kind). Stalls keep
+/// the flat pause; drops double it per attempt up to `max_backoff_ms`.
+pub fn backoffMs(stalled: bool, attempt: usize) u64 {
+ if (stalled or attempt <= 1) return base_backoff_ms;
+ const shift: u6 = @intCast(@min(attempt - 1, 4));
+ return @min(base_backoff_ms << shift, max_backoff_ms);
+}
+
+pub fn maxRetries(stalled: bool) usize {
+ return if (stalled) max_stall_retries else max_drop_retries;
+}
+
+/// The assistant row for a turn the transport ended: the partial reply (if
+/// any), then a marker saying it was cut short and why — never a user cancel.
+pub fn endedEarlyText(arena: std.mem.Allocator, partial_raw: []const u8, stalled: bool) ![]const u8 {
+ const marker = if (stalled)
+ "[response ended early: stream stalled]"
+ else
+ "[response ended early: connection dropped]";
+ const partial = std.mem.trim(u8, partial_raw, " \t\r\n");
+ if (partial.len == 0) return marker;
+ return std.fmt.allocPrint(arena, "{s}\n\n{s}", .{ partial, marker });
+}
+
+test "drops get a larger, growing reconnect budget than stalls" {
+ try std.testing.expect(max_drop_retries > max_stall_retries);
+ try std.testing.expectEqual(@as(u64, 750), backoffMs(true, 1));
+ try std.testing.expectEqual(@as(u64, 750), backoffMs(true, 2));
+ try std.testing.expectEqual(@as(u64, 750), backoffMs(false, 1));
+ try std.testing.expectEqual(@as(u64, 1500), backoffMs(false, 2));
+ try std.testing.expectEqual(@as(u64, 3000), backoffMs(false, 3));
+ try std.testing.expectEqual(@as(u64, 6000), backoffMs(false, 4));
+ try std.testing.expectEqual(@as(u64, 8000), backoffMs(false, 5));
+ try std.testing.expectEqual(@as(u64, 8000), backoffMs(false, 50));
+}
+
+test "a transport-ended turn keeps its partial and says why it ended" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ try std.testing.expectEqualStrings(
+ "half a reply\n\n[response ended early: connection dropped]",
+ try endedEarlyText(arena, " half a reply \n", false),
+ );
+ try std.testing.expectEqualStrings(
+ "[response ended early: stream stalled]",
+ try endedEarlyText(arena, " \n", true),
+ );
+}