Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
18.3 KB
Raw Blame
//! (#401) The loopback WebSocket peer the codex-WS transport tests drive, and
//! the root-shaped Agent pointed at it. Harness only — no tests live here; they
//! live in agent_ws_stall_test.zig (the budgets and the guards),
//! agent_ws_reuse_test.zig (reuse vs fresh connect) and
//! agent_ws_fallback_test.zig (#427, the WS→SSE ladder). Split out because both
//! files would otherwise carry a copy, and a second copy of a mock server is
//! exactly how two suites start testing different things.
//!
//! Same spirit as the GRAFF_CODEX_URL/lmstudio HTTP mocks: `provider.url` points
//! at 127.0.0.1, and no network, key or provider is used.

const std = @import("std");
const Io = std.Io;

const Agent = @import("agent.zig").Agent;

pub fn nowMs(io: Io) i64 {
    return @intCast(@divTrunc(Io.Timestamp.now(io, .awake).nanoseconds, std.time.ns_per_ms));
}

pub const delta_event = "{\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}";
pub const completed_event = "{\"type\":\"response.completed\"}";
pub const prewarm_completed_event = "{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_prewarm\"}}";
const PREWARM_MARKER = "\"generate\":false";
pub const generic_error_event = "{\"type\":\"error\",\"error\":{\"code\":\"invalid_request_error\",\"message\":\"mock bad request\"}}";

/// What the backend actually puts on the socket in the first milliseconds after
/// a send, before the model has produced anything: the two protocol events, then
/// a reasoning delta. None of these grows partial_text on the SSE path, so none
/// may tighten the WS read budget either.
pub const protocol_events = [_][]const u8{
    "{\"type\":\"response.created\",\"response\":{\"id\":\"r1\"}}",
    "{\"type\":\"response.in_progress\",\"response\":{\"id\":\"r1\"}}",
    "{\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"thinking\"}",
};

/// A codex turn whose whole visible answer is attempt_completion's `result`
/// prose: the model opens the meta call and streams its ARGUMENTS. Not one
/// output_text delta in the turn — on SSE this prints (argLiveDelta →
/// emitArgText → partial_text), so it is tokens-flowing there and must be here.
pub const arg_prose_events = [_][]const u8{
    "{\"type\":\"response.created\",\"response\":{\"id\":\"r1\"}}",
    "{\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"name\":\"attempt_completion\",\"id\":\"fc1\"}}",
    "{\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"delta\":\"{\\\"result\\\":\\\"the ans\"}",
    "{\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"delta\":\"wer is 42\"}",
};

/// How long `slow_first_frame` thinks before its first frame: past a shrunken
/// head budget, inside the pre-first-token budget.
pub var slow_first_frame_ms: i64 = 700;

/// ADR 0241: the item holding the prose closes, then `prose_close_then_think`
/// thinks this long: past the tests' tightened 500ms, inside their full 2000ms.
pub const item_done_event = "{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\"}}";
pub const think_after_prose_ms = 1000;

/// ADR 0248: a commentary heads-up the server holds open while it composes the
/// tool calls it then releases together (`commentary_then_think`).
pub const commentary_added_event = "{\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"phase\":\"commentary\"}}";
pub const commentary_done_event = "{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"phase\":\"commentary\"}}";

/// A loopback WebSocket peer with a scripted failure mode.
pub const Mock = struct {
    pub const Mode = enum {
        /// Accept the TCP connection and never answer the upgrade — the dial
        /// that hangs before the 101 status line.
        no_upgrade,
        /// Upgrade, then never read again: the client's frame fills the socket
        /// buffers and its write parks in the kernel.
        never_drain,
        /// Upgrade, take the client's frame, answer with ONE delta frame and
        /// then go permanently silent. This is #401's reported signature: the
        /// turn is under way, data flowed, and the server stopped.
        frame_then_silence,
        /// The healthy control: delta, then response.completed.
        frame_then_complete,
        /// Upgrade, take the client's frame, answer with `protocol_events` and
        /// then go quiet while the model "thinks". Frames flow, none of them is
        /// output text — so the budget must stay at the pre-first-token value.
        protocol_then_silence,
        /// Upgrade, take the client's frame, answer NOTHING — ever. On a REUSED
        /// socket that is the dead-socket signature: the backend acks a send
        /// with response.created within milliseconds, so zero frames means the
        /// peer is gone, not thinking.
        read_then_silence,
        /// Upgrade, take the client's frame, think past the head budget, then
        /// answer normally. A FRESH connect must survive this.
        slow_first_frame,
        /// Upgrade, take the client's frame, stream a whitelisted meta call's
        /// arguments (`arg_prose_events`) and then go silent.
        arg_prose_then_silence,
        /// Send a generic terminal API error and close immediately, matching
        /// Codex's failure sequence from issue #692.
        generic_error_then_close,
        /// ADR 0241: a heads-up delta, its item closes, a silent think past the
        /// tightened budget (a tool call being composed), then completion.
        prose_close_then_think,
        /// ADR 0248: a commentary heads-up whose message stays OPEN through a
        /// silent compose past the tightened budget; then it closes and the
        /// turn completes.
        commentary_then_think,
    };

    pub fn run(io: Io, server: *std.Io.net.Server, mode: Mode, done: *std.atomic.Value(bool)) void {
        const c = server.accept(io) catch return;
        defer c.close(io);
        var rbuf: [8192]u8 = undefined;
        var wbuf: [4096]u8 = undefined;
        var sr = std.Io.net.Stream.Reader.init(c, io, &rbuf);
        var sw = std.Io.net.Stream.Writer.init(c, io, &wbuf);
        if (mode == .no_upgrade) return idle(io, done);

        while (true) {
            const line = sr.interface.takeDelimiterInclusive('\n') catch return idle(io, done);
            if (line.len <= 2) break; // the blank line ends the upgrade request
        }
        sw.interface.writeAll("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n") catch return idle(io, done);
        sw.interface.flush() catch return idle(io, done);

        switch (mode) {
            .no_upgrade, .never_drain => {},
            .frame_then_silence, .frame_then_complete => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                writeTextFrame(&sw.interface, delta_event) catch return idle(io, done);
                if (mode == .frame_then_complete)
                    writeTextFrame(&sw.interface, completed_event) catch return idle(io, done);
            },
            .protocol_then_silence => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                for (protocol_events) |ev|
                    writeTextFrame(&sw.interface, ev) catch return idle(io, done);
            },
            .read_then_silence => readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done),
            .slow_first_frame => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                io.sleep(.fromMilliseconds(slow_first_frame_ms), .awake) catch return idle(io, done);
                writeTextFrame(&sw.interface, delta_event) catch return idle(io, done);
                writeTextFrame(&sw.interface, completed_event) catch return idle(io, done);
            },
            .arg_prose_then_silence => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                for (arg_prose_events) |ev|
                    writeTextFrame(&sw.interface, ev) catch return idle(io, done);
            },
            .prose_close_then_think => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                writeTextFrame(&sw.interface, delta_event) catch return idle(io, done);
                writeTextFrame(&sw.interface, item_done_event) catch return idle(io, done);
                io.sleep(.fromMilliseconds(think_after_prose_ms), .awake) catch return idle(io, done);
                writeTextFrame(&sw.interface, completed_event) catch return idle(io, done);
            },
            .commentary_then_think => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return idle(io, done);
                writeTextFrame(&sw.interface, commentary_added_event) catch return idle(io, done);
                writeTextFrame(&sw.interface, delta_event) catch return idle(io, done);
                io.sleep(.fromMilliseconds(think_after_prose_ms), .awake) catch return idle(io, done);
                writeTextFrame(&sw.interface, commentary_done_event) catch return idle(io, done);
                writeTextFrame(&sw.interface, completed_event) catch return idle(io, done);
            },
            .generic_error_then_close => {
                readRealFrame(io, &sr.interface, &sw.interface) catch return;
                writeTextFrame(&sw.interface, generic_error_event) catch return;
                return;
            },
        }
        idle(io, done);
    }

    /// Hold the connection open (and, for never_drain, undrained) until the test
    /// releases it. Closing early hands the client a clean EOF, a different
    /// failure than the silence being reproduced.
    pub fn idle(io: Io, done: *std.atomic.Value(bool)) void {
        while (!done.load(.acquire)) io.sleep(.fromMilliseconds(20), .awake) catch break;
    }

    /// Consume one masked client frame (RFC 6455 §5.2) and return its payload
    /// when it fits the reader buffer, else null (large frames are streamed and
    /// discarded; only prewarm frames are that large in practice).
    pub fn readClientFramePayload(r: *Io.Reader) !?[]const u8 {
        const h = try r.takeArray(2);
        var len: u64 = h[1] & 0x7f;
        if (len == 126) {
            len = std.mem.readInt(u16, try r.takeArray(2), .big);
        } else if (len == 127) {
            len = std.mem.readInt(u64, try r.takeArray(8), .big);
        }
        if ((h[1] & 0x80) != 0) _ = try r.takeArray(4); // mask key
        if (len <= 4096) return try r.take(@intCast(len));
        // Chunked discard: take() cannot exceed the reader buffer.
        var tail: [16]u8 = undefined;
        var tail_len: usize = 0;
        var left = len;
        while (left > 0) {
            const chunk: usize = @intCast(@min(left, 4096));
            const got = try r.take(chunk);
            // Carry a 16-byte overlap so the marker cannot hide at a boundary.
            const scan = tail[0..tail_len].len + got.len;
            _ = scan;
            if (std.mem.indexOf(u8, got, PREWARM_MARKER) != null) return null;
            left -= chunk;
            tail_len = @min(tail_len, 16);
            @memcpy(tail[tail_len..][0..@min(got.len, 16 - tail_len)], got[got.len - @min(got.len, 16 - tail_len) ..]);
        }
        return null;
    }

    /// Read client frames until a REAL request arrives; a codex-style
    /// `generate:false` prewarm frame is answered with a completed id and
    /// skipped, so every mode keeps testing its own first real frame.
    pub fn readRealFrame(io: Io, r: *Io.Reader, w: *Io.Writer) !void {
        _ = io;
        while (true) {
            const payload = (try readClientFramePayload(r)) orelse {
                try writeTextFrame(w, prewarm_completed_event);
                continue;
            };
            if (std.mem.indexOf(u8, payload, PREWARM_MARKER) == null) return;
            try writeTextFrame(w, prewarm_completed_event);
        }
    }

    pub fn readClientFrame(r: *Io.Reader) !void {
        _ = try readClientFramePayload(r);
    }

    /// One unmasked server->client text frame (payloads here are all < 126 B).
    pub fn writeTextFrame(w: *Io.Writer, payload: []const u8) !void {
        try w.writeAll(&[_]u8{ 0x81, @intCast(payload.len) });
        try w.writeAll(payload);
        try w.flush();
    }
};

/// A root-shaped Agent pointed at the mock. `out`/`in` stay null: no TTY means no
/// spinner, no Esc poll and no user-facing stall line — the transport alone.
pub fn mockAgent(gpa: std.mem.Allocator, arena: std.mem.Allocator, io: Io, url: []const u8) Agent {
    return .{
        .gpa = gpa,
        .arena = arena,
        .io = io,
        .client = undefined, // the WS path never touches the HTTP client
        .provider = .{
            .id = "codex",
            .kind = .responses,
            .auth = .bearer,
            .url = url,
            .api_key = "k",
            .model = "gpt-5",
            .context = 100_000,
        },
        .messages = std.json.Array.init(arena),
        .sub = false,
        .label = "",
        .out = null,
    };
}

pub fn traced(tw: *Io.Writer.Allocating, needle: []const u8) bool {
    return std.mem.indexOf(u8, tw.written(), needle) != null;
}

/// How the client ended the connection, as seen from the other side.
pub const saw_nothing: u8 = 0;
pub const saw_close_frame: u8 = 1;
pub const saw_fin: u8 = 2;

/// Complete the upgrade, then classify the next thing the client does: a ws close
/// frame (opcode 0x8) or EOF. `deinit` normally writes a courtesy close frame —
/// one more BLOCKING write on a socket that may be what wedged us.
pub fn closeObserver(io: Io, server: *std.Io.net.Server, seen: *std.atomic.Value(u8), done: *std.atomic.Value(bool)) void {
    const c = server.accept(io) catch return;
    defer c.close(io);
    var rbuf: [8192]u8 = undefined;
    var wbuf: [4096]u8 = undefined;
    var sr = std.Io.net.Stream.Reader.init(c, io, &rbuf);
    var sw = std.Io.net.Stream.Writer.init(c, io, &wbuf);
    while (true) {
        const line = sr.interface.takeDelimiterInclusive('\n') catch return Mock.idle(io, done);
        if (line.len <= 2) break;
    }
    sw.interface.writeAll("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n") catch return Mock.idle(io, done);
    sw.interface.flush() catch return Mock.idle(io, done);
    const h = sr.interface.takeArray(2) catch {
        seen.store(saw_fin, .release); // stream ended: a plain TCP FIN
        return Mock.idle(io, done);
    };
    seen.store(if ((h[0] & 0x0f) == 0x8) saw_close_frame else saw_nothing, .release);
    Mock.idle(io, done);
}

/// (#427) The SSE turn the fallback POST is answered with: the same two events
/// the WS modes frame, as `data:` lines, so a test can assert the turn finished
/// over the other transport with the same needles.
pub const sse_body = "data: " ++ delta_event ++ "\n\n" ++ "data: " ++ completed_event ++ "\n\n";

/// (#427) The whole WS→SSE ladder over ONE loopback port. The first `refusals`
/// connections are WebSocket handshakes answered with `status` — any non-101
/// status line, i.e. "426 Upgrade Required" (authoritative: never retry) or
/// something generic like "500 Internal Server Error" — and the connection
/// after them is the SSE POST graff falls back to. `dials` counts refused
/// handshakes and `sse` served fallback turns: together they prove whether the
/// ladder burned a retry the server had already ruled out.
pub fn refuseUpgrade(
    io: Io,
    server: *std.Io.net.Server,
    status: []const u8,
    refusals: u8,
    dials: *std.atomic.Value(u8),
    sse: *std.atomic.Value(u8),
    done: *std.atomic.Value(bool),
) void {
    var refused: u8 = 0;
    while (refused < refusals) : (refused += 1) {
        if (done.load(.acquire)) return; // torn down early — see releaseAccept
        const c = server.accept(io) catch return Mock.idle(io, done);
        var rbuf: [8192]u8 = undefined;
        var wbuf: [4096]u8 = undefined;
        var sr = std.Io.net.Stream.Reader.init(c, io, &rbuf);
        var sw = std.Io.net.Stream.Writer.init(c, io, &wbuf);
        _ = requestHead(&sr.interface) catch {};
        sw.interface.print("HTTP/1.1 {s}\r\nContent-Length: 0\r\n\r\n", .{status}) catch {};
        sw.interface.flush() catch {};
        dials.store(refused + 1, .release);
        c.close(io); // an upgrade request carries no body, so nothing unread to RST over
    }
    if (done.load(.acquire)) return;
    const c = server.accept(io) catch return Mock.idle(io, done);
    defer c.close(io);
    var rbuf: [8192]u8 = undefined;
    var wbuf: [4096]u8 = undefined;
    var sr = std.Io.net.Stream.Reader.init(c, io, &rbuf);
    var sw = std.Io.net.Stream.Writer.init(c, io, &wbuf);
    // The POST body must be drained: closing a socket with unread bytes in the
    // receive queue sends an RST, which discards the reply we just wrote.
    const body_len = requestHead(&sr.interface) catch return Mock.idle(io, done);
    sr.interface.discardAll(body_len) catch {};
    sw.interface.print(
        "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {d}\r\n\r\n{s}",
        .{ sse_body.len, sse_body },
    ) catch return Mock.idle(io, done);
    sw.interface.flush() catch {};
    sse.store(1, .release);
    Mock.idle(io, done);
}

/// (#427) Unblock a task parked in `accept` so it can observe `done` and exit.
/// A blocked accept is not cancellable, so a test whose assertion fails BEFORE
/// the ladder reached its last leg would otherwise hang at `fut.await` instead
/// of reporting the failure. Register it after the await defer (LIFO: `done`,
/// this, then the join). Same trick as http.zig's #177 poisoned-conn test.
pub fn releaseAccept(io: Io, server: *std.Io.net.Server) void {
    var bound = server.socket.address;
    if (std.Io.net.IpAddress.connect(&bound, io, .{ .mode = .stream })) |s| s.close(io) else |_| {}
}

/// Consume an HTTP request head, returning its Content-Length (0 when absent).
fn requestHead(r: *Io.Reader) !usize {
    var body_len: usize = 0;
    while (true) {
        const line = try r.takeDelimiterInclusive('\n');
        if (line.len <= 2) return body_len; // the blank line ends the head
        const tag = "content-length:";
        if (line.len > tag.len and std.ascii.eqlIgnoreCase(line[0..tag.len], tag))
            body_len = std.fmt.parseInt(usize, std.mem.trim(u8, line[tag.len..], " \r\n"), 10) catch 0;
    }
}