Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
12.1 KB
Raw Blame
//! Offline production request-loop coverage for the construction-only TLS outage budget.
const std = @import("std");
const Io = std.Io;
const Agent = @import("agent.zig").Agent;
const http = @import("http.zig");
const clients = @import("http_client.zig");
const transport = @import("http_client_integration_tests.zig");
const trace = @import("trace.zig");

// Replace only this test owner's backoff sleeps, not concurrent HTTP watchdogs.
// All other I/O keeps the real testing Io and its original userdata.
var current_probe: ?*SleepProbe = null;
const SleepProbe = struct {
    owner: std.Thread.Id,
    runtime: *clients.Runtime,
    waited_ms: u64 = 0,
    cancel_generation: u64 = 0,
    cancel_io: bool = false,

    fn sleep(ctx: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
        const probe = current_probe.?;
        if (std.Thread.getCurrentId() != probe.owner)
            return std.testing.io.vtable.sleep(ctx, timeout);
        probe.waited_ms += @intCast(timeout.duration.raw.toMilliseconds());
        if (probe.cancel_generation > 0 and probe.runtime.recovery.stats().active_id >= probe.cancel_generation) {
            if (probe.cancel_io) return error.Canceled;
            Agent.esc_cancel.store(true, .release);
        }
    }
};

const Fixture = struct {
    runtime: clients.Runtime,
    arena: std.heap.ArenaAllocator,
    server: std.Io.net.Server,
    server_future: ?Io.Future(void),
    accepted: std.atomic.Value(usize),
    url: [64]u8,
    vtable: Io.VTable,
    probe: SleepProbe,
    agent: Agent,
    trace_output: Io.Writer.Allocating,
    tracer: trace.Tracer,
    previous_cancel: bool,

    fn init(self: *Fixture) !void {
        // Serialize: init installs (and clears) the process-global TLS test hooks.
        clients.lockTestHooksForTest();
        errdefer clients.unlockTestHooksForTest();
        const gpa = std.testing.allocator;
        const io = std.testing.io;
        self.runtime.init(gpa, io);
        errdefer self.runtime.deinit(io);
        http.waitForClientReady(io);
        self.arena = .init(gpa);
        errdefer self.arena.deinit();
        var address = try std.Io.net.IpAddress.parseLiteral("127.0.0.1:0");
        self.server = try std.Io.net.IpAddress.listen(&address, io, .{});
        self.server_future = null;
        self.accepted = .init(0);
        self.vtable = io.vtable.*;
        self.vtable.sleep = SleepProbe.sleep;
        self.probe = .{ .owner = std.Thread.getCurrentId(), .runtime = &self.runtime };
        current_probe = &self.probe;
        self.previous_cancel = Agent.esc_cancel.swap(false, .acq_rel);
        self.trace_output = .init(gpa);
        const agent_io: Io = .{ .userdata = io.userdata, .vtable = &self.vtable };
        self.tracer = .{ .io = agent_io, .gpa = gpa, .out = &self.trace_output.writer, .start = .now(io, .awake) };
        const url = try std.fmt.bufPrint(&self.url, "http://127.0.0.1:{d}/v1/test", .{self.server.socket.address.getPort()});
        self.agent = .{
            .gpa = gpa,
            .arena = self.arena.allocator(),
            .io = agent_io,
            .client = &self.runtime.client,
            .provider = transport.provider(url),
            .messages = std.json.Array.init(self.arena.allocator()),
            .sub = true,
            .label = "tls-budget-test",
            .tracer = &self.tracer,
            .out = null,
        };
    }

    fn serve(self: *Fixture, replies: []const transport.Reply) void {
        // Eager start: io.async may run inline, so a peer awaited only in deinit
        // is not guaranteed to be serving while the request under test runs.
        self.server_future = std.testing.io.concurrent(transport.serveReplies, .{ std.testing.io, &self.server, replies, &self.accepted }) catch
            @panic("test server task could not start");
    }

    fn deinit(self: *Fixture) void {
        const io = std.testing.io;
        if (self.server_future) |*future| {
            transport.releaseAccept(io, &self.server);
            future.await(io);
        }
        self.server.deinit(io);
        @import("agent_render.zig").deinitMarkdown(&self.agent);
        self.trace_output.deinit();
        self.runtime.deinit(io);
        self.arena.deinit();
        Agent.esc_cancel.store(self.previous_cancel, .release);
        current_probe = null;
        clients.unlockTestHooksForTest();
    }

    fn expectReleased(self: *Fixture) !void {
        try std.testing.expectEqual(@as(usize, 0), self.runtime.recovery.stats().total_refs);
        try std.testing.expectEqual(@as(usize, 0), self.runtime.recovery.stats().retired);
    }

    fn tlsNotes(self: *Fixture) usize {
        return std.mem.count(u8, self.trace_output.written(), "\"ev\":\"tls_request_construction\"");
    }
};

test "TLS outage budget: eight constructor failures recover in the same production request" {
    for ([_]bool{ false, true }) |sub| {
        var fixture: Fixture = undefined;
        try fixture.init();
        defer fixture.deinit();
        fixture.agent.sub = sub;
        const replies = [_]transport.Reply{.{ .body = transport.chat_body }};
        fixture.serve(&replies);
        clients.injectConstructionTlsThroughGenerationForTest(7);
        clients.injectReplacementCaWarmFailureForTest();

        const response = try fixture.agent.request(null);
        const content = response.get("choices").?.array.items[0].object.get("message").?.object.get("content").?.string;
        try std.testing.expectEqualStrings("child-ok", content);
        try std.testing.expectEqual(@as(u64, 8), fixture.runtime.recovery.stats().active_id);
        try std.testing.expectEqual(@as(usize, 8), fixture.tlsNotes());
        try std.testing.expectEqual(@as(usize, 1), fixture.accepted.load(.acquire));
        try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, fixture.trace_output.written(), "replacement CA prewarm failed"));
        // Eight retry waits cross the old six-attempt window, with real policy jitter.
        try std.testing.expect(fixture.probe.waited_ms >= 19_750);
        try std.testing.expect(fixture.probe.waited_ms <= 24_688);
        try fixture.expectReleased();
    }
}

test "TLS outage budget: permanent failure is bounded and preserves recovery for a later request" {
    var fixture: Fixture = undefined;
    try fixture.init();
    defer fixture.deinit();
    const limit = http.RetryPlan.tls_construction_attempts;
    clients.injectConstructionTlsThroughGenerationForTest(limit - 1);

    try std.testing.expectError(error.ApiError, fixture.agent.request(null));
    try std.testing.expectEqual(@as(u64, limit), fixture.runtime.recovery.stats().active_id);
    try std.testing.expectEqual(limit, fixture.tlsNotes());
    try std.testing.expectEqual(@as(usize, 0), fixture.accepted.load(.acquire));
    try std.testing.expect(std.mem.indexOf(u8, fixture.agent.last_api_error.?, "gave up after 16 attempts") != null);
    try std.testing.expect(fixture.probe.waited_ms >= 47_750);
    try std.testing.expect(fixture.probe.waited_ms <= 59_688);
    try fixture.expectReleased();

    const replies = [_]transport.Reply{.{ .body = transport.chat_body }};
    fixture.serve(&replies);
    _ = try fixture.agent.request(null);
    try std.testing.expectEqual(@as(usize, 1), fixture.accepted.load(.acquire));
    try fixture.expectReleased();
}

test "TLS outage budget: request rebuild cannot reset the construction failure counter" {
    var fixture: Fixture = undefined;
    try fixture.init();
    defer fixture.deinit();
    const replies = [_]transport.Reply{.{
        .status = "400 Bad Request",
        .body = "{\"error\":{\"message\":\"unsupported stream_options\"}}",
        // First eight TLS failures recover; the parsed rejection then rebuilds
        // the request with stream_options dropped while TLS starts failing again.
        .fail_tls_through_generation = 1000,
    }};
    fixture.serve(&replies);
    clients.injectConstructionTlsThroughGenerationForTest(7);

    try std.testing.expectError(error.ApiError, fixture.agent.request(null));
    try std.testing.expectEqual(@as(u64, 16), fixture.runtime.recovery.stats().active_id);
    try std.testing.expectEqual(@as(usize, 16), fixture.tlsNotes());
    try std.testing.expectEqual(@as(usize, 1), fixture.accepted.load(.acquire));
    try std.testing.expect(fixture.probe.waited_ms >= 47_750);
    try std.testing.expect(fixture.probe.waited_ms <= 59_688);
    try fixture.expectReleased();
}

test "TLS outage budget: Esc and I/O cancellation stop the extended ladder without a replacement attempt" {
    for ([_]bool{ false, true }) |cancel_io| {
        var fixture: Fixture = undefined;
        try fixture.init();
        defer fixture.deinit();
        fixture.probe.cancel_generation = 9;
        fixture.probe.cancel_io = cancel_io;
        clients.injectConstructionTlsThroughGenerationForTest(1000);

        try std.testing.expectError(error.Interrupted, fixture.agent.request(null));
        try std.testing.expectEqual(@as(u64, 9), fixture.runtime.recovery.stats().active_id);
        try std.testing.expectEqual(@as(usize, 9), fixture.tlsNotes());
        try std.testing.expectEqual(@as(usize, 0), fixture.accepted.load(.acquire));
        // Cancellation at the first chunk of wait nine, rather than its full 4–5 seconds.
        try std.testing.expect(fixture.probe.waited_ms <= 24_788);
        try fixture.expectReleased();
    }
}

test "TLS outage budget: ordinary transport still gets exactly six attempts after TLS recovery" {
    var fixture: Fixture = undefined;
    try fixture.init();
    defer fixture.deinit();
    // URI construction fails deterministically, without a network call, after the injected TLS failures.
    fixture.agent.provider.url = ":invalid";
    clients.injectConstructionTlsThroughGenerationForTest(7);

    try std.testing.expectError(error.ApiError, fixture.agent.request(null));
    try std.testing.expectEqual(@as(usize, 8), fixture.tlsNotes());
    try std.testing.expect(std.mem.indexOf(u8, fixture.agent.last_api_error.?, "gave up after 6 attempts") != null);
    // Eight TLS retries plus five ordinary retries, not a new 16-attempt transport budget.
    try std.testing.expectEqual(@as(usize, 13), std.mem.count(u8, fixture.trace_output.written(), "\"ev\":\"retry\""));
    try fixture.expectReleased();
}

test "TLS outage budget: throttles still get exactly five attempts after TLS recovery" {
    for ([_][]const u8{ "429 Too Many Requests", "503 Service Unavailable" }) |status| {
        var fixture: Fixture = undefined;
        try fixture.init();
        defer fixture.deinit();
        const reply: transport.Reply = .{ .status = status, .body = "{\"error\":{\"message\":\"try later\"}}" };
        const replies = [_]transport.Reply{ reply, reply, reply, reply, reply };
        fixture.serve(&replies);
        clients.injectConstructionTlsThroughGenerationForTest(7);

        try std.testing.expectError(error.ApiError, fixture.agent.request(null));
        try std.testing.expectEqual(@as(usize, 8), fixture.tlsNotes());
        try std.testing.expectEqual(@as(usize, 5), fixture.accepted.load(.acquire));
        try std.testing.expect(std.mem.indexOf(u8, fixture.agent.last_api_error.?, "gave up after 5 attempts") != null);
        try fixture.expectReleased();
    }
}

test "TLS outage budget: terminal authentication and quota remain fail-fast after TLS recovery" {
    const cases = [_]struct { reply: transport.Reply, err: anyerror }{
        .{ .reply = .{ .status = "401 Unauthorized", .body = "{}" }, .err = error.ProviderAuthenticationFailed },
        .{ .reply = .{ .status = "429 Too Many Requests", .body = "{\"error\":{\"code\":\"insufficient_quota\"}}" }, .err = error.ProviderQuotaExceeded },
    };
    for (cases) |case| {
        var fixture: Fixture = undefined;
        try fixture.init();
        defer fixture.deinit();
        const replies = [_]transport.Reply{case.reply};
        fixture.serve(&replies);
        clients.injectConstructionTlsThroughGenerationForTest(7);

        try std.testing.expectError(case.err, fixture.agent.request(null));
        try std.testing.expectEqual(@as(usize, 8), fixture.tlsNotes());
        try std.testing.expectEqual(@as(usize, 1), fixture.accepted.load(.acquire));
        try fixture.expectReleased();
    }
}