Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
24.4 KB
Raw Blame
//! Capped subprocess execution with optional whole-process-tree ownership.

const std = @import("std");
const builtin = @import("builtin");
const Io = std.Io;
const Allocator = std.mem.Allocator;
const Agent = @import("agent.zig").Agent;

const posix_process_groups = switch (builtin.os.tag) {
    .windows, .wasi => false,
    else => true,
};
const tool_pulse = @import("tool_pulse.zig");
const output_elide = @import("output_elide.zig");
const GroupId = if (posix_process_groups) std.posix.pid_t else void;
const WindowsJobHandle = if (builtin.os.tag == .windows) ?std.os.windows.HANDLE else void;

const WinJob = if (builtin.os.tag == .windows) struct {
    const windows = std.os.windows;
    const kill_on_close: windows.DWORD = 0x00002000;
    const extended_limit_information: c_int = 9;

    const BasicLimitInformation = extern struct {
        per_process_user_time_limit: windows.LARGE_INTEGER = 0,
        per_job_user_time_limit: windows.LARGE_INTEGER = 0,
        limit_flags: windows.DWORD = 0,
        minimum_working_set_size: windows.SIZE_T = 0,
        maximum_working_set_size: windows.SIZE_T = 0,
        active_process_limit: windows.DWORD = 0,
        affinity: windows.ULONG_PTR = 0,
        priority_class: windows.DWORD = 0,
        scheduling_class: windows.DWORD = 0,
    };
    const IoCounters = extern struct {
        read_operation_count: u64 = 0,
        write_operation_count: u64 = 0,
        other_operation_count: u64 = 0,
        read_transfer_count: u64 = 0,
        write_transfer_count: u64 = 0,
        other_transfer_count: u64 = 0,
    };
    const ExtendedLimitInformation = extern struct {
        basic_limit_information: BasicLimitInformation = .{},
        io_info: IoCounters = .{},
        process_memory_limit: windows.SIZE_T = 0,
        job_memory_limit: windows.SIZE_T = 0,
        peak_process_memory_used: windows.SIZE_T = 0,
        peak_job_memory_used: windows.SIZE_T = 0,
    };

    extern "kernel32" fn CreateJobObjectW(
        job_attributes: ?*windows.SECURITY_ATTRIBUTES,
        name: ?windows.LPCWSTR,
    ) callconv(.winapi) ?windows.HANDLE;
    extern "kernel32" fn SetInformationJobObject(
        job: windows.HANDLE,
        info_class: c_int,
        info: *const anyopaque,
        info_len: windows.DWORD,
    ) callconv(.winapi) windows.BOOL;
    extern "kernel32" fn AssignProcessToJobObject(
        job: windows.HANDLE,
        process: windows.HANDLE,
    ) callconv(.winapi) windows.BOOL;

    fn setup(child: *std.process.Child) !windows.HANDLE {
        const job = CreateJobObjectW(null, null) orelse return error.ProcessTreeSetupFailed;
        errdefer windows.CloseHandle(job);
        var limits: ExtendedLimitInformation = .{};
        limits.basic_limit_information.limit_flags = kill_on_close;
        if (!SetInformationJobObject(job, extended_limit_information, &limits, @sizeOf(ExtendedLimitInformation)).toBool())
            return error.ProcessTreeSetupFailed;
        if (!AssignProcessToJobObject(job, child.id.?).toBool()) return error.ProcessTreeSetupFailed;
        if (windows.ntdll.NtResumeThread(child.thread_handle, null) != .SUCCESS)
            return error.ProcessTreeSetupFailed;
        return job;
    }
} else struct {};

pub const CappedRun = struct {
    term: std.process.Child.Term,
    stdout: []u8,
    stderr: []u8,
    stdout_truncated: bool,
    stderr_truncated: bool,
    timed_out: bool,
    cancelled: bool = false, // user Esc ended it (the process tree was killed)
};

/// Supplying an explicit environment replaces (rather than augments) the
/// parent environment. `kill_process_tree` creates an owned process group/job
/// so descendants cannot outlive the orchestrated command.
///
/// `stream` is grok-build's bash_output_chunk: each newly-arrived stdout (0)
/// or stderr (1) slice while the child is still running. Foreground bash used
/// to stay silent until exit, so a long build looked like a hang.
pub const StreamFn = *const fn (ctx: ?*anyopaque, which: u8, chunk: []const u8) void;

pub const CappedRunOptions = struct {
    cwd: std.process.Child.Cwd = .inherit,
    environ_map: ?*const std.process.Environ.Map = null,
    kill_process_tree: bool = false,
    stream: ?StreamFn = null,
    stream_ctx: ?*anyopaque = null,
    /// Silence heartbeat (#607): called with ms since spawn once a run has
    /// been quiet past tool_pulse's schedule, so a long build never reads as
    /// a hang. Raw chunks still go through `stream`; this is chrome only.
    pulse: ?*const fn (ctx: ?*anyopaque, elapsed_ms: u64) void = null,
    /// Schedule overrides (tests run the real thresholds in milliseconds).
    pulse_first_ms: u64 = tool_pulse.first_silence_ms,
    pulse_interval_ms: u64 = tool_pulse.silence_interval_ms,
    /// Clipboard export is not a user turn. Esc would kill osascript and
    /// Ctrl-V would report a cancelled clipboard read (#1146).
    ignore_cancel: bool = false,
    /// Passed to the child as its stdin. A `--json` child reads one request
    /// line, runs the turn and exits on EOF, so this drives a machine-readable
    /// one-shot (the summarizer) without a pipe the parent would have to feed
    /// concurrently — a write larger than the pipe buffer would deadlock
    /// against a child that has not started reading yet.
    stdin_file: ?Io.File = null,
    /// #1271: over the cap, keep the first third and the LAST two thirds of
    /// each stream around a `[... N bytes truncated ...]` gap instead of only
    /// the first `cap` bytes. The tail lives in a fixed ring (never more than
    /// the cap), and the joined result still fits the cap. Off by default:
    /// programmatic callers parse the capture or reject a truncated one, and
    /// the bash tool is the reader that needs the ending (test summaries,
    /// final errors).
    keep_tail: bool = false,
};

/// #253: RLIMIT_NOFILE as this process actually observes it. main() calls
/// `std.process.raiseFileDescriptorLimit()` at startup but nothing ever
/// reported the result, so a "ProcessFdQuotaExceeded blocks every shell
/// command" report could not tell "the soft limit never rose" apart from "it
/// rose and the parallel tool/subagent fan-out still exhausted it".
pub const FdLimit = struct { soft: u64, hard: u64 };

/// Same platform predicate as `posix_process_groups`: no rlimits on Windows/WASI.
const have_rlimits = posix_process_groups;

/// `null` where the platform has no rlimits (Windows/WASI) or the query fails.
pub fn fdLimitSnapshot() ?FdLimit {
    if (comptime have_rlimits) {
        const lim = std.posix.getrlimit(.NOFILE) catch return null;
        return .{
            .soft = std.math.cast(u64, lim.cur) orelse std.math.maxInt(u64),
            .hard = std.math.cast(u64, lim.max) orelse std.math.maxInt(u64),
        };
    } else {
        return null;
    }
}

/// #253: the one line the next fd-exhaustion report needs. Rendered into
/// `buf` rather than printed so it can be asserted on without exhausting a
/// real descriptor table; truncation is impossible for the shapes we format
/// but is reported as the bare error name rather than silently dropped.
pub fn fdQuotaNote(buf: []u8, err_name: []const u8, limit: ?FdLimit) []const u8 {
    const rendered = if (limit) |lim| std.fmt.bufPrint(
        buf,
        "  [fd] spawn failed: {s}; RLIMIT_NOFILE soft={d} hard={d} (#253)\n",
        .{ err_name, lim.soft, lim.hard },
    ) else std.fmt.bufPrint(
        buf,
        "  [fd] spawn failed: {s}; RLIMIT_NOFILE unavailable on this platform (#253)\n",
        .{err_name},
    );
    return rendered catch err_name;
}

fn reportFdQuota(err: anyerror) void {
    var buf: [160]u8 = undefined;
    std.debug.print("{s}", .{fdQuotaNote(&buf, @errorName(err), fdLimitSnapshot())});
}

fn setupWindowsJob(child: *std.process.Child, enabled: bool) !WindowsJobHandle {
    if (comptime builtin.os.tag == .windows) {
        return if (enabled) try WinJob.setup(child) else null;
    } else {
        return {};
    }
}

fn closeWindowsJob(job: *WindowsJobHandle) void {
    if (comptime builtin.os.tag == .windows) if (job.*) |handle| {
        std.os.windows.CloseHandle(handle);
        job.* = null;
    };
}

fn killProcessGroup(group_id: GroupId, enabled: bool) void {
    if (comptime posix_process_groups) {
        if (!enabled) return;
        std.posix.kill(-group_id, .KILL) catch {};
    }
}

fn terminateTree(child: *std.process.Child, io: Io, job: *WindowsJobHandle, group_id: GroupId, enabled: bool) void {
    closeWindowsJob(job);
    killProcessGroup(group_id, enabled);
    child.kill(io);
}

// A canceled POSIX wait closes the pipes and clears id without reaping.
// Retain the id so the caller can still kill/reap after a deadline or Esc.
fn waitForExit(child: *std.process.Child, io: Io, started: Io.Timestamp, deadline_ms: u64, cancelled: *bool, timed_out: *bool, ignore_cancel: bool) !std.process.Child.Term {
    const id = child.id;
    const Event = union(enum) { term: std.process.Child.WaitError!std.process.Child.Term, tick: Io.Cancelable!void };
    var buffer: [2]Event = undefined;
    var select = Io.Select(Event).init(io, &buffer);
    try select.concurrent(.term, std.process.Child.wait, .{ child, io });
    defer {
        while (select.cancel()) |event| switch (event) {
            .term => |result| {
                _ = result catch |err| {
                    if (comptime posix_process_groups) if (err == error.Canceled) {
                        child.id = id;
                    };
                };
            },
            .tick => {},
        };
    }
    while (true) {
        select.async(.tick, Io.sleep, .{ io, .fromMilliseconds(20), .awake });
        switch (try select.await()) {
            .term => |result| return result catch |err| {
                if (comptime posix_process_groups) if (err == error.Canceled) {
                    child.id = id;
                };
                return err;
            },
            .tick => |result| try result,
        }
        cancelled.* = !ignore_cancel and Agent.esc_cancel.load(.acquire);
        timed_out.* = deadline_ms > 0 and started.untilNow(io, .awake).toMilliseconds() >= deadline_ms;
        if (cancelled.* or timed_out.*) {
            while (select.cancel()) |event| switch (event) {
                .term => |result| return result catch |err| {
                    if (err != error.Canceled) return err;
                    if (comptime posix_process_groups) child.id = id;
                    return .{ .signal = .TERM };
                },
                .tick => {},
            };
            unreachable;
        }
    }
}

pub fn runCapped(gpa: Allocator, io: Io, argv: []const []const u8, stdout_cap: usize, stderr_cap: usize, deadline_ms: u64) !CappedRun {
    return runCappedWithOptions(gpa, io, argv, stdout_cap, stderr_cap, deadline_ms, .{});
}

pub fn runCappedWithOptions(gpa: Allocator, io: Io, argv: []const []const u8, stdout_cap: usize, stderr_cap: usize, deadline_ms: u64, options: CappedRunOptions) !CappedRun {
    var child = std.process.spawn(io, .{
        .argv = argv,
        .cwd = options.cwd,
        .environ_map = options.environ_map,
        .stdin = if (options.stdin_file) |f| .{ .file = f } else .ignore,
        .stdout = .pipe,
        .stderr = .pipe,
        .pgid = if (posix_process_groups and options.kill_process_tree) 0 else null,
        .start_suspended = builtin.os.tag == .windows and options.kill_process_tree,
    }) catch |err| switch (err) {
        // #253: fd quota is the one spawn failure where the *limit* is the
        // evidence, and it is gone by the time anyone reads the transcript.
        // Name it at the failure site; the error itself propagates unchanged.
        error.ProcessFdQuotaExceeded, error.SystemFdQuotaExceeded => {
            reportFdQuota(err);
            return err;
        },
        else => return err,
    };
    var setup_cleanup_needed = true;
    errdefer if (setup_cleanup_needed) child.kill(io);
    const group_id: GroupId = if (posix_process_groups) child.id.? else {};
    var windows_job = try setupWindowsJob(&child, options.kill_process_tree);
    setup_cleanup_needed = false;
    var cleanup_needed = true;
    defer if (cleanup_needed) terminateTree(&child, io, &windows_job, group_id, options.kill_process_tree);

    var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
    var multi_reader: Io.File.MultiReader = undefined;
    multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
    defer multi_reader.deinit();

    const readers = [2]*Io.Reader{ multi_reader.reader(0), multi_reader.reader(1) };
    const caps = [2]usize{ stdout_cap, stderr_cap };
    var saved: [2]?[]u8 = .{ null, null };
    errdefer for (saved) |item| if (item) |bytes| gpa.free(bytes);
    var streamed: [2]usize = .{ 0, 0 };
    var rings: [2]?output_elide.TailRing = .{ null, null };
    defer for (&rings) |*ring| if (ring.*) |*r| r.deinit(gpa);
    var after_head: [2]usize = .{ 0, 0 };

    var esc_killed = false;
    var timed_out = false;
    const started: Io.Timestamp = .now(io, .awake);
    // Silence-heartbeat clock (#607): reset whenever either pipe produces a
    // fresh byte, read by tool_pulse's schedule at each 200ms poll tick.
    var quiet_since: Io.Timestamp = started;
    var pulse: tool_pulse.Pulse = .{ .next_at_ms = options.pulse_first_ms, .interval_ms = options.pulse_interval_ms };
    loop: while (true) {
        var eof = false;
        multi_reader.fill(64, .{ .duration = .{ .raw = .fromMilliseconds(200), .clock = .awake } }) catch |err| switch (err) {
            error.EndOfStream => eof = true,
            error.Timeout => {},
            else => |other| return other,
        };
        for (readers, caps, &saved, &streamed, &rings, &after_head, [_]u8{ 0, 1 }) |reader, cap, *item, *seen, *ring, *tailed, which| {
            const buffered = reader.buffered();
            // Fresh bytes this tick: anything past the streamed mark, or any
            // residue at all once capping started tossing what it saved.
            if ((item.* != null and buffered.len > 0) or buffered.len > seen.*) quiet_since = .now(io, .awake);
            if (options.stream) |fn_emit| {
                if (buffered.len > seen.*) {
                    const end = if (item.* != null) @min(buffered.len, cap) else buffered.len;
                    if (end > seen.*) fn_emit(options.stream_ctx, which, buffered[seen.*..end]);
                    seen.* = end;
                }
            }
            if (item.* == null) {
                if (buffered.len > cap) {
                    const split = options.keep_tail and cap >= output_elide.min_budget;
                    const avail = cap -| output_elide.gap_reserve;
                    const head_len = if (split) @import("util.zig").utf8Prefix(buffered, output_elide.headShare(avail)).len else cap;
                    item.* = try gpa.dupe(u8, buffered[0..head_len]);
                    if (split) {
                        ring.* = try output_elide.TailRing.init(gpa, avail - head_len);
                        ring.*.?.push(buffered[head_len..]);
                        tailed.* = buffered.len - head_len;
                    }
                }
            } else if (ring.*) |*r| {
                r.push(buffered);
                tailed.* += buffered.len;
            }
            if (item.* != null) reader.toss(buffered.len);
        }
        if (!eof) {
            const silence: u64 = @intCast(@max(0, quiet_since.untilNow(io, .awake).toMilliseconds()));
            if (pulse.due(silence))
                if (options.pulse) |pulse_fn| pulse_fn(options.stream_ctx, @intCast(@max(0, started.untilNow(io, .awake).toMilliseconds())));
        }
        if (eof) break :loop;
        if (!options.ignore_cancel and Agent.esc_cancel.load(.acquire)) {
            esc_killed = true;
            terminateTree(&child, io, &windows_job, group_id, options.kill_process_tree);
            cleanup_needed = false;
            break :loop;
        }
        if (deadline_ms > 0 and started.untilNow(io, .awake).toMilliseconds() >= deadline_ms) {
            timed_out = true;
            terminateTree(&child, io, &windows_job, group_id, options.kill_process_tree);
            cleanup_needed = false;
            break :loop;
        }
    }
    if (!esc_killed and !timed_out) try multi_reader.checkAnyError();

    const term: std.process.Child.Term = if (esc_killed or timed_out) .{ .signal = .TERM } else blk: {
        const term = try waitForExit(&child, io, started, deadline_ms, &esc_killed, &timed_out, options.ignore_cancel);
        if (esc_killed or timed_out) {
            terminateTree(&child, io, &windows_job, group_id, options.kill_process_tree);
            cleanup_needed = false;
        }
        break :blk term;
    };
    for (&saved, &rings, after_head) |*item, *ring, tailed| if (ring.*) |*r| {
        const joined = try output_elide.joinRing(gpa, item.*.?, r, tailed);
        gpa.free(item.*.?);
        item.* = joined;
    };
    const stdout = if (saved[0]) |bytes| bytes else try gpa.dupe(u8, readers[0].buffered());
    errdefer if (saved[0] == null) gpa.free(stdout);
    const stderr = if (saved[1]) |bytes| bytes else try gpa.dupe(u8, readers[1].buffered());
    return .{
        .term = term,
        .stdout = stdout,
        .stderr = stderr,
        .stdout_truncated = saved[0] != null,
        .stderr_truncated = saved[1] != null,
        .timed_out = timed_out,
        .cancelled = esc_killed,
    };
}

pub fn ranOk(run: CappedRun) bool {
    return run.term == .exited and run.term.exited == 0;
}

const PulseCapture = struct {
    var fired: usize = 0;
    var last_ms: u64 = 0;
    fn fire(ctx: ?*anyopaque, elapsed_ms: u64) void {
        _ = ctx;
        fired += 1;
        last_ms = elapsed_ms;
    }
};

test "#607: a quiet run fires the silence heartbeat; output resets it" {
    if (builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest;
    const gpa = std.testing.allocator;
    const io = std.testing.io;
    PulseCapture.fired = 0;
    const quiet = try runCappedWithOptions(gpa, io, &.{ "/bin/sh", "-c", "sleep 0.45" }, 1024, 1024, 0, .{
        .pulse = PulseCapture.fire,
        .pulse_first_ms = 50,
        .pulse_interval_ms = 120,
    });
    defer {
        gpa.free(quiet.stdout);
        gpa.free(quiet.stderr);
    }
    // ~450ms of silence past a 50ms first beat at 120ms spacing: several beats.
    try std.testing.expect(PulseCapture.fired >= 2);
    try std.testing.expect(PulseCapture.last_ms >= 50);

    // A run that keeps producing never crosses the silence schedule.
    PulseCapture.fired = 0;
    const chatty = try runCappedWithOptions(gpa, io, &.{ "/bin/sh", "-c", "i=0; while [ $i -lt 12 ]; do echo tick; i=$((i+1)); sleep 0.05; done" }, 4096, 1024, 0, .{
        .pulse = PulseCapture.fire,
        .pulse_first_ms = 50,
        .pulse_interval_ms = 120,
    });
    defer {
        gpa.free(chatty.stdout);
        gpa.free(chatty.stderr);
    }
    try std.testing.expectEqual(@as(usize, 0), PulseCapture.fired);
}

/// #253: descriptor high-water probe. `dup` always hands back the LOWEST free
/// slot, so claiming a fixed burst and taking the highest number returned
/// counts everything the process is holding below that ceiling — unlike a
/// single `dup`, which only sees the lowest gap and is blind to a leak that
/// sits above one. Every descriptor claimed here is released again.
/// `std.posix.system.dup` is the RAW syscall wrapper and its return type is
/// version-dependent: signed on Zig 0.16, unsigned on the 0.17 nightly CI
/// pins (there is no checked `std.posix.dup` in either). Assigning it
/// directly built locally and failed the CI zig job, so normalize on
/// signedness at comptime instead of assuming a width.
fn dupLowestFree() ?std.posix.fd_t {
    const raw = std.posix.system.dup(0);
    const signed: isize = if (@typeInfo(@TypeOf(raw)).int.signedness == .signed) @intCast(raw) else @bitCast(raw);
    if (signed < 0) return null; // out of descriptors: the probe stops here
    return @intCast(signed);
}

fn fdHighWater() i32 {
    if (comptime !have_rlimits) return -1; // no POSIX descriptor table to probe
    var claimed: [64]std.posix.fd_t = undefined;
    var n: usize = 0;
    var high: i32 = -1;
    while (n < claimed.len) : (n += 1) {
        claimed[n] = dupLowestFree() orelse break;
        const numbered: i32 = @intCast(claimed[n]);
        if (numbered > high) high = numbered;
    }
    for (claimed[0..n]) |fd| _ = std.posix.system.close(fd);
    return high;
}

test "#253: timed-out runs do not leak the child's stdout/stderr descriptors" {
    if (builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest;
    const io = std.testing.io;
    const gpa = std.testing.allocator;
    const timeout_run = struct {
        fn once(a: Allocator, i: Io) !void {
            const r = try runCapped(a, i, &.{ "/bin/sh", "-c", "sleep 30" }, 1024, 1024, 50);
            a.free(r.stdout);
            a.free(r.stderr);
            try std.testing.expect(r.timed_out);
        }
    }.once;

    // Two warm-up runs first: whatever the Io backend claims once is then
    // already inside the baseline, so only per-run growth can move the probe.
    try timeout_run(gpa, io);
    try timeout_run(gpa, io);
    const warm = fdHighWater();
    var i: usize = 0;
    while (i < 6) : (i += 1) try timeout_run(gpa, io);
    const after = fdHighWater();
    // A branch that terminated the child but never closed its two pipes would
    // push this out by 12. The tolerance absorbs unrelated one-off
    // descriptors, never a per-run leak.
    try std.testing.expect(after - warm < 4);
}

test "#253: fdLimitSnapshot observes a real NOFILE soft limit" {
    const snapshot = fdLimitSnapshot();
    if (comptime !have_rlimits) {
        try std.testing.expect(snapshot == null);
        return;
    }
    const lim = snapshot orelse return error.NoFdLimitObserved;
    // main() raises the soft limit at startup; a snapshot that cannot even see
    // the POSIX floor is reporting nothing useful.
    try std.testing.expect(lim.soft >= 20);
    try std.testing.expect(lim.hard >= lim.soft);
}

test "#253: fdQuotaNote names the error and the observed limit" {
    var buf: [160]u8 = undefined;
    const with = fdQuotaNote(&buf, "ProcessFdQuotaExceeded", .{ .soft = 256, .hard = 10240 });
    try std.testing.expect(std.mem.indexOf(u8, with, "ProcessFdQuotaExceeded") != null);
    try std.testing.expect(std.mem.indexOf(u8, with, "soft=256") != null);
    try std.testing.expect(std.mem.indexOf(u8, with, "hard=10240") != null);
    try std.testing.expect(std.mem.indexOf(u8, with, "#253") != null);
    const without = fdQuotaNote(&buf, "SystemFdQuotaExceeded", null);
    try std.testing.expect(std.mem.indexOf(u8, without, "unavailable") != null);
}

test "runCapped streams stdout before the child exits" {
    if (builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest;
    const io = std.testing.io;
    const gpa = std.testing.allocator;
    const State = struct {
        seen: bool = false,
        fn emit(ctx: ?*anyopaque, which: u8, chunk: []const u8) void {
            const self: *@This() = @ptrCast(@alignCast(ctx orelse return));
            if (which == 0 and std.mem.indexOf(u8, chunk, "STREAM_MARK") != null) self.seen = true;
        }
    };
    var state = State{};
    const r = try runCappedWithOptions(gpa, io, &.{ "/bin/sh", "-c", "printf STREAM_MARK; sleep 0.4; printf DONE" }, 1024, 1024, 2000, .{
        .stream = &State.emit,
        .stream_ctx = &state,
    });
    defer gpa.free(r.stdout);
    defer gpa.free(r.stderr);
    try std.testing.expect(state.seen);
    try std.testing.expect(std.mem.indexOf(u8, r.stdout, "STREAM_MARK") != null);
    try std.testing.expect(std.mem.indexOf(u8, r.stdout, "DONE") != null);
}

test "#1271: keep_tail keeps the end of 3 MB of output within the 1 MB cap" {
    if (builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest;
    const gpa = std.testing.allocator;
    const cap: usize = 1024 * 1024;
    const script = "printf START; head -c 3145728 /dev/zero | tr '\\000' x; printf FINAL-SENTINEL-1271; printf ERR-TAIL >&2";
    const r = try runCappedWithOptions(gpa, std.testing.io, &.{ "/bin/sh", "-c", script }, cap, 4096, 30_000, .{ .keep_tail = true });
    defer gpa.free(r.stdout);
    defer gpa.free(r.stderr);
    try std.testing.expect(r.stdout_truncated);
    try std.testing.expect(r.stdout.len <= cap);
    try std.testing.expect(std.mem.startsWith(u8, r.stdout, "START"));
    try std.testing.expect(std.mem.endsWith(u8, r.stdout, "FINAL-SENTINEL-1271"));
    try std.testing.expect(std.mem.indexOf(u8, r.stdout, "bytes truncated ...]") != null);
    try std.testing.expectEqualStrings("ERR-TAIL", r.stderr);
}