Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
13.3 KB
Raw Blame
//! RLM `llm_query`: a tools-off sub-LM call on the session provider.
//!
//! [Recursive Language Models](https://github.com/alexzhang13/rlm) treat a
//! long context as a REPL variable and peel questions off with `llm_query` /
//! `rlm_query`. Here the host function is that sub-call: one bounded
//! completion, no tools, so spec_ptc can launch it the moment the prompt
//! literal closes — overlapping generation of the rest of the script.
//! On Claude Agent (ADR 0309) the call goes to a throwaway `claude` process
//! instead of an HTTP endpoint (claude_agent_query.zig).

const std = @import("std");
const Io = std.Io;
const Value = std.json.Value;
const Allocator = std.mem.Allocator;

const tools = @import("tools.zig");
const ToolCtx = tools.ToolCtx;
const ToolOutput = tools.ToolOutput;
const provider_mod = @import("provider.zig");
const Provider = provider_mod.Provider;
const http = @import("http.zig");
const title = @import("title.zig");

pub const max_out_tokens: u32 = 256;

pub fn run(ctx: ToolCtx, args_json: []const u8) ToolOutput {
    var parsed = std.json.parseFromSlice(Value, ctx.gpa, args_json, .{}) catch {
        return .{ .text = ctx.gpa.dupe(u8, "rlm: llm_query needs prompt") catch &.{}, .is_error = true };
    };
    defer parsed.deinit();
    const prompt = tools.strField(parsed.value, "prompt") orelse {
        return .{ .text = ctx.gpa.dupe(u8, "rlm: llm_query needs prompt") catch &.{}, .is_error = true };
    };
    if (prompt.len == 0) {
        return .{ .text = ctx.gpa.dupe(u8, "rlm: llm_query prompt is empty") catch &.{}, .is_error = true };
    }
    var permit: ?@import("run_budget.zig").Permit = null;
    if (ctx.run_budget) |budget| {
        permit = budget.acquire(ctx.io, ctx.depth, .child) catch |err| return .{
            .text = std.fmt.allocPrint(ctx.gpa, "rlm: llm_query refused: {s}", .{@errorName(err)}) catch unreachable,
            .is_error = true,
        };
    }
    defer if (permit) |*p| p.release();
    const body = buildBody(ctx.gpa, ctx.provider, prompt) catch {
        return .{ .text = ctx.gpa.dupe(u8, "rlm: llm_query could not build request") catch &.{}, .is_error = true };
    };
    defer ctx.gpa.free(body);
    // Claude Agent has no HTTP endpoint (ADR 0309). The query runs on its own
    // throwaway `claude` process, like a title; the chat's held session is
    // keyed by its Agent and never taken here.
    if (std.mem.eql(u8, ctx.provider.id, "claude-agent")) return viaClaudeAgent(ctx, body);
    const raw = http.postWatched(ctx.gpa, ctx.io, ctx.client, ctx.provider, body, null) catch |err| {
        return .{
            .text = std.fmt.allocPrint(ctx.gpa, "rlm: llm_query failed: {s}", .{@errorName(err)}) catch &.{},
            .is_error = true,
        };
    };
    defer ctx.gpa.free(raw);
    return extract(ctx.gpa, ctx.provider.kind, raw);
}

fn viaClaudeAgent(ctx: ToolCtx, body: []const u8) ToolOutput {
    const reply = @import("claude_agent_query.zig").post(ctx.gpa, ctx.io, ctx.provider, body, .{
        .cwd = ctx.agent_cwd,
        .effort = "low",
        .tracer = ctx.tracer,
    }) catch |err| return .{
        .text = std.fmt.allocPrint(ctx.gpa, "rlm: llm_query failed: {s}", .{@errorName(err)}) catch &.{},
        .is_error = true,
    };
    defer reply.deinit(ctx.gpa);
    return switch (reply) {
        .message => |raw| extract(ctx.gpa, .anthropic, raw),
        .problem => |text| .{
            .text = std.fmt.allocPrint(ctx.gpa, "rlm: llm_query failed: {s}", .{text}) catch &.{},
            .is_error = true,
        },
    };
}

pub fn buildBody(gpa: Allocator, provider: Provider, prompt: []const u8) ![]u8 {
    var aw: Io.Writer.Allocating = .init(gpa);
    errdefer aw.deinit();
    var s: std.json.Stringify = .{ .writer = &aw.writer };
    try s.beginObject();
    try s.objectField("model");
    try s.write(provider.model);
    switch (provider.kind) {
        .openai => {
            try s.objectField("max_tokens");
            try s.write(max_out_tokens);
            try s.objectField("messages");
            try s.beginArray();
            try s.beginObject();
            try s.objectField("role");
            try s.write("user");
            try s.objectField("content");
            try s.write(prompt);
            try s.endObject();
            try s.endArray();
        },
        .responses => if (std.mem.eql(u8, provider.id, "chatgpt-new")) {
            // The ChatGPT plan route takes only stored-nothing streamed
            // requests with an input array, and no output cap (ADR 0221).
            try s.objectField("input");
            try s.beginArray();
            try s.beginObject();
            try s.objectField("role");
            try s.write("user");
            try s.objectField("content");
            try s.write(prompt);
            try s.endObject();
            try s.endArray();
            try s.objectField("reasoning");
            try s.beginObject();
            try s.objectField("effort");
            try s.write("low");
            try s.endObject();
            try s.objectField("store");
            try s.write(false);
            try s.objectField("stream");
            try s.write(true);
        } else {
            try s.objectField("max_output_tokens");
            try s.write(max_out_tokens);
            try s.objectField("input");
            try s.write(prompt);
        },
        // Interactions takes a bare string as `input` like Responses, but caps
        // output under generation_config and keeps nothing when store is false.
        .interactions => {
            try s.objectField("store");
            try s.write(false);
            try s.objectField("generation_config");
            try s.beginObject();
            try s.objectField("max_output_tokens");
            try s.write(max_out_tokens);
            try s.endObject();
            try s.objectField("input");
            try s.write(prompt);
        },
        .anthropic => {
            const oauth = @import("anthropic_oauth_wire.zig");
            if (oauth.active(provider)) {
                try s.objectField("system");
                try s.beginArray();
                try oauth.writeSystemPrefix(&s);
                try s.endArray();
            }
            // ADR 0219: a Claude model that thinks by default spends max_tokens
            // on thinking first; 256 left no answer. Room, and little thinking.
            const claude = @import("claude_wire.zig");
            const thinks = claude.isClaudeApi(provider.id, provider.model) and claude.thinksByDefault(provider.model);
            try s.objectField("max_tokens");
            try s.write(if (thinks) @as(u32, 4096) else max_out_tokens);
            if (thinks and claude.takesEffort(provider.model)) {
                try s.objectField("output_config");
                try s.print("{{\"effort\":\"low\"}}", .{});
            }
            try s.objectField("messages");
            try s.beginArray();
            try s.beginObject();
            try s.objectField("role");
            try s.write("user");
            try s.objectField("content");
            try s.write(prompt);
            try s.endObject();
            try s.endArray();
        },
    }
    try s.endObject();
    return aw.toOwnedSlice();
}

/// The `response.output_text.delta` text of a streamed body, or null when
/// `raw` is a plain JSON response.
fn sseText(gpa: Allocator, raw: []const u8) ?[]u8 {
    const head = std.mem.trimStart(u8, raw, " \t\r\n");
    if (!std.mem.startsWith(u8, head, "event:") and !std.mem.startsWith(u8, head, "data:")) return null;
    var out: std.ArrayList(u8) = .empty;
    var lines = std.mem.splitScalar(u8, raw, '\n');
    while (lines.next()) |line| {
        const data = std.mem.trim(u8, line, " \r");
        if (!std.mem.startsWith(u8, data, "data:")) continue;
        var parsed = std.json.parseFromSlice(Value, gpa, std.mem.trim(u8, data["data:".len..], " "), .{}) catch continue;
        defer parsed.deinit();
        if (parsed.value != .object) continue;
        const kind = tools.strField(parsed.value, "type") orelse continue;
        if (!std.mem.eql(u8, kind, "response.output_text.delta")) continue;
        out.appendSlice(gpa, tools.strField(parsed.value, "delta") orelse continue) catch {};
    }
    return out.toOwnedSlice(gpa) catch null;
}

fn extract(gpa: Allocator, kind: Provider.Kind, raw: []const u8) ToolOutput {
    if (sseText(gpa, raw)) |streamed| {
        defer gpa.free(streamed);
        const text = std.mem.trim(u8, streamed, " \t\r\n");
        if (text.len == 0) return .{ .text = gpa.dupe(u8, "rlm: llm_query returned no text") catch &.{}, .is_error = true };
        return .{ .text = gpa.dupe(u8, text) catch &.{} };
    }
    var parsed = std.json.parseFromSlice(Value, gpa, raw, .{}) catch {
        return .{ .text = gpa.dupe(u8, "rlm: llm_query returned non-JSON") catch &.{}, .is_error = true };
    };
    defer parsed.deinit();
    if (parsed.value != .object) {
        return .{ .text = gpa.dupe(u8, "rlm: llm_query returned a non-object") catch &.{}, .is_error = true };
    }
    const text = std.mem.trim(u8, title.assistantText(kind, parsed.value.object), " \t\r\n");
    if (text.len == 0) {
        return .{ .text = gpa.dupe(u8, "rlm: llm_query returned no text") catch &.{}, .is_error = true };
    }
    return .{ .text = gpa.dupe(u8, text) catch &.{} };
}

fn sampleProvider(kind: Provider.Kind) Provider {
    return .{
        .id = "xai",
        .kind = kind,
        .auth = .bearer,
        .url = "http://127.0.0.1/v1",
        .api_key = "test",
        .model = "grok-4.6",
        .context = 128_000,
    };
}

test "llm_query body is tools-off and carries the prompt on every wire" {
    const gpa = std.testing.allocator;
    const prompt = "sum the first line of each chunk";
    for ([_]Provider.Kind{ .openai, .responses, .anthropic }) |kind| {
        const body = try buildBody(gpa, sampleProvider(kind), prompt);
        defer gpa.free(body);
        try std.testing.expect(std.mem.indexOf(u8, body, prompt) != null);
        try std.testing.expect(std.mem.indexOf(u8, body, "\"tools\"") == null);
        try std.testing.expect(std.mem.indexOf(u8, body, "grok-4.6") != null);
    }
}

test "llm_query on the ChatGPT plan route streams, stores nothing and reads SSE text" {
    const gpa = std.testing.allocator;
    var p = sampleProvider(.responses);
    p.id = "chatgpt-new";
    p.model = "gpt-6.1-sol";
    const body = try buildBody(gpa, p, "hi");
    defer gpa.free(body);
    for ([_][]const u8{ "\"stream\":true", "\"store\":false", "\"role\":\"user\"" }) |part|
        try std.testing.expect(std.mem.indexOf(u8, body, part) != null);
    try std.testing.expect(std.mem.indexOf(u8, body, "max_output_tokens") == null);
    const sse = "event: response.created\ndata: {\"type\":\"response.created\"}\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"4\"}\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"2\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"output\":[]}}\n\n";
    const out = extract(gpa, .responses, sse);
    defer gpa.free(out.text);
    try std.testing.expect(!out.is_error);
    try std.testing.expectEqualStrings("42", out.text);
}

test "llm_query extract reads assistant text from each wire shape" {
    const gpa = std.testing.allocator;
    const openai = "{\"choices\":[{\"message\":{\"content\":\"42\"}}]}";
    const out_oa = extract(gpa, .openai, openai);
    defer gpa.free(out_oa.text);
    try std.testing.expect(!out_oa.is_error);
    try std.testing.expectEqualStrings("42", out_oa.text);

    const resp = "{\"output\":[{\"type\":\"message\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\"}]}]}";
    const out_r = extract(gpa, .responses, resp);
    defer gpa.free(out_r.text);
    try std.testing.expect(!out_r.is_error);
    try std.testing.expectEqualStrings("ok", out_r.text);
}

test "llm_query cannot bypass exhausted aggregate model budget" {
    const a = std.testing.allocator;
    var client: std.http.Client = .{ .allocator = a, .io = std.testing.io };
    defer client.deinit();
    var budget: @import("run_budget.zig").RunBudget = .{ .max_model_calls = 1 };
    var first = try budget.acquire(std.testing.io, 0, .root);
    first.release();
    const out = run(.{ .gpa = a, .io = std.testing.io, .client = &client, .provider = undefined, .registry = null, .from_sub = true, .approvals = null, .tracer = null, .run_budget = &budget }, "{\"prompt\":\"never contact a provider\"}");
    defer a.free(out.text);
    try std.testing.expect(out.is_error);
    try std.testing.expect(std.mem.indexOf(u8, out.text, "RunBudgetExhausted") != null);
    try std.testing.expectEqual(@as(u64, 1), budget.used());
}

test "llm_query gives a thinking Claude model room to answer, at low effort (ADR 0219)" {
    const gpa = std.testing.allocator;
    var p = sampleProvider(.anthropic);
    p.id = "anthropic";
    p.model = "claude-opus-5-5";
    const body = try buildBody(gpa, p, "one word");
    defer gpa.free(body);
    try std.testing.expect(std.mem.indexOf(u8, body, "\"max_tokens\":4096") != null);
    try std.testing.expect(std.mem.indexOf(u8, body, "\"output_config\":{\"effort\":\"low\"}") != null);
    p.model = "claude-haiku-4-5"; // no default thinking: the small budget stands
    const small = try buildBody(gpa, p, "one word");
    defer gpa.free(small);
    try std.testing.expect(std.mem.indexOf(u8, small, "\"max_tokens\":256") != null);
    try std.testing.expect(std.mem.indexOf(u8, small, "output_config") == null);
}