Skip to content

Lightpanda: the headless browser designed for AI and automation

7.4 KB
Raw Blame
// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
//
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

//! `--trace`: one JSON line per model call and tool call, then a summary.

const std = @import("std");
const zenai = @import("zenai");
const lp = @import("lightpanda");

const ToolResult = lp.tools.ToolResult;

const Trace = @This();

file: std.Io.File,
offset: u64 = 0,
// Model calls report from `ModelCall`'s helper thread.
mutex: std.Io.Mutex = .init,
line: std.Io.Writer.Allocating,
started_ms: u64,
model_started_ms: u64 = 0,
model_calls: u32 = 0,
tool_calls: u32 = 0,
with_results: bool,

pub fn create(allocator: std.mem.Allocator, path: []const u8, with_results: bool) !Trace {
    const file = try std.Io.Dir.cwd().createFile(lp.io, path, .{ .truncate = true });
    return .{ .file = file, .line = .init(allocator), .started_ms = now(), .with_results = with_results };
}

pub fn deinit(self: *Trace) void {
    self.line.deinit();
    self.file.close(lp.io);
}

pub fn run(self: *Trace, provider: []const u8, model: []const u8, task: ?[]const u8) void {
    self.write(.{ .type = "run", .t = self.elapsed(), .provider = provider, .model = model, .task = task });
}

pub fn modelStarted(self: *Trace) void {
    self.model_started_ms = now();
}

pub fn modelCall(self: *Trace, phase: []const u8, result: *const zenai.provider.GenerateResult) void {
    self.model_calls += 1;
    self.write(.{
        .type = "model",
        .t = self.elapsed(),
        .step = self.model_calls,
        .phase = phase,
        .ms = now() - self.model_started_ms,
        .tokens = Tokens.of(result.usage),
        .finish = @tagName(result.finish_reason),
        .tool_calls = if (result.tool_calls) |calls| calls.len else 0,
    });
}

pub fn toolCall(self: *Trace, name: []const u8, arguments: ?std.json.Value, result: *const ToolResult, ms: u64, frame: ?*lp.Frame) void {
    self.tool_calls += 1;
    self.write(.{
        .type = "tool",
        .t = self.elapsed(),
        .step = self.tool_calls,
        .name = name,
        .args = arguments,
        .ms = ms,
        .result_bytes = result.text.len,
        .result = if (self.with_results) result.text else null,
        .@"error" = result.is_error,
        .navigated = result.navigated,
        .page = if (frame) |f| lp.tools.pageState(f) else null,
    });
}

pub fn end(self: *Trace, usage: zenai.provider.Usage) void {
    self.write(.{
        .type = "end",
        .t = self.elapsed(),
        .model_calls = self.model_calls,
        .tool_calls = self.tool_calls,
        .tokens = Tokens.of(usage),
    });
}

/// Shared with the `$usage` line so both use the same names and meanings.
pub const Tokens = struct {
    input: i32,
    cached: i32,
    cache_creation: i32,
    output: i32,

    pub fn of(u: zenai.provider.Usage) Tokens {
        return .{
            .input = u.inputTokens(),
            .cached = u.cached_tokens orelse 0,
            .cache_creation = u.cache_creation_tokens orelse 0,
            .output = u.completion_tokens orelse 0,
        };
    }
};

fn now() u64 {
    return lp.datetime.milliTimestamp(.boot);
}

fn elapsed(self: *const Trace) u64 {
    return now() - self.started_ms;
}

/// Written as each step ends, so an interrupted run still leaves a usable file.
fn write(self: *Trace, record: anytype) void {
    self.mutex.lockUncancelable(lp.io);
    defer self.mutex.unlock(lp.io);
    self.line.clearRetainingCapacity();
    std.json.Stringify.value(record, .{ .emit_null_optional_fields = false }, &self.line.writer) catch return;
    self.line.writer.writeByte('\n') catch return;
    self.file.writePositionalAll(lp.io, self.line.written(), self.offset) catch return;
    self.offset += self.line.written().len;
}

const testing = @import("../testing.zig");

test "Trace writes one JSON line per step, arguments as given" {
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    const path = try testing.tmpPath(&tmp, "trace.jsonl");

    var trace: Trace = try .create(std.testing.allocator, path, false);
    trace.run("openai", "gpt-test", "find courts");

    var response: zenai.provider.GenerateResult = .init(std.testing.allocator);
    defer response.deinit();
    response.finish_reason = .tool_call;
    response.usage = .{ .prompt_tokens = 30, .cached_tokens = 70, .completion_tokens = 5 };
    trace.modelStarted();
    trace.modelCall("turn", &response);

    const args = try std.json.parseFromSlice(std.json.Value, std.testing.allocator,
        \\{"selector":"#pw","value":"$LP_PASSWORD"}
    , .{});
    defer args.deinit();
    trace.toolCall("fill", args.value, &.{ .text = "Filled." }, 12, null);
    trace.end(response.usage);
    trace.deinit();

    const written = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, std.testing.allocator, .limited(4096));
    defer std.testing.allocator.free(written);
    var lines = std.mem.splitScalar(u8, std.mem.trimEnd(u8, written, "\n"), '\n');
    const expected_types = [_][]const u8{ "run", "model", "tool", "end" };
    var records: [expected_types.len]std.json.Parsed(std.json.Value) = undefined;
    for (&records, expected_types) |*record, expected| {
        record.* = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, lines.next().?, .{});
        try std.testing.expectEqualStrings(expected, record.value.object.get("type").?.string);
    }
    defer for (&records) |*record| record.deinit();
    try std.testing.expect(lines.next() == null);

    const tokens = records[1].value.object.get("tokens").?.object;
    try std.testing.expectEqual(100, tokens.get("input").?.integer);
    try std.testing.expectEqual(70, tokens.get("cached").?.integer);

    const tool = records[2].value.object;
    try std.testing.expectEqualStrings("$LP_PASSWORD", tool.get("args").?.object.get("value").?.string);
    try std.testing.expectEqual(7, tool.get("result_bytes").?.integer);
    try std.testing.expect(tool.get("result") == null);
    try std.testing.expect(tool.get("page") == null);

    try std.testing.expectEqual(1, records[3].value.object.get("model_calls").?.integer);
}

test "Trace writes result text only when asked" {
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    const path = try testing.tmpPath(&tmp, "trace.jsonl");

    var trace: Trace = try .create(std.testing.allocator, path, true);
    trace.toolCall("markdown", null, &.{ .text = "# Title" }, 3, null);
    trace.deinit();

    const written = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, std.testing.allocator, .limited(4096));
    defer std.testing.allocator.free(written);
    const record = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, std.mem.trimEnd(u8, written, "\n"), .{});
    defer record.deinit();
    try std.testing.expectEqualStrings("# Title", record.value.object.get("result").?.string);
}