Skip to content

Lightpanda: the headless browser designed for AI and automation

agent: --trace writes one JSON line per step

Adrià Arrufat928c9649bc97
diff --git a/build.zig.zon b/build.zig.zon
index b530a5e7a..4d52a5ed9 100644
--- a/build.zig.zon
+++ b/build.zig.zon
@@ -40,8 +40,8 @@
.hash = "N-V-__8AALc_rgC04POe18O0LKFRm4ouQpLtL3riscEmWp-Q",
},
.zenai = .{
- .url = "git+https://github.com/lightpanda-io/zenai.git#4a1cb225e99598bc6726612c859a7d811d0341b9",
- .hash = "zenai-0.0.0-iOY_VBGZBwDtENiQi52iT4tzS7M8pkB-P-KQvFtVnX_J",
+ .url = "git+https://github.com/lightpanda-io/zenai.git#85f9e359096a82732d59d2f1d96777cb31615daf",
+ .hash = "zenai-0.0.0-iOY_VASbBwCT391IzMrstG7bV8ExUlIhqMDEtcs4dc4N",
},
.isocline = .{
.url = "git+https://github.com/arrufat/isocline#4a99434bee4a5ed04c1639514224d48cd55e1405",
diff --git a/src/Config.zig b/src/Config.zig
index 4a9f3da9b..227a044fb 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -469,6 +469,7 @@ const Commands = cli.Builder(.{
.{ .name = "base_url", .type = ?[:0]const u8 },
.{ .name = "system_prompt", .type = ?[:0]const u8 },
.{ .name = "task", .type = ?[]const u8 },
+ .{ .name = "trace", .type = ?[]const u8 },
.{ .name = "save", .type = ?[]const u8 },
.{ .name = "attach", .short = 'a', .type = []const u8, .multiple = true },
.{ .name = "verbosity", .type = ?AgentVerbosity },
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index fca5b7e42..333668be5 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -34,6 +34,7 @@ const Candidate = zenai.provider.Candidate;
const App = @import("../App.zig");
const Conversation = @import("Conversation.zig");
const ModelCall = @import("ModelCall.zig");
+const Trace = @import("Trace.zig");
const Terminal = @import("Terminal.zig");
const SlashCommand = @import("SlashCommand.zig");
const settings = @import("settings.zig");
@@ -196,6 +197,8 @@ capturing_for_save: bool = false,
/// Printed as a structured `$usage ...` line on stderr at the end of `--task`
/// (one-shot) mode so wrappers can capture per-task cost.
total_usage: zenai.provider.Usage = .{},
+trace: ?Trace = null,
+trace_phase: []const u8 = "turn",
/// Set when the last turn ended in a model refusal (safety stop).
last_turn_refused: bool = false,
/// Whether assistant text streams to the terminal as the model produces it.
@@ -374,10 +377,19 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent
try self.ts.init(app);
errdefer self.ts.deinit();
- self.installCancelHook();
self.ai_client = if (self.credential) |*c| try zenai.provider.Client.init(lp.io, allocator, c.provider, c.keySlice(), .{ .base_url = opts.base_url, .retry_policy = .long_running, .bill_to = hfBillTo(c.provider), .environ = lp.environ(), .account_id = c.accountId() }) else null;
errdefer if (self.ai_client) |c| c.deinit(allocator);
+
+ if (opts.trace) |path| {
+ self.trace = Trace.create(allocator, path) catch |err| {
+ log.fatal(.app, "cannot write trace", .{ .path = path, .err = err });
+ return err;
+ };
+ self.trace.?.run(if (self.credential) |c| @tagName(c.provider) else "none", self.model, opts.task);
+ }
+ errdefer if (self.trace) |*t| t.deinit();
+ self.installSessionHooks();
if (self.ai_client) |c| c.setInterrupt(&self.http_interrupt);
if (will_repl) {
@@ -395,6 +407,10 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent
}
pub fn deinit(self: *Agent) void {
+ if (self.trace) |*t| {
+ t.end(self.total_usage);
+ t.deinit();
+ }
self.terminal.uninstallLogSink();
self.save_buffer.deinit();
self.save_calls.deinit(self.allocator);
@@ -419,8 +435,9 @@ fn idlePump(arg: ?*anyopaque) callconv(.c) c_long {
/// Wire the session's cancel hook back to this agent so Ctrl-C aborts
/// in-flight page work. Startup and `/reset`.
-fn installCancelHook(self: *Agent) void {
+fn installSessionHooks(self: *Agent) void {
self.ts.session.cancel_hook = .{ .context = @ptrCast(self), .check = checkCancel };
+ if (self.trace != null) self.ts.session.tool_observer = .{ .context = @ptrCast(self), .onCall = traceToolCall };
}
// Compile-time constant; projected once per process to avoid rebuilding per call.
@@ -476,9 +493,25 @@ pub const SigBridge = struct {
}
};
-fn compactForRequest(ctx: *anyopaque) void {
+fn beforeTurnRequest(ctx: *anyopaque) void {
const self: *Agent = @ptrCast(@alignCast(ctx));
self.conversation.compactForRequest();
+ beforeRequest(ctx);
+}
+
+fn beforeRequest(ctx: *anyopaque) void {
+ const self: *Agent = @ptrCast(@alignCast(ctx));
+ if (self.trace) |*t| t.modelStarted();
+}
+
+fn afterResponse(ctx: *anyopaque, result: *const zenai.provider.GenerateResult) void {
+ const self: *Agent = @ptrCast(@alignCast(ctx));
+ if (self.trace) |*t| t.modelCall(self.trace_phase, result);
+}
+
+fn traceToolCall(ctx: *anyopaque, tool_name: []const u8, arguments: ?std.json.Value, result: *const browser_tools.ToolResult, ms: u64, frame: ?*lp.Frame) void {
+ const self: *Agent = @ptrCast(@alignCast(ctx));
+ self.trace.?.toolCall(tool_name, arguments, result, ms, frame);
}
fn checkCancel(ctx: *anyopaque) bool {
@@ -602,17 +635,8 @@ fn gotoStart(self: *Agent, url: [:0]const u8) bool {
/// $usage prompt=N completion=N total=N cached=N cache_creation=N
/// Fields emit 0 when the provider didn't report them.
fn printUsageSummary(self: *Agent) void {
- const u = self.total_usage;
- std.debug.print(
- "$usage prompt={d} completion={d} total={d} cached={d} cache_creation={d}\n",
- .{
- u.prompt_tokens orelse 0,
- u.completion_tokens orelse 0,
- u.total_tokens orelse 0,
- u.cached_tokens orelse 0,
- u.cache_creation_tokens orelse 0,
- },
- );
+ const t = Trace.Tokens.of(self.total_usage);
+ std.debug.print("$usage input={d} cached={d} cache_creation={d} output={d}\n", .{ t.input, t.cached, t.cache_creation, t.output });
}
fn runTurn(self: *Agent, input: TurnInput) bool {
@@ -873,7 +897,7 @@ fn handleReset(self: *Agent) void {
self.terminal.printError("reset failed: {s}", .{@errorName(err)});
return;
};
- self.installCancelHook();
+ self.installSessionHooks();
self.clearConversation();
self.terminal.printInfo("Reset conversation and browser session. Page, cookies, and storage cleared.", .{});
}
@@ -1320,6 +1344,8 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo
self.http_interrupt.reset();
self.terminal.spinner.start();
+ self.trace_phase = "save";
+ defer self.trace_phase = "turn";
var result = ModelCall.run(
self.ts.session,
provider_client,
@@ -1335,6 +1361,8 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo
.tool_choice = .none,
.effort = bumpedEffort(self.effort),
.cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel },
+ .before_request = .{ .context = @ptrCast(self), .callFn = beforeRequest },
+ .after_response = .{ .context = @ptrCast(self), .callFn = afterResponse },
},
) catch |err| {
self.terminal.spinner.cancel();
@@ -1789,7 +1817,8 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
.{ .context = @ptrCast(self), .callFn = handleToolCall },
.{
.tools = globalTools(),
- .before_request = .{ .context = @ptrCast(self), .callFn = compactForRequest },
+ .before_request = .{ .context = @ptrCast(self), .callFn = beforeTurnRequest },
+ .after_response = .{ .context = @ptrCast(self), .callFn = afterResponse },
.max_turns = 100,
.max_tool_calls = 200,
.max_tokens = 4096,
@@ -1875,6 +1904,8 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
.content = try ma.dupe(u8, synthesis_prompt),
});
+ self.trace_phase = "synthesis";
+ defer self.trace_phase = "turn";
var synth = ModelCall.run(
self.ts.session,
provider_client,
@@ -1892,6 +1923,8 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
// `.none` stays off to opt out on models that reject it.
.effort = if (self.effort == .none) .none else .low,
.cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel },
+ .before_request = .{ .context = @ptrCast(self), .callFn = beforeRequest },
+ .after_response = .{ .context = @ptrCast(self), .callFn = afterResponse },
.stream = if (input.suppress_answer) null else self.streamHook(),
},
) catch |err| {
@@ -2154,6 +2187,7 @@ test {
_ = picker;
_ = Conversation;
_ = Terminal;
+ _ = Trace;
}
test "savePrompt: save instructions followed by the rendered script skill" {
diff --git a/src/agent/Trace.zig b/src/agent/Trace.zig
new file mode 100644
index 000000000..382ed2478
--- /dev/null
+++ b/src/agent/Trace.zig
@@ -0,0 +1,177 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! `--trace`: one JSON line per model call and tool call, then a summary.
+
+const std = @import("std");
+const zenai = @import("zenai");
+const lp = @import("lightpanda");
+
+const ToolResult = lp.tools.ToolResult;
+
+const Trace = @This();
+
+file: std.Io.File,
+offset: u64 = 0,
+// Model calls report from `ModelCall`'s helper thread.
+mutex: std.Io.Mutex = .init,
+line: std.Io.Writer.Allocating,
+started_ms: u64,
+model_started_ms: u64 = 0,
+model_calls: u32 = 0,
+tool_calls: u32 = 0,
+
+pub fn create(allocator: std.mem.Allocator, path: []const u8) !Trace {
+ const file = try std.Io.Dir.cwd().createFile(lp.io, path, .{ .truncate = true });
+ return .{ .file = file, .line = .init(allocator), .started_ms = now() };
+}
+
+pub fn deinit(self: *Trace) void {
+ self.line.deinit();
+ self.file.close(lp.io);
+}
+
+pub fn run(self: *Trace, provider: []const u8, model: []const u8, task: ?[]const u8) void {
+ self.write(.{ .type = "run", .t = self.elapsed(), .provider = provider, .model = model, .task = task });
+}
+
+pub fn modelStarted(self: *Trace) void {
+ self.model_started_ms = now();
+}
+
+pub fn modelCall(self: *Trace, phase: []const u8, result: *const zenai.provider.GenerateResult) void {
+ self.model_calls += 1;
+ self.write(.{
+ .type = "model",
+ .t = self.elapsed(),
+ .step = self.model_calls,
+ .phase = phase,
+ .ms = now() - self.model_started_ms,
+ .tokens = Tokens.of(result.usage),
+ .finish = @tagName(result.finish_reason),
+ .tool_calls = if (result.tool_calls) |calls| calls.len else 0,
+ });
+}
+
+pub fn toolCall(self: *Trace, name: []const u8, arguments: ?std.json.Value, result: *const ToolResult, ms: u64, frame: ?*lp.Frame) void {
+ self.tool_calls += 1;
+ self.write(.{
+ .type = "tool",
+ .t = self.elapsed(),
+ .step = self.tool_calls,
+ .name = name,
+ .args = arguments,
+ .ms = ms,
+ .result_bytes = result.text.len,
+ .@"error" = result.is_error,
+ .navigated = result.navigated,
+ .page = if (frame) |f| lp.tools.pageState(f) else null,
+ });
+}
+
+pub fn end(self: *Trace, usage: zenai.provider.Usage) void {
+ self.write(.{
+ .type = "end",
+ .t = self.elapsed(),
+ .model_calls = self.model_calls,
+ .tool_calls = self.tool_calls,
+ .tokens = Tokens.of(usage),
+ });
+}
+
+/// Shared with the `$usage` line so both use the same names and meanings.
+pub const Tokens = struct {
+ input: i32,
+ cached: i32,
+ cache_creation: i32,
+ output: i32,
+
+ pub fn of(u: zenai.provider.Usage) Tokens {
+ return .{
+ .input = u.inputTokens(),
+ .cached = u.cached_tokens orelse 0,
+ .cache_creation = u.cache_creation_tokens orelse 0,
+ .output = u.completion_tokens orelse 0,
+ };
+ }
+};
+
+fn now() u64 {
+ return lp.datetime.milliTimestamp(.boot);
+}
+
+fn elapsed(self: *const Trace) u64 {
+ return now() - self.started_ms;
+}
+
+/// Written as each step ends, so an interrupted run still leaves a usable file.
+fn write(self: *Trace, record: anytype) void {
+ self.mutex.lockUncancelable(lp.io);
+ defer self.mutex.unlock(lp.io);
+ self.line.clearRetainingCapacity();
+ std.json.Stringify.value(record, .{ .emit_null_optional_fields = false }, &self.line.writer) catch return;
+ self.line.writer.writeByte('\n') catch return;
+ self.file.writePositionalAll(lp.io, self.line.written(), self.offset) catch return;
+ self.offset += self.line.written().len;
+}
+
+test "Trace writes one JSON line per step, arguments as given" {
+ const path = "trace-test.jsonl";
+ defer std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};
+
+ var trace: Trace = try .create(std.testing.allocator, path);
+ trace.run("openai", "gpt-test", "find courts");
+
+ var response: zenai.provider.GenerateResult = .init(std.testing.allocator);
+ defer response.deinit();
+ response.finish_reason = .tool_call;
+ response.usage = .{ .prompt_tokens = 30, .cached_tokens = 70, .completion_tokens = 5 };
+ trace.modelStarted();
+ trace.modelCall("turn", &response);
+
+ const args = try std.json.parseFromSlice(std.json.Value, std.testing.allocator,
+ \\{"selector":"#pw","value":"$LP_PASSWORD"}
+ , .{});
+ defer args.deinit();
+ trace.toolCall("fill", args.value, &.{ .text = "Filled." }, 12, null);
+ trace.end(response.usage);
+ trace.deinit();
+
+ const written = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, std.testing.allocator, .limited(4096));
+ defer std.testing.allocator.free(written);
+ var lines = std.mem.splitScalar(u8, std.mem.trimEnd(u8, written, "\n"), '\n');
+ const expected_types = [_][]const u8{ "run", "model", "tool", "end" };
+ var records: [expected_types.len]std.json.Parsed(std.json.Value) = undefined;
+ for (&records, expected_types) |*record, expected| {
+ record.* = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, lines.next().?, .{});
+ try std.testing.expectEqualStrings(expected, record.value.object.get("type").?.string);
+ }
+ defer for (&records) |*record| record.deinit();
+ try std.testing.expect(lines.next() == null);
+
+ const tokens = records[1].value.object.get("tokens").?.object;
+ try std.testing.expectEqual(100, tokens.get("input").?.integer);
+ try std.testing.expectEqual(70, tokens.get("cached").?.integer);
+
+ const tool = records[2].value.object;
+ try std.testing.expectEqualStrings("$LP_PASSWORD", tool.get("args").?.object.get("value").?.string);
+ try std.testing.expectEqual(7, tool.get("result_bytes").?.integer);
+ try std.testing.expect(tool.get("page") == null);
+
+ try std.testing.expectEqual(1, records[3].value.object.get("model_calls").?.integer);
+}
diff --git a/src/browser/Session.zig b/src/browser/Session.zig
index 2ecdf8a66..7fefb6cae 100644
--- a/src/browser/Session.zig
+++ b/src/browser/Session.zig
@@ -118,6 +118,7 @@ experimental_features: Config.ExperimentalFeatures,
/// (goto, search, waitForSelector, …) without sitting through the full
/// timeout.
cancel_hook: ?CancelHook = null,
+tool_observer: ?@import("tools.zig").Observer = null,
// Download handling configured via the `Browser.setDownloadBehavior` CDP
// method (see issue #2701). When `download_behavior` is `.allow` or
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 7ff544485..b77b2c3ba 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -857,7 +857,7 @@ pub const PageState = struct {
title: ?[]const u8,
};
-fn pageState(frame: *lp.Frame) PageState {
+pub fn pageState(frame: *lp.Frame) PageState {
return .{
.url = frame.url,
.httpStatus = frame._http_status,
@@ -901,6 +901,12 @@ pub const CallOpts = struct {
nav_note: bool = false,
};
+/// Told about every call once it returns, failures included.
+pub const Observer = struct {
+ context: *anyopaque,
+ onCall: *const fn (context: *anyopaque, tool_name: []const u8, arguments: ?std.json.Value, result: *const ToolResult, ms: u64, frame: ?*lp.Frame) void,
+};
+
// An inline screenshot is re-sent on every turn; keep it within what models
// consume. Files written to `path` are full size.
const inline_image_max_width = 1280;
@@ -913,6 +919,22 @@ pub fn call(
tool_name: []const u8,
arguments: ?std.json.Value,
opts: CallOpts,
+) ToolError!ToolResult {
+ const observer = session.tool_observer orelse return callUnobserved(arena, session, registry, tool_name, arguments, opts);
+ const started = lp.datetime.milliTimestamp(.boot);
+ const result = callUnobserved(arena, session, registry, tool_name, arguments, opts);
+ const reported: ToolResult = result catch |err| .{ .text = errorMessage(err), .is_error = true };
+ observer.onCall(observer.context, tool_name, arguments, &reported, lp.datetime.milliTimestamp(.boot) - started, session.currentFrame());
+ return result;
+}
+
+fn callUnobserved(
+ arena: std.mem.Allocator,
+ session: *lp.Session,
+ registry: *NodeRegistry,
+ tool_name: []const u8,
+ arguments: ?std.json.Value,
+ opts: CallOpts,
) ToolError!ToolResult {
// In-band so an LLM that invented a tool name (e.g. OpenAI's internal
// `multi_tool_use.parallel` wrapper) learns the name is wrong instead of
@@ -2777,10 +2799,10 @@ pub fn reverseSubstituteEnvVars(arena: std.mem.Allocator, input: []const u8) err
test "call: unknown tool name surfaces in-band" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
- // Session/registry are never touched on this branch; the name check is
- // the first thing `call` does.
- const r = try call(arena.allocator(), undefined, undefined, "multi_tool_use.parallel", null, .{});
+ const r = try call(arena.allocator(), testing.test_session, &registry, "multi_tool_use.parallel", null, .{});
try std.testing.expect(r.is_error);
try std.testing.expectEqualStrings("Unknown tool: multi_tool_use.parallel", r.text);
}
diff --git a/src/help.zon b/src/help.zon
index 0e612b22c..5fa765d8a 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -312,6 +312,12 @@
\\ One-shot mode: run a single user turn, print the final answer
\\ to stdout, and exit. Conflicts with the positional script. With
\\ --save, the answer is suppressed and a script is written instead.
+ \\ --trace <PATH>
+ \\ Write one JSON line per step to PATH: each model call (latency,
+ \\ input, cached and output tokens) and each tool call (arguments,
+ \\ latency, result size, page URL and HTTP status), then a summary.
+ \\ Arguments keep their $LP_* placeholders; results are not written.
+ \\ Overwrites PATH if it exists.
\\ --url <URL>
\\ Open this page before the first turn. With --task it saves the
\\ model a turn spent navigating; with --save the opening
diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig
index 4ddda0837..fe7ef3141 100644
--- a/src/script/Runtime.zig
+++ b/src/script/Runtime.zig
@@ -74,6 +74,7 @@ const PendingGoto = struct {
receiver: v8.Global,
/// `run_timer` reading (ms) past which the navigation is abandoned.
deadline_ms: u64,
+ started_ms: u64,
until: lp.Config.WaitUntil,
fn reset(self: *PendingGoto) void {
@@ -501,6 +502,7 @@ fn invokeGoto(
// startGoto is browser-side work; run it under the browser's isolate.
// Settle the resolver only after the block: a `return` inside it runs
// script-isolate work before the deferred exit.
+ const started_ms = lp.datetime.milliTimestamp(.boot);
const maybe_started: ?browser_tools.StartedGoto = blk: {
self.session.browser.env.isolate.enter();
defer self.session.browser.env.isolate.exit();
@@ -514,6 +516,7 @@ fn invokeGoto(
.resolver = undefined,
.receiver = undefined,
.deadline_ms = @as(u64, @intCast(self.run_timer.untilNow(lp.io, .boot).toMilliseconds())) + started.timeout_ms,
+ .started_ms = started_ms,
.until = started.until,
};
v8.v8__Global__New(self.env.isolate.handle, resolver, &pending.resolver);
@@ -623,6 +626,11 @@ fn settlePending(self: *Runtime, context: *const v8.Context, pending: *PendingGo
.failed => self.rejectResolver(context, resolver, "navigation failed"),
.timed_out => self.rejectResolver(context, resolver, "navigation timed out"),
}
+ // Async gotos settle here, outside `browser_tools.call`.
+ if (self.session.tool_observer) |observer| {
+ const reported: browser_tools.ToolResult = .{ .text = @tagName(outcome), .is_error = outcome != .loaded, .navigated = true };
+ observer.onCall(observer.context, "goto", null, &reported, lp.datetime.milliTimestamp(.boot) - pending.started_ms, self.session.findFrameByFrameId(pending.frame_id));
+ }
pending.reset();
}