Skip to content

Lightpanda: the headless browser designed for AI and automation

agent: resolve the replay selector in the tool layer

Adrià Arrufat4da152e9aaec
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index 5207605ec..ae870f162 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -167,6 +167,11 @@ cancel_requested: std.atomic.Value(bool) = .init(false),
/// mid-request instead of blocking until the model's full response arrives.
http_interrupt: zenai.http.Interrupt = .{},
synthetic_tool_call_id: u32 = 0,
+/// Per-turn CSS selector for each tool call the model made, in call order, so
+/// `--save` can record a call that addressed its element by `backendNodeId`.
+/// Only filled while `capturing_for_save`.
+save_selectors: std.ArrayListUnmanaged(?[]const u8) = .empty,
+capturing_for_save: bool = false,
/// Aggregate Anthropic/OpenAI/Gemini token usage across every model call.
/// Printed as a structured `$usage ...` line on stderr at the end of `--task`
/// (one-shot) mode so wrappers can capture per-task cost.
@@ -355,6 +360,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent
pub fn deinit(self: *Agent) void {
self.terminal.uninstallLogSink();
self.save_buffer.deinit();
+ self.save_selectors.deinit(self.allocator);
if (self.save_path) |p| self.allocator.free(p);
self.terminal.deinit();
self.conversation.deinit();
@@ -1328,6 +1334,21 @@ fn logSaveBufferError(self: *Agent, err: anyerror) void {
self.terminal.printError("save buffer disabled: {s}", .{@errorName(err)});
}
+/// Swap a call's ephemeral `backendNodeId` for the selector the tool layer
+/// resolved, so the call can be replayed. Returns `args` untouched when there
+/// is nothing to swap.
+fn withSelector(arena: std.mem.Allocator, args: ?std.json.Value, selector: ?[]const u8) ?std.json.Value {
+ const sel = selector orelse return args;
+ const original = args orelse return args;
+ if (original != .object) return args;
+ if (!original.object.contains("backendNodeId")) return args;
+
+ var rewritten = original.object.clone(arena) catch return args;
+ _ = rewritten.swapRemove("backendNodeId");
+ rewritten.put(arena, "selector", .{ .string = sel }) catch return args;
+ return .{ .object = rewritten };
+}
+
fn recordSaveCommand(self: *Agent, cmd: Command) void {
self.save_buffer.record(cmd) catch |err| self.logSaveBufferError(err);
}
@@ -1671,6 +1692,10 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
const provider_client = self.ai_client orelse return error.NoAiClient;
self.refreshAuthIfNeeded();
+ self.capturing_for_save = input.capture_for_save;
+ defer self.capturing_for_save = false;
+ self.save_selectors.clearRetainingCapacity();
+
self.terminal.spinner.start();
var result = provider_client.runTools(
self.model,
@@ -1731,11 +1756,15 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
const args = browser_tools.normalizeArgKeys(ca, tool, tc.arguments) catch tc.arguments;
// Fall back to the navigation a read tool performed, so a
// markdown/tree-driven turn isn't lost from `/save`.
- const cmd = Command.fromToolCall(tool, args);
+ // A call that named its element by id is unreplayable as-is; the
+ // tool layer resolved a selector for it while the node still
+ // existed.
+ const replayable = withSelector(ca, args, if (i < self.save_selectors.items.len) self.save_selectors.items[i] else null);
+ const cmd = Command.fromToolCall(tool, replayable);
const to_record = if (cmd.isRecorded())
cmd
else
- navigationGoto(ca, tool, args) orelse continue;
+ navigationGoto(ca, tool, replayable) orelse continue;
if (!recorded_any) {
if (input.record_comment) |c| self.recordSaveComment(c);
recorded_any = true;
@@ -1888,10 +1917,17 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
self.terminal.spinner.setTool(tool_name, args_str);
defer self.terminal.spinner.setThinking();
- const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{
+ var selector: ?[]const u8 = null;
+ const outcome = self.toolOutcome(allocator, tool_name, arguments, &selector) catch |err| zenai.provider.Client.ToolHandler.Result{
.content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed",
.is_error = true,
};
+ if (self.capturing_for_save) {
+ // One entry per call, errors included, so the index lines up with
+ // `RunToolsResult.tool_calls_made`.
+ const kept = if (selector) |sel| self.allocator.dupe(u8, sel) catch null else null;
+ self.save_selectors.append(self.allocator, kept) catch {};
+ }
self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);
if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);
@@ -1899,8 +1935,12 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
}
/// The text plus the rendered PNG, for backends that can show the model an image.
-fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
- const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{ .inline_image = true });
+fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value, selector: *?[]const u8) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
+ const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{
+ .inline_image = true,
+ .record = self.capturing_for_save,
+ });
+ selector.* = result.selector;
const content = capToolOutput(allocator, tool_name, result.text);
return .{
.content = content,
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 9b8d31e52..9a7bfcf88 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -24,6 +24,8 @@ const NodeRegistry = @import("../NodeRegistry.zig");
const DOMNode = @import("webapi/Node.zig");
const Selector = @import("webapi/selector/Selector.zig");
+const SelectorPath = @import("SelectorPath.zig");
+const Element = @import("webapi/Element.zig");
const log = lp.log;
const tavily = zenai.search.tavily;
@@ -823,6 +825,10 @@ pub const ToolResult = struct {
is_error: bool = false,
/// Only set when the caller passed `CallOpts.inline_image`.
image: ?lp.screenshot.Prepared = null,
+ /// Only set when the caller passed `CallOpts.record` and the call named its
+ /// element by `backendNodeId`. Resolved before the action runs, because a
+ /// navigation takes the node with it.
+ selector: ?[]const u8 = null,
};
const GotoParams = struct {
@@ -854,6 +860,11 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg
pub const CallOpts = struct {
/// The caller can hand an image to a model.
inline_image: bool = false,
+ /// The caller is recording for `--save`/`/save`. A call that addresses its
+ /// element by `backendNodeId` gets `ToolResult.selector` filled in, since
+ /// a registry id means nothing in a later session and the node may be gone
+ /// by the time the caller wants to record it.
+ record: bool = false,
};
// An inline screenshot is re-sent on every turn; keep it within what models
@@ -886,13 +897,39 @@ pub fn call(
};
const substituted = try substituteStringArgs(arena, tool, normalized);
- return dispatch(arena, session, registry, tool, substituted, opts) catch |err| {
+ // Before dispatch: after a navigation the node is gone.
+ const selector = if (opts.record) selectorForArgs(arena, session, registry, substituted) else null;
+
+ var result = dispatch(arena, session, registry, tool, substituted, opts) catch |err| {
if (err == error.NavigationFailed) {
if (formatNavigationError(arena, session)) |text|
return .{ .text = text, .is_error = true };
}
return err;
};
+ result.selector = selector;
+ return result;
+}
+
+/// The CSS selector for a call's `backendNodeId`, so the call can be recorded
+/// in a form that still resolves in a later session. Null when the arguments
+/// name no node, already carry a selector, or the node cannot be named.
+fn selectorForArgs(
+ arena: std.mem.Allocator,
+ session: *lp.Session,
+ registry: *NodeRegistry,
+ arguments: ?std.json.Value,
+) ?[]const u8 {
+ const args = arguments orelse return null;
+ if (args != .object) return null;
+ if (args.object.contains("selector")) return null;
+ const id = args.object.get("backendNodeId") orelse return null;
+ if (id != .integer) return null;
+
+ const node = registry.lookup_by_id.get(std.math.cast(NodeRegistry.Id, id.integer) orelse return null) orelse return null;
+ const el = node.dom.is(Element) orelse return null;
+ const frame = session.currentFrame() orelse return null;
+ return SelectorPath.init(arena, frame).build(el) catch null;
}
fn dispatch(