Lightpanda: the headless browser designed for AI and automation
agent: resolve the replay selector in the tool layer
Adrià Arrufat4da152e9aaec
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex 5207605ec..ae870f162 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -167,6 +167,11 @@ cancel_requested: std.atomic.Value(bool) = .init(false),/// mid-request instead of blocking until the model's full response arrives.http_interrupt: zenai.http.Interrupt = .{},synthetic_tool_call_id: u32 = 0,+/// Per-turn CSS selector for each tool call the model made, in call order, so+/// `--save` can record a call that addressed its element by `backendNodeId`.+/// Only filled while `capturing_for_save`.+save_selectors: std.ArrayListUnmanaged(?[]const u8) = .empty,+capturing_for_save: bool = false,/// Aggregate Anthropic/OpenAI/Gemini token usage across every model call./// Printed as a structured `$usage ...` line on stderr at the end of `--task`/// (one-shot) mode so wrappers can capture per-task cost.@@ -355,6 +360,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agentpub fn deinit(self: *Agent) void {self.terminal.uninstallLogSink();self.save_buffer.deinit();+ self.save_selectors.deinit(self.allocator);if (self.save_path) |p| self.allocator.free(p);self.terminal.deinit();self.conversation.deinit();@@ -1328,6 +1334,21 @@ fn logSaveBufferError(self: *Agent, err: anyerror) void {self.terminal.printError("save buffer disabled: {s}", .{@errorName(err)});}+/// Swap a call's ephemeral `backendNodeId` for the selector the tool layer+/// resolved, so the call can be replayed. Returns `args` untouched when there+/// is nothing to swap.+fn withSelector(arena: std.mem.Allocator, args: ?std.json.Value, selector: ?[]const u8) ?std.json.Value {+ const sel = selector orelse return args;+ const original = args orelse return args;+ if (original != .object) return args;+ if (!original.object.contains("backendNodeId")) return args;++ var rewritten = original.object.clone(arena) catch return args;+ _ = rewritten.swapRemove("backendNodeId");+ rewritten.put(arena, "selector", .{ .string = sel }) catch return args;+ return .{ .object = rewritten };+}+fn recordSaveCommand(self: *Agent, cmd: Command) void {self.save_buffer.record(cmd) catch |err| self.logSaveBufferError(err);}@@ -1671,6 +1692,10 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {const provider_client = self.ai_client orelse return error.NoAiClient;self.refreshAuthIfNeeded();+ self.capturing_for_save = input.capture_for_save;+ defer self.capturing_for_save = false;+ self.save_selectors.clearRetainingCapacity();+self.terminal.spinner.start();var result = provider_client.runTools(self.model,@@ -1731,11 +1756,15 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {const args = browser_tools.normalizeArgKeys(ca, tool, tc.arguments) catch tc.arguments;// Fall back to the navigation a read tool performed, so a// markdown/tree-driven turn isn't lost from `/save`.- const cmd = Command.fromToolCall(tool, args);+ // A call that named its element by id is unreplayable as-is; the+ // tool layer resolved a selector for it while the node still+ // existed.+ const replayable = withSelector(ca, args, if (i < self.save_selectors.items.len) self.save_selectors.items[i] else null);+ const cmd = Command.fromToolCall(tool, replayable);const to_record = if (cmd.isRecorded())cmdelse- navigationGoto(ca, tool, args) orelse continue;+ navigationGoto(ca, tool, replayable) orelse continue;if (!recorded_any) {if (input.record_comment) |c| self.recordSaveComment(c);recorded_any = true;@@ -1888,10 +1917,17 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []coself.terminal.spinner.setTool(tool_name, args_str);defer self.terminal.spinner.setThinking();- const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{+ var selector: ?[]const u8 = null;+ const outcome = self.toolOutcome(allocator, tool_name, arguments, &selector) catch |err| zenai.provider.Client.ToolHandler.Result{.content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed",.is_error = true,};+ if (self.capturing_for_save) {+ // One entry per call, errors included, so the index lines up with+ // `RunToolsResult.tool_calls_made`.+ const kept = if (selector) |sel| self.allocator.dupe(u8, sel) catch null else null;+ self.save_selectors.append(self.allocator, kept) catch {};+ }self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);@@ -1899,8 +1935,12 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co}/// The text plus the rendered PNG, for backends that can show the model an image.-fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {- const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{ .inline_image = true });+fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value, selector: *?[]const u8) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {+ const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{+ .inline_image = true,+ .record = self.capturing_for_save,+ });+ selector.* = result.selector;const content = capToolOutput(allocator, tool_name, result.text);return .{.content = content,diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 9b8d31e52..9a7bfcf88 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -24,6 +24,8 @@ const NodeRegistry = @import("../NodeRegistry.zig");const DOMNode = @import("webapi/Node.zig");const Selector = @import("webapi/selector/Selector.zig");+const SelectorPath = @import("SelectorPath.zig");+const Element = @import("webapi/Element.zig");const log = lp.log;const tavily = zenai.search.tavily;@@ -823,6 +825,10 @@ pub const ToolResult = struct {is_error: bool = false,/// Only set when the caller passed `CallOpts.inline_image`.image: ?lp.screenshot.Prepared = null,+ /// Only set when the caller passed `CallOpts.record` and the call named its+ /// element by `backendNodeId`. Resolved before the action runs, because a+ /// navigation takes the node with it.+ selector: ?[]const u8 = null,};const GotoParams = struct {@@ -854,6 +860,11 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTargpub const CallOpts = struct {/// The caller can hand an image to a model.inline_image: bool = false,+ /// The caller is recording for `--save`/`/save`. A call that addresses its+ /// element by `backendNodeId` gets `ToolResult.selector` filled in, since+ /// a registry id means nothing in a later session and the node may be gone+ /// by the time the caller wants to record it.+ record: bool = false,};// An inline screenshot is re-sent on every turn; keep it within what models@@ -886,13 +897,39 @@ pub fn call(};const substituted = try substituteStringArgs(arena, tool, normalized);- return dispatch(arena, session, registry, tool, substituted, opts) catch |err| {+ // Before dispatch: after a navigation the node is gone.+ const selector = if (opts.record) selectorForArgs(arena, session, registry, substituted) else null;++ var result = dispatch(arena, session, registry, tool, substituted, opts) catch |err| {if (err == error.NavigationFailed) {if (formatNavigationError(arena, session)) |text|return .{ .text = text, .is_error = true };}return err;};+ result.selector = selector;+ return result;+}++/// The CSS selector for a call's `backendNodeId`, so the call can be recorded+/// in a form that still resolves in a later session. Null when the arguments+/// name no node, already carry a selector, or the node cannot be named.+fn selectorForArgs(+ arena: std.mem.Allocator,+ session: *lp.Session,+ registry: *NodeRegistry,+ arguments: ?std.json.Value,+) ?[]const u8 {+ const args = arguments orelse return null;+ if (args != .object) return null;+ if (args.object.contains("selector")) return null;+ const id = args.object.get("backendNodeId") orelse return null;+ if (id != .integer) return null;++ const node = registry.lookup_by_id.get(std.math.cast(NodeRegistry.Id, id.integer) orelse return null) orelse return null;+ const el = node.dom.is(Element) orelse return null;+ const frame = session.currentFrame() orelse return null;+ return SelectorPath.init(arena, frame).build(el) catch null;}fn dispatch(