Skip to content

Lightpanda: the headless browser designed for AI and automation

Merge branch 'main' into tool-telemetry

Adrià Arrufat9c1436a20c1e
diff --cc src/browser/tools.zig
index 13fe2021f,09c2c946e..edb65ba97
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@@ -935,9 -899,14 +935,15 @@@ pub const CallOpts = struct
record: bool = false,
/// Scripts parse the text, so only model-facing callers set it.
nav_note: bool = false,
+ source: TelemetryTool.Source,
};
+ /// Told about every call once it returns, failures included.
+ pub const Observer = struct {
+ context: *anyopaque,
+ onCall: *const fn (context: *anyopaque, tool_name: []const u8, arguments: ?std.json.Value, result: *const ToolResult, ms: u64, frame: ?*lp.Frame) void,
+ };
+
// An inline screenshot is re-sent on every turn; keep it within what models
// consume. Files written to `path` are full size.
const inline_image_max_width = 1280;
@@@ -951,11 -920,11 +957,15 @@@ pub fn call
arguments: ?std.json.Value,
opts: CallOpts,
) ToolError!ToolResult {
- const observer = session.tool_observer orelse return callUnobserved(arena, session, registry, tool_name, arguments, opts);
- const started = lp.datetime.milliTimestamp(.boot);
- const result = callUnobserved(arena, session, registry, tool_name, arguments, opts);
- const reported: ToolResult = result catch |err| .{ .text = errorMessage(err), .is_error = true };
- observer.onCall(observer.context, tool_name, arguments, &reported, lp.datetime.milliTimestamp(.boot) - started, session.currentFrame());
+ const start = lp.datetime.milliTimestamp(.awake);
+ const maybe_tool = std.meta.stringToEnum(Tool, tool_name);
+ const result = callInner(arena, session, registry, maybe_tool, tool_name, arguments, opts);
+ const id: u8 = if (maybe_tool) |t| t.telemetryId() else 0;
+ session.browser.app.telemetry.recordTool(id, opts.source, telemetryOutcome(result), start);
++ if (session.tool_observer) |observer| {
++ const reported: ToolResult = result catch |err| .{ .text = errorMessage(err), .is_error = true };
++ observer.onCall(observer.context, tool_name, arguments, &reported, lp.datetime.milliTimestamp(.awake) -| start, session.currentFrame());
++ }
return result;
}
@@@ -2849,10 -2799,10 +2859,10 @@@ pub fn reverseSubstituteEnvVars(arena:
test "call: unknown tool name surfaces in-band" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
- // The registry is never touched on this branch; the name check is the
- // first thing `call` does.
- const r = try call(arena.allocator(), testing.test_session, undefined, "multi_tool_use.parallel", null, .{ .source = .user });
- const r = try call(arena.allocator(), testing.test_session, &registry, "multi_tool_use.parallel", null, .{});
++ const r = try call(arena.allocator(), testing.test_session, &registry, "multi_tool_use.parallel", null, .{ .source = .user });
try std.testing.expect(r.is_error);
try std.testing.expectEqualStrings("Unknown tool: multi_tool_use.parallel", r.text);
}
@@@ -2881,6 -2831,28 +2891,28 @@@ test "tree and nodeDetails read the nod
try std.testing.expect(std.mem.find(u8, details.text, "child-label") != null);
}
+ test "tree and nodeDetails list iframes with their URL" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ var page = try testing.pageTest("cdp/semantic_tree_iframe.html", .{});
+ defer page.close();
+
+ const aa = testing.arena_allocator;
+ _ = try call(aa, page.session, &registry, "evaluate", try std.json.parseFromSliceLeaky(std.json.Value, aa,
+ \\{"script":"document.querySelector('iframe').contentWindow.history.replaceState(null, '', '?moved'); const f = document.createElement('iframe'); f.setAttribute('src', 'semantic_tree_iframe_child.html?scripted'); document.body.appendChild(f)"}
- , .{}), .{});
++ , .{}), .{ .source = .user });
+
- const tree = try call(aa, page.session, &registry, "tree", null, .{});
++ const tree = try call(aa, page.session, &registry, "tree", null, .{ .source = .user });
+ try std.testing.expect(std.mem.find(u8, tree.text, "Iframe value='http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?moved'") != null);
+ try std.testing.expect(std.mem.find(u8, tree.text, "Iframe value='http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?scripted'") != null);
+
+ const iframe = (try page.frame().?.document.querySelector(.wrap("iframe"), page.frame().?)).?.asNode();
+ const details_args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print("{{\"backendNodeId\":{d}}}", .{(try registry.register(iframe)).id}), .{});
- const details = try call(aa, page.session, &registry, "nodeDetails", details_args, .{});
++ const details = try call(aa, page.session, &registry, "nodeDetails", details_args, .{ .source = .user });
+ try std.testing.expect(std.mem.find(u8, details.text, "\"value\":\"http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?moved\"") != null);
+ }
+
test "goto: a navigation stuck waiting for a connection is an error" {
var registry: NodeRegistry = .init(std.testing.allocator);
defer registry.deinit();
diff --cc src/script/Runtime.zig
index 54e5a6290,fe7ef3141..f8d86ec94
--- a/src/script/Runtime.zig
+++ b/src/script/Runtime.zig
@@@ -637,7 -626,11 +637,12 @@@ fn settlePending(self: *Runtime, contex
.failed => self.rejectResolver(context, resolver, "navigation failed"),
.timed_out => self.rejectResolver(context, resolver, "navigation timed out"),
}
+ self.recordGoto(result, pending.started_ms);
+ // Async gotos settle here, outside `browser_tools.call`.
+ if (self.session.tool_observer) |observer| {
+ const reported: browser_tools.ToolResult = .{ .text = @tagName(outcome), .is_error = outcome != .loaded, .navigated = true };
- observer.onCall(observer.context, "goto", null, &reported, lp.datetime.milliTimestamp(.boot) - pending.started_ms, self.session.findFrameByFrameId(pending.frame_id));
++ observer.onCall(observer.context, "goto", null, &reported, lp.datetime.milliTimestamp(.awake) -| pending.started_ms, self.session.findFrameByFrameId(pending.frame_id));
+ }
pending.reset();
}