Lightpanda: the headless browser designed for AI and automation
Merge branch 'main' into tool-telemetry
Adrià Arrufat9c1436a20c1e
diff --cc src/browser/tools.zigindex 13fe2021f,09c2c946e..edb65ba97--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@@ -935,9 -899,14 +935,15 @@@ pub const CallOpts = structrecord: bool = false,/// Scripts parse the text, so only model-facing callers set it.nav_note: bool = false,+ source: TelemetryTool.Source,};+ /// Told about every call once it returns, failures included.+ pub const Observer = struct {+ context: *anyopaque,+ onCall: *const fn (context: *anyopaque, tool_name: []const u8, arguments: ?std.json.Value, result: *const ToolResult, ms: u64, frame: ?*lp.Frame) void,+ };+// An inline screenshot is re-sent on every turn; keep it within what models// consume. Files written to `path` are full size.const inline_image_max_width = 1280;@@@ -951,11 -920,11 +957,15 @@@ pub fn callarguments: ?std.json.Value,opts: CallOpts,) ToolError!ToolResult {- const observer = session.tool_observer orelse return callUnobserved(arena, session, registry, tool_name, arguments, opts);- const started = lp.datetime.milliTimestamp(.boot);- const result = callUnobserved(arena, session, registry, tool_name, arguments, opts);- const reported: ToolResult = result catch |err| .{ .text = errorMessage(err), .is_error = true };- observer.onCall(observer.context, tool_name, arguments, &reported, lp.datetime.milliTimestamp(.boot) - started, session.currentFrame());+ const start = lp.datetime.milliTimestamp(.awake);+ const maybe_tool = std.meta.stringToEnum(Tool, tool_name);+ const result = callInner(arena, session, registry, maybe_tool, tool_name, arguments, opts);+ const id: u8 = if (maybe_tool) |t| t.telemetryId() else 0;+ session.browser.app.telemetry.recordTool(id, opts.source, telemetryOutcome(result), start);++ if (session.tool_observer) |observer| {++ const reported: ToolResult = result catch |err| .{ .text = errorMessage(err), .is_error = true };++ observer.onCall(observer.context, tool_name, arguments, &reported, lp.datetime.milliTimestamp(.awake) -| start, session.currentFrame());++ }return result;}@@@ -2849,10 -2799,10 +2859,10 @@@ pub fn reverseSubstituteEnvVars(arena:test "call: unknown tool name surfaces in-band" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();- // The registry is never touched on this branch; the name check is the- // first thing `call` does.- const r = try call(arena.allocator(), testing.test_session, undefined, "multi_tool_use.parallel", null, .{ .source = .user });- const r = try call(arena.allocator(), testing.test_session, ®istry, "multi_tool_use.parallel", null, .{});++ const r = try call(arena.allocator(), testing.test_session, ®istry, "multi_tool_use.parallel", null, .{ .source = .user });try std.testing.expect(r.is_error);try std.testing.expectEqualStrings("Unknown tool: multi_tool_use.parallel", r.text);}@@@ -2881,6 -2831,28 +2891,28 @@@ test "tree and nodeDetails read the nodtry std.testing.expect(std.mem.find(u8, details.text, "child-label") != null);}+ test "tree and nodeDetails list iframes with their URL" {+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();++ var page = try testing.pageTest("cdp/semantic_tree_iframe.html", .{});+ defer page.close();++ const aa = testing.arena_allocator;+ _ = try call(aa, page.session, ®istry, "evaluate", try std.json.parseFromSliceLeaky(std.json.Value, aa,+ \\{"script":"document.querySelector('iframe').contentWindow.history.replaceState(null, '', '?moved'); const f = document.createElement('iframe'); f.setAttribute('src', 'semantic_tree_iframe_child.html?scripted'); document.body.appendChild(f)"}- , .{}), .{});++ , .{}), .{ .source = .user });+- const tree = try call(aa, page.session, ®istry, "tree", null, .{});++ const tree = try call(aa, page.session, ®istry, "tree", null, .{ .source = .user });+ try std.testing.expect(std.mem.find(u8, tree.text, "Iframe value='http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?moved'") != null);+ try std.testing.expect(std.mem.find(u8, tree.text, "Iframe value='http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?scripted'") != null);++ const iframe = (try page.frame().?.document.querySelector(.wrap("iframe"), page.frame().?)).?.asNode();+ const details_args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print("{{\"backendNodeId\":{d}}}", .{(try registry.register(iframe)).id}), .{});- const details = try call(aa, page.session, ®istry, "nodeDetails", details_args, .{});++ const details = try call(aa, page.session, ®istry, "nodeDetails", details_args, .{ .source = .user });+ try std.testing.expect(std.mem.find(u8, details.text, "\"value\":\"http://127.0.0.1:9582/src/browser/tests/cdp/semantic_tree_iframe_child.html?moved\"") != null);+ }+test "goto: a navigation stuck waiting for a connection is an error" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();diff --cc src/script/Runtime.zigindex 54e5a6290,fe7ef3141..f8d86ec94--- a/src/script/Runtime.zig+++ b/src/script/Runtime.zig@@@ -637,7 -626,11 +637,12 @@@ fn settlePending(self: *Runtime, contex.failed => self.rejectResolver(context, resolver, "navigation failed"),.timed_out => self.rejectResolver(context, resolver, "navigation timed out"),}+ self.recordGoto(result, pending.started_ms);+ // Async gotos settle here, outside `browser_tools.call`.+ if (self.session.tool_observer) |observer| {+ const reported: browser_tools.ToolResult = .{ .text = @tagName(outcome), .is_error = outcome != .loaded, .navigated = true };- observer.onCall(observer.context, "goto", null, &reported, lp.datetime.milliTimestamp(.boot) - pending.started_ms, self.session.findFrameByFrameId(pending.frame_id));++ observer.onCall(observer.context, "goto", null, &reported, lp.datetime.milliTimestamp(.awake) -| pending.started_ms, self.session.findFrameByFrameId(pending.frame_id));+ }pending.reset();}