Skip to content

Lightpanda: the headless browser designed for AI and automation

simplify: recorder rewind, Tool.waitsForReadiness, single goto default

Adrià Arrufatb85fb7c6beab
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index 7cce38161..d77433a83 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -1155,11 +1155,7 @@ fn handleSave(self: *Agent, arena: std.mem.Allocator, rest: []const u8) void {
null;
defer if (new_save_path) |p| self.allocator.free(p);
- const recorded = self.save_buffer.bytes() catch |err| {
- self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) });
- return;
- };
- save.writeContentFile(path, recorded, mode) catch |err| {
+ save.writeContentFile(path, self.save_buffer.bytes(), mode) catch |err| {
self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) });
return;
};
@@ -1229,7 +1225,7 @@ fn bumpedEffort(effort: Config.Effort) Config.Effort {
fn synthesizeSave(self: *Agent, arena: std.mem.Allocator, filename: ?[]const u8, prompt: ?[]const u8) void {
// With nothing recorded and no prompt, a re-synthesis has nothing to act
// on — it would just re-roll the previous output.
- if (self.save_buffer.isEmpty() and prompt == null) {
+ if (self.save_buffer.bytes().len == 0 and prompt == null) {
if (self.save_path) |saved| {
self.terminal.printWarning("nothing ran since the last save; run more commands or give a prompt to revise {s}, e.g. /save <what to change>", .{saved});
} else {
@@ -1356,7 +1352,7 @@ fn buildSaveSynthesisMessage(self: *Agent, arena: std.mem.Allocator, path: []con
try w.print("\nThe previously saved script in {s}, the base you are revising:\n", .{path});
try w.writeAll(script);
}
- const recorded = try self.save_buffer.bytes();
+ const recorded = self.save_buffer.bytes();
if (recorded.len > 0) {
const since: []const u8 = if (previous_script != null) " since the last save" else " this session";
try w.print("\nCommands and JS that actually ran{s}:\n", .{since});
diff --git a/src/browser/actions.zig b/src/browser/actions.zig
index d24abf135..a976f31c8 100644
--- a/src/browser/actions.zig
+++ b/src/browser/actions.zig
@@ -276,9 +276,9 @@ fn remainingMs(timeout_ms: u32, timer: std.Io.Timestamp) u32 {
pub fn waitForSelector(selector: [:0]const u8, timeout_ms: u32, frame_id: u32, session: *Session) !*DOMNode {
const timer: std.Io.Timestamp = .now(lp.io, .boot);
var runner = session.runner(.{});
- // Only parsing has to be done before polling — gating on `.load` would
- // re-wait for the late-script tail a `waitUntil: domcontentloaded`
- // navigation deliberately skipped (#3138).
+ // Polling needs a parsed document, nothing more. Gating on `.load` would
+ // re-wait the late-script tail a `waitUntil: domcontentloaded` navigation
+ // deliberately skipped (#3138).
try runner.waitForFrame(frame_id, timeout_ms, .{ .until = .domcontentloaded });
const el = try runner.waitForSelector(frame_id, selector, remainingMs(timeout_ms, timer));
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index f1871e400..c68454387 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -276,6 +276,17 @@ pub const Tool = enum {
};
}
+ /// Waits for page readiness on its own, making a preceding `goto`'s full
+ /// `load` wait redundant — the recorder downgrades such gotos to
+ /// `domcontentloaded` (#3138). Exhaustive like the sibling predicates so a
+ /// new wait tool makes an explicit choice here.
+ pub fn waitsForReadiness(self: Tool) bool {
+ return switch (self) {
+ .waitForSelector, .waitForScript, .waitForState => true,
+ .goto, .evaluate, .extract, .click, .fill, .scroll, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
+ };
+ }
+
/// A read tool that navigates when handed a `url`. The read isn't recorded,
/// but the navigation is, so the recorder captures it as a `goto`. Excludes
/// `evaluate` (carries its own `url`), `search` (derived engine URL), and
@@ -330,7 +341,9 @@ pub const Tool = enum {
\\ "properties": {
\\ "url": { "type": "string", "description": "The URL to navigate to, must be a valid URL." },
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." },
- \\ "waitUntil": { "type": "string", "enum": ["load", "domcontentloaded", "networkalmostidle", "networkidle"], "description": "Event that completes the navigation. Defaults to 'load'. Prefer 'domcontentloaded' followed by waitForSelector on pages whose late scripts (ads) hold 'load' back." }
+ \\ "waitUntil": { "type": "string", "enum":
+ ++ lp.Config.tagJsonArray(lp.Config.WaitUntil) ++
+ \\, "description": "Event that completes the navigation. Defaults to 'load'. Prefer 'domcontentloaded' followed by waitForSelector on pages whose late scripts (ads) hold 'load' back." }
\\ },
\\ "required": ["url"]
\\}
@@ -759,7 +772,7 @@ pub const ToolResult = struct {
pub const GotoParams = struct {
url: [:0]const u8,
timeout: ?u32 = null,
- waitUntil: ?lp.Config.WaitUntil = null,
+ waitUntil: lp.Config.WaitUntil = default_nav_wait,
};
pub const UrlParams = struct {
@@ -938,7 +951,7 @@ const schema_walker_suffix = ")";
fn execGoto(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
const args = try parseArgs(GotoParams, arena, arguments);
- return switch (try performGoto(session, registry, args.url, args.timeout, args.waitUntil)) {
+ return switch (try performGoto(session, registry, args.url, .{ .timeout = args.timeout, .wait_until = args.waitUntil })) {
.completed => "Navigated successfully.",
.timeout => "Navigation started but the page did not finish loading before the timeout.",
};
@@ -990,7 +1003,7 @@ fn execSearch(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode
.{encoded},
0,
) catch return ToolError.OutOfMemory;
- _ = try performGoto(session, registry, ddg_url, args.timeout, null);
+ _ = try performGoto(session, registry, ddg_url, .{ .timeout = args.timeout });
const ddg_frame = try requireFrame(session);
return .{ .text = try renderFrameMarkdown(arena, ddg_frame) };
}
@@ -1923,7 +1936,7 @@ fn ensurePage(session: *lp.Session, registry: *CDPNode.Registry, url: ?[:0]const
if (session.currentFrame()) |frame| {
if (std.mem.eql(u8, frame.url, u)) return frame;
}
- _ = try performGoto(session, registry, u, timeout, null);
+ _ = try performGoto(session, registry, u, .{ .timeout = timeout });
}
return session.currentFrame() orelse ToolError.FrameNotLoaded;
}
@@ -1977,11 +1990,16 @@ pub fn startGoto(
return .{
.frame_id = page.frame_id,
.timeout_ms = args.timeout orelse default_nav_timeout_ms,
- .until = args.waitUntil orelse default_nav_wait,
+ .until = args.waitUntil,
};
}
-fn performGoto(session: *lp.Session, registry: *CDPNode.Registry, url: [:0]const u8, timeout: ?u32, wait_until: ?lp.Config.WaitUntil) ToolError!lp.Session.Runner.WaitResult {
+const PerformGotoOpts = struct {
+ timeout: ?u32 = null,
+ wait_until: lp.Config.WaitUntil = default_nav_wait,
+};
+
+fn performGoto(session: *lp.Session, registry: *CDPNode.Registry, url: [:0]const u8, opts: PerformGotoOpts) ToolError!lp.Session.Runner.WaitResult {
if (session.primaryPage()) |old_page| {
registry.reset();
old_page.close();
@@ -1989,9 +2007,9 @@ fn performGoto(session: *lp.Session, registry: *CDPNode.Registry, url: [:0]const
const page = try openPage(session, url);
var runner = session.runner(.{});
- const condition = lp.Session.Runner.WaitCondition{ .frame_id = page.frame_id, .until = wait_until orelse default_nav_wait };
+ const condition = lp.Session.Runner.WaitCondition{ .frame_id = page.frame_id, .until = opts.wait_until };
var conditions = [_]lp.Session.Runner.WaitCondition{condition};
- const result = runner.waitResult(timeout orelse default_nav_timeout_ms, &conditions) catch |err| {
+ const result = runner.waitResult(opts.timeout orelse default_nav_timeout_ms, &conditions) catch |err| {
return if (err == error.Cancelled) ToolError.Cancelled else ToolError.NavigationFailed;
};
diff --git a/src/script/Recorder.zig b/src/script/Recorder.zig
index a13b500a5..b7ba7223a 100644
--- a/src/script/Recorder.zig
+++ b/src/script/Recorder.zig
@@ -38,15 +38,18 @@ content: std.Io.Writer.Allocating,
buf: std.Io.Writer.Allocating,
/// Reset per write — backs short-lived scrub allocations.
arena: std.heap.ArenaAllocator,
-/// A recorded `goto` held back one step. If the next recorded call waits on
-/// its own (waitForSelector/waitForState), the `domcontentloaded` variant is
-/// emitted instead of the default load wait — the follow-up wait covers
-/// readiness, and dcl skips the ad chains that hold `load` back (#3138).
+/// The just-emitted `goto` line stays rewritable for exactly one step: if the
+/// next recorded call waits for readiness on its own (`Tool.waitsForReadiness`),
+/// the line is rewound and re-emitted with `waitUntil: "domcontentloaded"` —
+/// the follow-up wait covers readiness, and dcl skips the ad chains that hold
+/// `load` back (#3138). Anything else recorded in between closes the window.
pending_goto: ?PendingGoto,
const PendingGoto = struct {
- plain: []u8,
- /// Null when the call carried an explicit waitUntil — the author chose.
+ /// `content` length just before the emitted goto line.
+ start: usize,
+ /// Scrubbed replacement line. Null when the call carried an explicit
+ /// waitUntil — the author chose.
downgraded: ?[]u8,
};
@@ -69,15 +72,13 @@ pub fn deinit(self: *Recorder) void {
self.arena.deinit();
}
-pub fn bytes(self: *Recorder) ![]const u8 {
- try self.flushPendingGoto(false);
+pub fn bytes(self: *Recorder) []const u8 {
+ // A snapshot closes the rewrite window: the caller may persist this exact
+ // text, so the goto line it contains must not change afterwards.
+ self.freePendingGoto();
return self.content.written();
}
-pub fn isEmpty(self: *const Recorder) bool {
- return self.content.writer.end == 0 and self.pending_goto == null;
-}
-
pub fn reset(self: *Recorder) void {
self.freePendingGoto();
self.lines = 0;
@@ -91,82 +92,83 @@ pub fn record(self: *Recorder, cmd: Command) !void {
if (!cmd.isRecorded()) return;
// `isRecorded` guarantees `.tool_call`.
const tool = cmd.tool_call.tool;
- try self.flushPendingGoto(tool == .waitForSelector or tool == .waitForState);
+ if (self.pending_goto) |pending| {
+ if (tool.waitsForReadiness()) try self.downgradePendingGoto(pending);
+ self.freePendingGoto();
+ }
// The page is born once, up front; every recorded call is then a method
- // on it — `goto` async, the rest sync.
+ // on it — `goto` async, the rest sync. Constant text: skip buf and scrub.
if (!self.page_declared) {
- self.buf.clearRetainingCapacity();
- try self.buf.writer.writeAll("const page = new Page();\n");
+ try self.content.writer.writeAll("const page = new Page();\n");
+ self.lines += 1;
self.page_declared = true;
- try self.appendScrubbed();
}
self.buf.clearRetainingCapacity();
_ = self.arena.reset(.retain_capacity);
- if (tool.isAsync()) try self.buf.writer.writeAll("await ");
- try self.buf.writer.writeAll("page.");
- try cmd.formatJs(self.arena.allocator(), &self.buf.writer);
- try self.buf.writer.writeByte('\n');
+ try self.renderCall(cmd, &self.buf.writer);
+
+ const start = self.content.written().len;
+ try self.appendScrubbed();
if (tool == .goto) {
- const plain = try self.allocator.dupe(u8, self.buf.written());
- errdefer self.allocator.free(plain);
- self.pending_goto = .{ .plain = plain, .downgraded = try self.renderDowngradedGoto(cmd) };
- return;
+ self.pending_goto = .{ .start = start, .downgraded = try self.renderDowngradedGoto(cmd) };
}
- try self.appendScrubbed();
}
pub fn recordComment(self: *Recorder, comment: []const u8) !void {
- try self.flushPendingGoto(false);
+ self.freePendingGoto();
self.buf.clearRetainingCapacity();
try writeCommentLines(&self.buf.writer, comment);
try self.appendScrubbed();
}
pub fn recordRaw(self: *Recorder, line: []const u8) !void {
- try self.flushPendingGoto(false);
+ self.freePendingGoto();
self.buf.clearRetainingCapacity();
try self.buf.writer.writeAll(line);
try self.buf.writer.writeByte('\n');
try self.appendScrubbed();
}
-/// The held goto's `domcontentloaded` twin, rendered up front while the
-/// command's args are still alive. Null when there's nothing to downgrade
-/// (explicit waitUntil, or args in a shape the recorder doesn't rewrite).
+/// One recorded call as a script line: receiver, await-ness, terminator.
+fn renderCall(self: *Recorder, cmd: Command, w: *std.Io.Writer) !void {
+ if (cmd.tool_call.tool.isAsync()) try w.writeAll("await ");
+ try w.writeAll("page.");
+ try cmd.formatJs(self.arena.allocator(), w);
+ try w.writeByte('\n');
+}
+
+/// The emitted goto's `domcontentloaded` twin, rendered and scrubbed up front
+/// while the command's args are still alive. Null when there's nothing to
+/// downgrade (explicit waitUntil, or non-object args).
fn renderDowngradedGoto(self: *Recorder, cmd: Command) !?[]u8 {
- const args = cmd.tool_call.args orelse return null;
+ // `isRecorded` guaranteed the required `url` arg, so args is present.
+ const args = cmd.tool_call.args.?;
if (args != .object) return null;
if (args.object.get("waitUntil") != null) return null;
const aa = self.arena.allocator();
- var cloned: std.json.ObjectMap = .empty;
- try cloned.ensureTotalCapacity(aa, args.object.count() + 1);
- var it = args.object.iterator();
- while (it.next()) |entry| try cloned.put(aa, entry.key_ptr.*, entry.value_ptr.*);
- try cloned.put(aa, "waitUntil", .{ .string = "domcontentloaded" });
-
- var w: std.Io.Writer.Allocating = .init(self.arena.allocator());
- try w.writer.writeAll("await page.");
- try Command.fromToolCall(.goto, .{ .object = cloned }).formatJs(self.arena.allocator(), &w.writer);
- try w.writer.writeByte('\n');
- return try self.allocator.dupe(u8, w.written());
-}
+ var cloned = try args.object.clone(aa);
+ try cloned.put(aa, "waitUntil", .{ .string = @tagName(lp.Config.WaitUntil.domcontentloaded) });
-fn flushPendingGoto(self: *Recorder, downgrade: bool) !void {
- const pending = self.pending_goto orelse return;
- defer self.freePendingGoto();
self.buf.clearRetainingCapacity();
- const line = if (downgrade) pending.downgraded orelse pending.plain else pending.plain;
- try self.buf.writer.writeAll(line);
- try self.appendScrubbed();
+ try self.renderCall(.fromToolCall(.goto, .{ .object = cloned }), &self.buf.writer);
+ const scrubbed = try lp.tools.reverseSubstituteEnvVars(aa, self.buf.written());
+ return try self.allocator.dupe(u8, scrubbed);
+}
+
+/// Swap the just-emitted plain goto line for its `domcontentloaded` twin.
+/// Both are exactly one line, so `lines` needs no adjustment.
+fn downgradePendingGoto(self: *Recorder, pending: PendingGoto) !void {
+ const line = pending.downgraded orelse return;
+ self.content.shrinkRetainingCapacity(pending.start);
+ try self.content.writer.writeAll(line);
}
fn freePendingGoto(self: *Recorder) void {
const pending = self.pending_goto orelse return;
- self.allocator.free(pending.plain);
if (pending.downgraded) |d| self.allocator.free(d);
self.pending_goto = null;
}
@@ -217,16 +219,16 @@ test "record filters state-mutating commands and comments" {
try std.testing.expectEqualStrings(
"const page = new Page();\nawait page.goto(\"https://example.com\");\npage.click({ selector: \"Login\" });\n// search for login\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
try std.testing.expectEqual(@as(u32, 4), recorder.lines);
recorder.reset();
- try std.testing.expectEqualStrings("", try recorder.bytes());
+ try std.testing.expectEqualStrings("", recorder.bytes());
try std.testing.expectEqual(@as(u32, 0), recorder.lines);
try recorder.record(parseLine(aa, "/scroll y=200"));
- try std.testing.expectEqualStrings("const page = new Page();\npage.scroll({ y: 200 });\n", try recorder.bytes());
+ try std.testing.expectEqualStrings("const page = new Page();\npage.scroll({ y: 200 });\n", recorder.bytes());
try std.testing.expectEqual(@as(u32, 2), recorder.lines);
}
@@ -237,7 +239,7 @@ test "recordRaw writes the JS line verbatim" {
try recorder.recordRaw("document.title");
try recorder.recordRaw("window.scrollTo(0, 100)");
- try std.testing.expectEqualStrings("document.title\nwindow.scrollTo(0, 100)\n", try recorder.bytes());
+ try std.testing.expectEqualStrings("document.title\nwindow.scrollTo(0, 100)\n", recorder.bytes());
}
test "record emits multi-line extract as JavaScript" {
@@ -253,7 +255,7 @@ test "record emits multi-line extract as JavaScript" {
try std.testing.expectEqualStrings(
"const page = new Page();\npage.extract({ title: \"span.title\", desc: \"p.description\" });\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -267,7 +269,7 @@ test "recordComment splits embedded newlines into separate comment lines" {
try std.testing.expectEqualStrings(
"// note\n// /goto https://attacker\n// more\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -284,7 +286,7 @@ test "recordComment scrubs literal LP_* values back to placeholders" {
try std.testing.expectEqualStrings(
"// a user noted that their password is $LP_RECORDER_COMMENT_TEST\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -301,7 +303,7 @@ test "record downgrades goto before waitForSelector to domcontentloaded" {
try std.testing.expectEqualStrings(
"const page = new Page();\nawait page.goto({ url: \"https://example.com\", waitUntil: \"domcontentloaded\" });\npage.waitForSelector(\".story\");\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -318,7 +320,7 @@ test "record keeps the load wait when goto is followed by extract" {
try std.testing.expectEqualStrings(
"const page = new Page();\nawait page.goto(\"https://example.com\");\npage.extract({ title: \"h1\" });\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -335,7 +337,7 @@ test "record preserves an explicit waitUntil on goto" {
try std.testing.expectEqualStrings(
"const page = new Page();\nawait page.goto({ url: \"https://example.com\", waitUntil: \"networkidle\" });\npage.waitForSelector(\".story\");\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -350,7 +352,7 @@ test "trailing goto is flushed unmodified by bytes" {
try recorder.record(parseLine(aa, "/goto https://example.com"));
try std.testing.expectEqualStrings(
"const page = new Page();\nawait page.goto(\"https://example.com\");\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}
@@ -370,6 +372,6 @@ test "record scrubs literal LP_* values in JavaScript calls" {
try recorder.record(parseLine(aa, "/fill selector='#user' value='secret-user'"));
try std.testing.expectEqualStrings(
"const page = new Page();\npage.fill({ selector: \"#user\", value: \"$LP_RECORDER_COMMAND_TEST\" });\n",
- try recorder.bytes(),
+ recorder.bytes(),
);
}