Skip to content

Lightpanda: the headless browser designed for AI and automation

cli: fetch multiple urls

Karl Seguin630826160e7b
diff --git a/src/Config.zig b/src/Config.zig
index 10cdd3a7c..68db591aa 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -205,8 +205,8 @@ const Commands = cli.Builder(.{
},
.{
.name = "fetch",
- // This argument can be given out of order.
- .positional = .{ .name = "url", .type = ?[:0]const u8 },
+ // One or more URLs; can be given out of order, interleaved with options.
+ .positional = .{ .name = "url", .type = [:0]const u8, .multiple = true },
.options = .{
.{ .name = "dump", .type = ?DumpFormat, .validator = dumpValidator },
.{ .name = "with_base", .type = bool },
diff --git a/src/cli.zig b/src/cli.zig
index 8b9812d75..a4798308a 100644
--- a/src/cli.zig
+++ b/src/cli.zig
@@ -49,11 +49,13 @@ const log = lp.log;
/// for none.
/// - `shared_options: tuple` (optional) — extra options merged into this
/// command. Useful for common flags shared across commands.
-/// - `positional: struct` (optional) — a single positional argument with
-/// `.name` and `.type`. Type must be an optional pointer-to-u8 slice
-/// (e.g. `?[:0]const u8`); it defaults to `null` and may appear anywhere
-/// in argv. Passing it more than once returns
-/// `error.TooManyPositionalArguments`.
+/// - `positional: struct` (optional) — a positional argument with `.name`
+/// and `.type` that may appear anywhere in argv. By default it holds a
+/// single value: `.type` must be an optional pointer-to-u8 slice (e.g.
+/// `?[:0]const u8`), it defaults to `null`, and passing it more than once
+/// returns `error.TooManyPositionalArguments`. With `.multiple = true`,
+/// `.type` is the (non-optional) element slice (e.g. `[:0]const u8`), the
+/// field becomes a `std.ArrayList(type)`, and each occurrence appends.
///
/// ## Option descriptor fields
///
@@ -269,15 +271,7 @@ pub fn Builder(comptime commands: anytype) type {
else
.{}) ++
(if (@hasField(Command, "positional"))
- [1]std.builtin.Type.StructField{
- .{
- .name = command.positional.name,
- .type = command.positional.type,
- .default_value_ptr = @ptrCast(&@as(command.positional.type, null)),
- .is_comptime = false,
- .alignment = @alignOf(command.positional.type),
- },
- }
+ [1]std.builtin.Type.StructField{positionalField(command.positional)}
else
.{});
@@ -307,6 +301,26 @@ pub fn Builder(comptime commands: anytype) type {
});
};
+ /// Builds the `StructField` for a command's positional argument. A plain
+ /// positional is an optional that defaults to `null`; a `multiple`
+ /// positional collects every occurrence into an `ArrayList` that
+ /// defaults to empty.
+ fn positionalField(comptime positional: anytype) std.builtin.Type.StructField {
+ const is_multiple = @hasField(@TypeOf(positional), "multiple") and positional.multiple;
+ const T = if (is_multiple) std.ArrayList(positional.type) else positional.type;
+ const default: *const anyopaque = if (is_multiple)
+ @ptrCast(&@as(T, .{}))
+ else
+ @ptrCast(&@as(T, null));
+ return .{
+ .name = positional.name,
+ .type = T,
+ .default_value_ptr = default,
+ .is_comptime = false,
+ .alignment = @alignOf(T),
+ };
+ }
+
/// Parses executable name, command and options via single call.
pub fn parse(allocator: Allocator) !struct { []const u8, Union } {
var args = try std.process.argsWithAllocator(allocator);
@@ -683,14 +697,17 @@ pub fn Builder(comptime commands: anytype) type {
// ---------------------------------^
if (comptime @hasField(@TypeOf(command), "positional")) {
const positional = command.positional;
+ const is_multiple = comptime @hasField(@TypeOf(positional), "multiple") and positional.multiple;
- // Already given one.
- if (@field(c, positional.name) != null) {
+ // A single (non-multiple) positional may only be given once.
+ if (!is_multiple and @field(c, positional.name) != null) {
return error.TooManyPositionalArguments;
}
- // The positional must be an optional type.
- const info = @typeInfo(@typeInfo(positional.type).optional.child);
+ // Element type: the optional's child for a single positional,
+ // the slice element itself for a `multiple` one.
+ const Child = if (is_multiple) positional.type else @typeInfo(positional.type).optional.child;
+ const info = @typeInfo(Child);
const str = @as([]const u8, option_name);
switch (info) {
@@ -715,7 +732,11 @@ pub fn Builder(comptime commands: anytype) type {
break :blk buf;
};
- @field(c, positional.name) = v;
+ if (is_multiple) {
+ try @field(c, positional.name).append(allocator, v);
+ } else {
+ @field(c, positional.name) = v;
+ }
},
inline else => @compileError("not supported"),
}
@@ -725,10 +746,13 @@ pub fn Builder(comptime commands: anytype) type {
}
}
- // A non-optional positional that is still null after parsing is missing.
+ // A non-optional, single positional that is still null after parsing
+ // is missing. A `multiple` positional is never required here — the
+ // caller decides whether an empty list is acceptable.
if (comptime @hasField(@TypeOf(command), "positional")) {
+ const is_multiple = @hasField(@TypeOf(command.positional), "multiple") and command.positional.multiple;
const is_optional = @typeInfo(command.positional.type) == .optional;
- if (!is_optional and @field(c, command.positional.name) == null) {
+ if (!is_multiple and !is_optional and @field(c, command.positional.name) == null) {
return error.MissingArgument;
}
}
diff --git a/src/help.zon b/src/help.zon
index ed3f04dd0..eb9d11c2d 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -46,9 +46,10 @@
\\ Defaults to no cookie loading.
,
.fetch =
- \\usage: {0s} fetch <url> [OPTIONS] [COMMON_OPTIONS]
+ \\usage: {0s} fetch <url>... [OPTIONS] [COMMON_OPTIONS]
\\
- \\Fetches the specified URL.
+ \\Fetches one or more URLs. Each URL is loaded in its own page. Passing more
+ \\than one URL requires --json.
\\
\\options:
\\ --dump <DUMP>
@@ -69,9 +70,10 @@
\\ invisible Best-effort (e.g. display:none) hidden elements
\\ full Strip everything.
\\ --json
- \\ Capture and print the status of the fetch in a JSON string and output
- \\ it. When used with --dump <MODE> this will wrap the dumped content
- \\ within the JSON value.
+ \\ Capture and print the status of the fetch as JSON. A single URL
+ \\ prints one object; multiple URLs print {{"results": [ ... ]}} with
+ \\ one object per URL. When used with --dump <MODE> the dumped
+ \\ content is wrapped within each object.
\\ --with-base
\\ Add a <base> tag in dump.
\\ Defaults to false.
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index 59519a407..3179e87be 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -68,7 +68,7 @@ pub const FetchOpts = struct {
writer: ?*std.Io.Writer = null,
json: bool = false,
};
-/// Loads `url` in a fresh session and waits per `opts`.
+/// Loads each url in `urls` in a fresh session and waits per `opts`.
///
/// Errors:
/// - `error.Timeout` if the wait deadline (`opts.wait_ms`) expires.
@@ -77,7 +77,7 @@ pub const FetchOpts = struct {
/// `session.cancel_hook = .{...}`; without it, this error never fires.
/// - Other errors from navigation / parsing / I/O surface as their
/// underlying tag.
-pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !void {
+pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: FetchOpts) !void {
const notification = try Notification.init(app.allocator);
defer notification.deinit();
@@ -98,10 +98,12 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !
// Stash scripts user want to inject.
session.inject_scripts = opts.inject_script.items;
- // A navigation-safe handle: `page.frame()` always re-resolves the live
- // frame, so there's no stale-*Frame footgun across navigate / wait.
- const page = try session.createPage();
- {
+ // One page per url. `PageHandle.frame()` always re-resolves the live frame,
+ // so the handles stay valid across navigate / wait. The Runner's wait paths
+ // already operate over every live page in the session.
+ var pages: std.ArrayList(Session.PageHandle) = try .initCapacity(session.arena, urls.len);
+ for (urls) |url| {
+ const page = try session.createPage();
const frame = page.frame().?;
// not guaranteed to be valid after navigate
const encoded_url = try URL.ensureEncoded(frame.call_arena, url, "UTF-8");
@@ -109,7 +111,9 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !
.reason = .address_bar,
.kind = .{ .push = null },
});
+ pages.appendAssumeCapacity(page);
}
+
var runner = session.runner(.{});
var timer = try std.time.Timer.start();
@@ -152,16 +156,37 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !
const writer = opts.writer orelse return;
if (opts.json) {
- var aw: std.Io.Writer.Allocating = .init(app.allocator);
- defer aw.deinit();
-
- if (opts.dump_mode) |mode| blk: {
- const frame = page.frame() orelse break :blk;
- try dumpContent(app, mode, opts.dump, frame, &aw.writer);
+ // A single url keeps the original bare-object output. Multiple urls are
+ // wrapped in an extensible `{"results": [...]}` envelope: a bare
+ // top-level array is hard to evolve (consumers index it directly),
+ // whereas an object lets us add sibling fields later without breaking
+ // anyone reading `results`.
+ const wrap = pages.items.len > 1;
+ if (wrap) {
+ try writer.writeAll("{\"results\":[");
}
+ for (pages.items, 0..) |page, i| {
+ if (i != 0) {
+ try writer.writeByte(',');
+ }
+
+ var aw: std.Io.Writer.Allocating = .init(app.allocator);
+ defer aw.deinit();
- try writeJsonEnvelope(writer, page.frame(), opts.dump_mode, aw.written());
+ if (opts.dump_mode) |mode| blk: {
+ const frame = page.frame() orelse break :blk;
+ try dumpContent(app, mode, opts.dump, frame, &aw.writer);
+ }
+
+ try writeJsonEnvelope(writer, page.frame(), opts.dump_mode, aw.written());
+ }
+ if (wrap) {
+ try writer.writeAll("]}");
+ }
+ try writer.writeByte('\n');
} else {
+ // main validates that non-JSON dump is only reached with a single url.
+ const page = pages.items[0];
if (opts.dump_mode) |mode| blk: {
const frame = page.frame() orelse {
try writer.writeAll("Frame closed. Please open a bug report including the URL\n");
@@ -199,6 +224,8 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:
}
}
+// Writes a single page's result object. Framing (the enclosing array and any
+// separators / trailing newline) is the caller's responsibility.
fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config.DumpFormat, content: []const u8) !void {
const meta: ?Frame.HttpMetadata = if (frame) |f| f.httpMetadata() else null;
try std.json.Stringify.value(.{
@@ -208,7 +235,6 @@ fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config.
.dump = if (dump_mode) |mode| @tagName(mode) else "",
.content = content,
}, .{}, writer);
- try writer.writeByte('\n');
}
fn dumpWPT(frame: *Frame, writer: *std.Io.Writer) !void {
diff --git a/src/main.zig b/src/main.zig
index 22f2b8f55..6495406e9 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -122,8 +122,34 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
app.network.run();
},
.fetch => |opts| {
- const url = opts.url;
- log.debug(.app, "startup", .{ .mode = "fetch", .dump_mode = opts.dump, .url = url, .snapshot = app.snapshot.fromEmbedded() });
+ const urls = opts.url.items;
+
+ if (urls.len == 0) {
+ log.fatal(.app, "missing URL", .{});
+ return error.MissingArgument;
+ }
+
+ // Plain (non-JSON) dump writes one document to stdout with no
+ // framing, so it can't disambiguate more than one page.
+ if (urls.len == 1) {
+ log.debug(.app, "startup", .{
+ .mode = "fetch",
+ .dump_mode = opts.dump,
+ .url = urls[0],
+ .snapshot = app.snapshot.fromEmbedded(),
+ });
+ } else {
+ if (opts.json == false) {
+ log.fatal(.app, "multiple URLs require --json", .{});
+ return error.InvalidArgument;
+ }
+ log.debug(.app, "startup", .{
+ .mode = "fetch",
+ .dump_mode = opts.dump,
+ .url_count = urls.len,
+ .snapshot = app.snapshot.fromEmbedded(),
+ });
+ }
var fetch_opts = lp.FetchOpts{
.wait_ms = opts.wait_ms,
@@ -157,7 +183,7 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
try sighandler.deadline(ms);
}
- var worker_thread = try std.Thread.spawn(.{}, fetchThread, .{ app, &ft, url.?, fetch_opts });
+ var worker_thread = try std.Thread.spawn(.{}, fetchThread, .{ app, &ft, urls, fetch_opts });
defer worker_thread.join();
app.network.run();
@@ -282,7 +308,7 @@ const FetchTerminator = struct {
}
};
-fn fetchThread(app: *App, ft: *FetchTerminator, url: [:0]const u8, fetch_opts: lp.FetchOpts) void {
+fn fetchThread(app: *App, ft: *FetchTerminator, urls: []const [:0]const u8, fetch_opts: lp.FetchOpts) void {
defer app.network.stop();
var browser: lp.Browser = undefined;
@@ -298,8 +324,8 @@ fn fetchThread(app: *App, ft: *FetchTerminator, url: [:0]const u8, fetch_opts: l
// process-of) shutting down browser/env
defer ft.releaseBrowser();
- lp.fetch(app, &browser, url, fetch_opts) catch |err| {
- log.fatal(.app, "fetch error", .{ .err = err, .url = url });
+ lp.fetch(app, &browser, urls, fetch_opts) catch |err| {
+ log.fatal(.app, "fetch error", .{ .err = err, .url_count = urls.len });
};
}