Lightpanda: the headless browser designed for AI and automation
cli: fetch multiple urls
Karl Seguin630826160e7b
diff --git a/src/Config.zig b/src/Config.zigindex 10cdd3a7c..68db591aa 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -205,8 +205,8 @@ const Commands = cli.Builder(.{},.{.name = "fetch",- // This argument can be given out of order.- .positional = .{ .name = "url", .type = ?[:0]const u8 },+ // One or more URLs; can be given out of order, interleaved with options.+ .positional = .{ .name = "url", .type = [:0]const u8, .multiple = true },.options = .{.{ .name = "dump", .type = ?DumpFormat, .validator = dumpValidator },.{ .name = "with_base", .type = bool },diff --git a/src/cli.zig b/src/cli.zigindex 8b9812d75..a4798308a 100644--- a/src/cli.zig+++ b/src/cli.zig@@ -49,11 +49,13 @@ const log = lp.log;/// for none./// - `shared_options: tuple` (optional) — extra options merged into this/// command. Useful for common flags shared across commands.-/// - `positional: struct` (optional) — a single positional argument with-/// `.name` and `.type`. Type must be an optional pointer-to-u8 slice-/// (e.g. `?[:0]const u8`); it defaults to `null` and may appear anywhere-/// in argv. Passing it more than once returns-/// `error.TooManyPositionalArguments`.+/// - `positional: struct` (optional) — a positional argument with `.name`+/// and `.type` that may appear anywhere in argv. By default it holds a+/// single value: `.type` must be an optional pointer-to-u8 slice (e.g.+/// `?[:0]const u8`), it defaults to `null`, and passing it more than once+/// returns `error.TooManyPositionalArguments`. With `.multiple = true`,+/// `.type` is the (non-optional) element slice (e.g. `[:0]const u8`), the+/// field becomes a `std.ArrayList(type)`, and each occurrence appends.////// ## Option descriptor fields///@@ -269,15 +271,7 @@ pub fn Builder(comptime commands: anytype) type {else.{}) ++(if (@hasField(Command, "positional"))- [1]std.builtin.Type.StructField{- .{- .name = command.positional.name,- .type = command.positional.type,- .default_value_ptr = @ptrCast(&@as(command.positional.type, null)),- .is_comptime = false,- .alignment = @alignOf(command.positional.type),- },- }+ [1]std.builtin.Type.StructField{positionalField(command.positional)}else.{});@@ -307,6 +301,26 @@ pub fn Builder(comptime commands: anytype) type {});};+ /// Builds the `StructField` for a command's positional argument. A plain+ /// positional is an optional that defaults to `null`; a `multiple`+ /// positional collects every occurrence into an `ArrayList` that+ /// defaults to empty.+ fn positionalField(comptime positional: anytype) std.builtin.Type.StructField {+ const is_multiple = @hasField(@TypeOf(positional), "multiple") and positional.multiple;+ const T = if (is_multiple) std.ArrayList(positional.type) else positional.type;+ const default: *const anyopaque = if (is_multiple)+ @ptrCast(&@as(T, .{}))+ else+ @ptrCast(&@as(T, null));+ return .{+ .name = positional.name,+ .type = T,+ .default_value_ptr = default,+ .is_comptime = false,+ .alignment = @alignOf(T),+ };+ }+/// Parses executable name, command and options via single call.pub fn parse(allocator: Allocator) !struct { []const u8, Union } {var args = try std.process.argsWithAllocator(allocator);@@ -683,14 +697,17 @@ pub fn Builder(comptime commands: anytype) type {// ---------------------------------^if (comptime @hasField(@TypeOf(command), "positional")) {const positional = command.positional;+ const is_multiple = comptime @hasField(@TypeOf(positional), "multiple") and positional.multiple;- // Already given one.- if (@field(c, positional.name) != null) {+ // A single (non-multiple) positional may only be given once.+ if (!is_multiple and @field(c, positional.name) != null) {return error.TooManyPositionalArguments;}- // The positional must be an optional type.- const info = @typeInfo(@typeInfo(positional.type).optional.child);+ // Element type: the optional's child for a single positional,+ // the slice element itself for a `multiple` one.+ const Child = if (is_multiple) positional.type else @typeInfo(positional.type).optional.child;+ const info = @typeInfo(Child);const str = @as([]const u8, option_name);switch (info) {@@ -715,7 +732,11 @@ pub fn Builder(comptime commands: anytype) type {break :blk buf;};- @field(c, positional.name) = v;+ if (is_multiple) {+ try @field(c, positional.name).append(allocator, v);+ } else {+ @field(c, positional.name) = v;+ }},inline else => @compileError("not supported"),}@@ -725,10 +746,13 @@ pub fn Builder(comptime commands: anytype) type {}}- // A non-optional positional that is still null after parsing is missing.+ // A non-optional, single positional that is still null after parsing+ // is missing. A `multiple` positional is never required here — the+ // caller decides whether an empty list is acceptable.if (comptime @hasField(@TypeOf(command), "positional")) {+ const is_multiple = @hasField(@TypeOf(command.positional), "multiple") and command.positional.multiple;const is_optional = @typeInfo(command.positional.type) == .optional;- if (!is_optional and @field(c, command.positional.name) == null) {+ if (!is_multiple and !is_optional and @field(c, command.positional.name) == null) {return error.MissingArgument;}}diff --git a/src/help.zon b/src/help.zonindex ed3f04dd0..eb9d11c2d 100644--- a/src/help.zon+++ b/src/help.zon@@ -46,9 +46,10 @@\\ Defaults to no cookie loading.,.fetch =- \\usage: {0s} fetch <url> [OPTIONS] [COMMON_OPTIONS]+ \\usage: {0s} fetch <url>... [OPTIONS] [COMMON_OPTIONS]\\- \\Fetches the specified URL.+ \\Fetches one or more URLs. Each URL is loaded in its own page. Passing more+ \\than one URL requires --json.\\\\options:\\ --dump <DUMP>@@ -69,9 +70,10 @@\\ invisible Best-effort (e.g. display:none) hidden elements\\ full Strip everything.\\ --json- \\ Capture and print the status of the fetch in a JSON string and output- \\ it. When used with --dump <MODE> this will wrap the dumped content- \\ within the JSON value.+ \\ Capture and print the status of the fetch as JSON. A single URL+ \\ prints one object; multiple URLs print {{"results": [ ... ]}} with+ \\ one object per URL. When used with --dump <MODE> the dumped+ \\ content is wrapped within each object.\\ --with-base\\ Add a <base> tag in dump.\\ Defaults to false.diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 59519a407..3179e87be 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -68,7 +68,7 @@ pub const FetchOpts = struct {writer: ?*std.Io.Writer = null,json: bool = false,};-/// Loads `url` in a fresh session and waits per `opts`.+/// Loads each url in `urls` in a fresh session and waits per `opts`.////// Errors:/// - `error.Timeout` if the wait deadline (`opts.wait_ms`) expires.@@ -77,7 +77,7 @@ pub const FetchOpts = struct {/// `session.cancel_hook = .{...}`; without it, this error never fires./// - Other errors from navigation / parsing / I/O surface as their/// underlying tag.-pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !void {+pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: FetchOpts) !void {const notification = try Notification.init(app.allocator);defer notification.deinit();@@ -98,10 +98,12 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !// Stash scripts user want to inject.session.inject_scripts = opts.inject_script.items;- // A navigation-safe handle: `page.frame()` always re-resolves the live- // frame, so there's no stale-*Frame footgun across navigate / wait.- const page = try session.createPage();- {+ // One page per url. `PageHandle.frame()` always re-resolves the live frame,+ // so the handles stay valid across navigate / wait. The Runner's wait paths+ // already operate over every live page in the session.+ var pages: std.ArrayList(Session.PageHandle) = try .initCapacity(session.arena, urls.len);+ for (urls) |url| {+ const page = try session.createPage();const frame = page.frame().?;// not guaranteed to be valid after navigateconst encoded_url = try URL.ensureEncoded(frame.call_arena, url, "UTF-8");@@ -109,7 +111,9 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !.reason = .address_bar,.kind = .{ .push = null },});+ pages.appendAssumeCapacity(page);}+var runner = session.runner(.{});var timer = try std.time.Timer.start();@@ -152,16 +156,37 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !const writer = opts.writer orelse return;if (opts.json) {- var aw: std.Io.Writer.Allocating = .init(app.allocator);- defer aw.deinit();-- if (opts.dump_mode) |mode| blk: {- const frame = page.frame() orelse break :blk;- try dumpContent(app, mode, opts.dump, frame, &aw.writer);+ // A single url keeps the original bare-object output. Multiple urls are+ // wrapped in an extensible `{"results": [...]}` envelope: a bare+ // top-level array is hard to evolve (consumers index it directly),+ // whereas an object lets us add sibling fields later without breaking+ // anyone reading `results`.+ const wrap = pages.items.len > 1;+ if (wrap) {+ try writer.writeAll("{\"results\":[");}+ for (pages.items, 0..) |page, i| {+ if (i != 0) {+ try writer.writeByte(',');+ }++ var aw: std.Io.Writer.Allocating = .init(app.allocator);+ defer aw.deinit();- try writeJsonEnvelope(writer, page.frame(), opts.dump_mode, aw.written());+ if (opts.dump_mode) |mode| blk: {+ const frame = page.frame() orelse break :blk;+ try dumpContent(app, mode, opts.dump, frame, &aw.writer);+ }++ try writeJsonEnvelope(writer, page.frame(), opts.dump_mode, aw.written());+ }+ if (wrap) {+ try writer.writeAll("]}");+ }+ try writer.writeByte('\n');} else {+ // main validates that non-JSON dump is only reached with a single url.+ const page = pages.items[0];if (opts.dump_mode) |mode| blk: {const frame = page.frame() orelse {try writer.writeAll("Frame closed. Please open a bug report including the URL\n");@@ -199,6 +224,8 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:}}+// Writes a single page's result object. Framing (the enclosing array and any+// separators / trailing newline) is the caller's responsibility.fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config.DumpFormat, content: []const u8) !void {const meta: ?Frame.HttpMetadata = if (frame) |f| f.httpMetadata() else null;try std.json.Stringify.value(.{@@ -208,7 +235,6 @@ fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config..dump = if (dump_mode) |mode| @tagName(mode) else "",.content = content,}, .{}, writer);- try writer.writeByte('\n');}fn dumpWPT(frame: *Frame, writer: *std.Io.Writer) !void {diff --git a/src/main.zig b/src/main.zigindex 22f2b8f55..6495406e9 100644--- a/src/main.zig+++ b/src/main.zig@@ -122,8 +122,34 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {app.network.run();},.fetch => |opts| {- const url = opts.url;- log.debug(.app, "startup", .{ .mode = "fetch", .dump_mode = opts.dump, .url = url, .snapshot = app.snapshot.fromEmbedded() });+ const urls = opts.url.items;++ if (urls.len == 0) {+ log.fatal(.app, "missing URL", .{});+ return error.MissingArgument;+ }++ // Plain (non-JSON) dump writes one document to stdout with no+ // framing, so it can't disambiguate more than one page.+ if (urls.len == 1) {+ log.debug(.app, "startup", .{+ .mode = "fetch",+ .dump_mode = opts.dump,+ .url = urls[0],+ .snapshot = app.snapshot.fromEmbedded(),+ });+ } else {+ if (opts.json == false) {+ log.fatal(.app, "multiple URLs require --json", .{});+ return error.InvalidArgument;+ }+ log.debug(.app, "startup", .{+ .mode = "fetch",+ .dump_mode = opts.dump,+ .url_count = urls.len,+ .snapshot = app.snapshot.fromEmbedded(),+ });+ }var fetch_opts = lp.FetchOpts{.wait_ms = opts.wait_ms,@@ -157,7 +183,7 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {try sighandler.deadline(ms);}- var worker_thread = try std.Thread.spawn(.{}, fetchThread, .{ app, &ft, url.?, fetch_opts });+ var worker_thread = try std.Thread.spawn(.{}, fetchThread, .{ app, &ft, urls, fetch_opts });defer worker_thread.join();app.network.run();@@ -282,7 +308,7 @@ const FetchTerminator = struct {}};-fn fetchThread(app: *App, ft: *FetchTerminator, url: [:0]const u8, fetch_opts: lp.FetchOpts) void {+fn fetchThread(app: *App, ft: *FetchTerminator, urls: []const [:0]const u8, fetch_opts: lp.FetchOpts) void {defer app.network.stop();var browser: lp.Browser = undefined;@@ -298,8 +324,8 @@ fn fetchThread(app: *App, ft: *FetchTerminator, url: [:0]const u8, fetch_opts: l// process-of) shutting down browser/envdefer ft.releaseBrowser();- lp.fetch(app, &browser, url, fetch_opts) catch |err| {- log.fatal(.app, "fetch error", .{ .err = err, .url = url });+ lp.fetch(app, &browser, urls, fetch_opts) catch |err| {+ log.fatal(.app, "fetch error", .{ .err = err, .url_count = urls.len });};}