Lightpanda: the headless browser designed for AI and automation
fetch: per-page errors, an "error" field in --json, --fail-on-http-error
Adrià Arrufat6811cc822653
diff --git a/src/Config.zig b/src/Config.zigindex e0208dbe9..7dbebfe6f 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -376,6 +376,7 @@ const Commands = cli.Builder(.{.{ .name = "strip_mode", .type = dump.Opts.Strip, .default = dump.Opts.Strip{} },.{ .name = "dump_selector", .type = ?[:0]const u8 },.{ .name = "dump_max_bytes", .type = ?u32 },+ .{ .name = "fail_on_http_error", .type = bool },.{ .name = "wait_ms", .type = u32, .default = 5_000 },.{ .name = "wait_until", .type = ?WaitUntil },.{diff --git a/src/help.zon b/src/help.zonindex 60df687ad..646a8a6b0 100644--- a/src/help.zon+++ b/src/help.zon@@ -79,6 +79,11 @@\\ --dump-selector <QUERY>\\ Dump only the first element matching the CSS selector instead\\ of the whole document. Fails if nothing matches.+ \\ --fail-on-http-error+ \\ Exit with code 22 when any page's HTTP status is 400 or above.+ \\ The dump is still written. Navigation and wait failures exit+ \\ with code 1 regardless of this flag.+ \\ Defaults to false.\\ --inject-script <EXPR>\\ JavaScript to execute as the document's <head> is parsed, before any\\ other scripts in the page run. Can be passed multiple times; scripts@@ -90,7 +95,10 @@\\ Capture and print the status of the fetch as JSON. A single URL\\ prints one object; multiple URLs print {{"results": [ ... ]}} with\\ one object per URL. When used with --dump <MODE> the dumped- \\ content is wrapped within each object.+ \\ content is wrapped within each object. Each object carries an+ \\ "error" field: null, or the name of the navigation, wait or+ \\ dump failure for that URL. One failed URL does not stop the+ \\ others; the process exits 1 if any URL failed.\\ --metrics\\ Write metrics (Prometheus text format) to stdout on exit, after\\ any --dump output.diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 7e5c2050d..fbbaf9010 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -193,6 +193,8 @@ pub const FetchOpts = struct {selector: ?[:0]const u8 = null,/// html and markdown only.max_bytes: ?u32 = null,+ /// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`.+ fail_on_http_error: bool = false,writer: ?*std.Io.Writer = null,json: bool = false,};@@ -208,15 +210,13 @@ fn resolveWaitUntil(opts: FetchOpts) ?Config.WaitUntil {}/// Loads each url in `urls` in a fresh session and waits per `opts`.///-/// Errors:-/// - `error.Timeout` if the deadline expires while a `wait_selector` or-/// `wait_script` is still unmet. The `wait_until` phase never raises it:-/// when the budget runs out the page is dumped as-is.-/// - `error.Cancelled` if the embedder installed a `Session.cancel_hook`-/// that returned true during the wait. The hook is opt-in via-/// `session.cancel_hook = .{...}`; without it, this error never fires.-/// - Other errors from navigation / parsing / I/O surface as their-/// underlying tag.+/// A page's navigation, wait or dump failure is recorded for that page and+/// does not stop the others: every page is still written (JSON carries the+/// failure under `error`), then the first failure is returned. Wait failures+/// are `error.Timeout` when the deadline expires while a `wait_selector` or+/// `wait_script` is still unmet (the `wait_until` phase never raises it: when+/// the budget runs out the page is dumped as-is), or `error.Cancelled` if the+/// embedder's opt-in `Session.cancel_hook` returned true.pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: FetchOpts) !void {const notification = try Notification.init(app.allocator);defer notification.deinit();@@ -304,32 +304,69 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fettry runner.waitForAll(opts.wait_ms, .{ .until = wu });}+ // One slot per page; the first failure sticks.+ const errors = try session.arena.allocator().alloc(?anyerror, pages.items.len);+ @memset(errors, null);+if (opts.wait_selector) |selector| {- const elapsed: u32 = @intCast(timer.untilNow(io, .boot).toMilliseconds());- const remaining = opts.wait_ms -| elapsed;- if (remaining == 0) {- return error.Timeout;- }- for (session.pages.items) |p| {- if (p.replacement == null) {- _ = try runner.waitForSelector(p.frame._frame_id, selector, remaining);+ for (pages.items, errors) |page, *err| {+ if (err.* != null) continue;+ const frame = page.frame() orelse {+ err.* = error.FrameClosed;+ continue;+ };+ const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));+ if (remaining == 0) {+ err.* = error.Timeout;+ continue;}+ _ = runner.waitForSelector(frame._frame_id, selector, remaining) catch |e| {+ err.* = e;+ };}}if (opts.wait_script) |wait_script| {- const elapsed: u32 = @intCast(timer.untilNow(io, .boot).toMilliseconds());- const remaining = opts.wait_ms -| elapsed;- if (remaining == 0) {- return error.Timeout;- }- for (session.pages.items) |p| {- if (p.replacement == null) {- try runner.waitForScript(p.frame._frame_id, wait_script, remaining);+ for (pages.items, errors) |page, *err| {+ if (err.* != null) continue;+ const frame = page.frame() orelse {+ err.* = error.FrameClosed;+ continue;+ };+ const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));+ if (remaining == 0) {+ err.* = error.Timeout;+ continue;}+ runner.waitForScript(frame._frame_id, wait_script, remaining) catch |e| {+ err.* = e;+ };+ }+ }++ var http_error = false;+ for (pages.items, errors) |page, *err| {+ const frame = page.frame() orelse {+ if (err.* == null) err.* = error.FrameClosed;+ continue;+ };+ if (err.* == null) err.* = frame._last_navigate_error;+ if (frame._http_status) |status| {+ if (status >= 400) http_error = true;}}+ try writeResults(app, opts, pages.items, errors);++ for (errors) |err| {+ if (err) |e| return e;+ }+ if (opts.fail_on_http_error and http_error) {+ return error.HttpError;+ }+}++fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, errors: []?anyerror) !void {const writer = opts.writer orelse return;if (opts.json) {@@ -338,11 +375,11 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet// top-level array is hard to evolve (consumers index it directly),// whereas an object lets us add sibling fields later without breaking// anyone reading `results`.- const wrap = pages.items.len > 1;+ const wrap = pages.len > 1;if (wrap) {try writer.writeAll("{\"results\":[");}- for (pages.items, 0..) |page, i| {+ for (pages, errors, 0..) |page, *err, i| {if (i != 0) {try writer.writeByte(',');}@@ -352,20 +389,26 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fetconst arena = try app.arena_pool.acquire(.large, "screenshot.dump");defer arena.release();- const shot = try screenshot.prepare(arena.allocator(), try dumpRoot(frame.?, opts.selector), .{+ if (screenshot.prepare(arena.allocator(), try dumpRoot(frame.?, opts.selector), .{.width = frame.?._page.getViewport().width,- }, frame.?);- try writeJsonEnvelope(writer, frame, opts.dump_mode, shot);- continue;+ }, frame.?)) |shot| {+ try writeJsonEnvelope(writer, frame, opts.dump_mode, shot, err.*);+ continue;+ } else |e| {+ if (err.* == null) err.* = e;+ }}var aw: std.Io.Writer.Allocating = .init(app.allocator);defer aw.deinit();if (opts.dump_mode) |mode| {- if (frame) |f| try dumpContent(app, mode, opts, f, &aw.writer);+ if (frame) |f| dumpContent(app, mode, opts, f, &aw.writer) catch |e| {+ aw.clearRetainingCapacity();+ if (err.* == null) err.* = e;+ };}- try writeJsonEnvelope(writer, frame, opts.dump_mode, aw.written());+ try writeJsonEnvelope(writer, frame, opts.dump_mode, aw.written(), err.*);}if (wrap) {try writer.writeAll("]}");@@ -373,7 +416,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fettry writer.writeByte('\n');} else {// main validates that non-JSON dump is only reached with a single url.- const page = pages.items[0];+ const page = pages[0];if (opts.dump_mode) |mode| blk: {const frame = page.frame() orelse {try writer.writeAll("Frame closed. Please open a bug report including the URL\n");@@ -445,7 +488,7 @@ pub fn checkVersion(allocator: std.mem.Allocator, config: *const Config) !void {// Writes a single page's result object. Framing (the enclosing array and any// separators / trailing newline) is the caller's responsibility.-fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config.DumpFormat, content: anytype) !void {+fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config.DumpFormat, content: anytype, err: ?anyerror) !void {const meta: ?Frame.HttpMetadata = if (frame) |f| f.httpMetadata() else null;try std.json.Stringify.value(.{.url = if (meta) |m| m.url else "",@@ -453,6 +496,7 @@ fn writeJsonEnvelope(writer: *std.Io.Writer, frame: ?*Frame, dump_mode: ?Config..headers = if (meta) |m| m.headers else &.{},.dump = if (dump_mode) |mode| @tagName(mode) else "",.content = content,+ .@"error" = if (err) |e| @errorName(e) else null,}, .{}, writer);}@@ -575,20 +619,33 @@ test "writeJsonEnvelope: null frame" {var aw: std.Io.Writer.Allocating = .init(testing.allocator);defer aw.deinit();- try writeJsonEnvelope(&aw.writer, null, null, "");+ try writeJsonEnvelope(&aw.writer, null, null, "", null);try testing.expectJson(.{.url = "",.http_status = 0,.dump = "",.content = "",}, aw.written());+ try testing.expect(std.mem.indexOf(u8, aw.written(), "\"error\":null") != null);+}++test "writeJsonEnvelope: page error" {+ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();++ try writeJsonEnvelope(&aw.writer, null, .markdown, "", error.Timeout);+ try testing.expectJson(.{+ .dump = "markdown",+ .content = "",+ .@"error" = "Timeout",+ }, aw.written());}test "writeJsonEnvelope: null frame with dump mode and content" {var aw: std.Io.Writer.Allocating = .init(testing.allocator);defer aw.deinit();- try writeJsonEnvelope(&aw.writer, null, .html, "<html><body>hello</body></html>");+ try writeJsonEnvelope(&aw.writer, null, .html, "<html><body>hello</body></html>", null);try testing.expectJson(.{.dump = "html",.content = "<html><body>hello</body></html>",diff --git a/src/main.zig b/src/main.zigindex 01ccd4727..7b68b7e6a 100644--- a/src/main.zig+++ b/src/main.zig@@ -52,7 +52,8 @@ pub fn main(init: std.process.Init) !void {// lp.Agent.UserError: a user-facing message was already printed.if (err == error.AgentFailed or lp.Agent.isUserError(err)) std.process.exit(1);log.fatal(.app, "exit", .{ .err = err });- std.process.exit(1);+ // curl's code for --fail on an HTTP error.+ std.process.exit(if (err == error.HttpError) 22 else 1);};}@@ -177,6 +178,7 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args).dump_mode = opts.dump,.selector = opts.dump_selector,.max_bytes = opts.dump_max_bytes,+ .fail_on_http_error = opts.fail_on_http_error,.dump = .{.strip = opts.strip_mode,.with_base = opts.with_base,