Skip to content

Lightpanda: the headless browser designed for AI and automation

handle error in json+png error

Karl Seguin05627c53684f
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index 148bf7a38..1ce1e85ff 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -313,10 +313,6 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
continue;
};
const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));
- if (remaining == 0) {
- err.* = error.Timeout;
- continue;
- }
_ = runner.waitForSelector(frame._frame_id, selector, remaining) catch |e| {
err.* = e;
};
@@ -331,10 +327,6 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
continue;
};
const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));
- if (remaining == 0) {
- err.* = error.Timeout;
- continue;
- }
runner.waitForScript(frame._frame_id, wait_script, remaining) catch |e| {
err.* = e;
};
@@ -344,19 +336,41 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
var http_error = false;
for (pages.items, errors) |page, *err| {
const frame = page.frame() orelse {
- if (err.* == null) err.* = error.FrameClosed;
+ if (err.* == null) {
+ err.* = error.FrameClosed;
+ }
continue;
};
- if (err.* == null) err.* = frame._last_navigate_error;
+ if (err.* == null) {
+ err.* = frame._last_navigate_error;
+ }
if (frame._http_status) |status| {
- if (status >= 400) http_error = true;
+ if (status >= 400) {
+ http_error = true;
+ }
}
}
try writeResults(app, opts, pages.items, errors);
- for (errors) |err| {
- if (err) |e| return e;
+ var failed = false;
+ for (urls, pages.items, errors) |url, page, err| {
+ if (err) |e| {
+ failed = true;
+ log.err(.app, "page failed", .{ .url = url, .err = e });
+ continue;
+ }
+ if (!opts.fail_on_http_error) {
+ continue;
+ }
+ const frame = page.frame() orelse continue;
+ const status = frame._http_status orelse continue;
+ if (status >= 400) {
+ log.err(.app, "page http error", .{ .url = url, .status = status });
+ }
+ }
+ if (failed) {
+ return error.PageFailed;
}
if (opts.fail_on_http_error and http_error) {
return error.HttpError;
@@ -386,14 +400,13 @@ fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, e
const arena = try app.arena_pool.acquire(.large, "screenshot.dump");
defer arena.release();
- if (screenshot.prepare(arena.allocator(), try dumpRoot(frame.?, opts.selector), .{
- .width = frame.?._page.getViewport().width,
- }, frame.?)) |shot| {
+ if (prepareShot(arena.allocator(), frame.?, opts)) |shot| {
try writeJsonEnvelope(writer, frame, opts.dump_mode, shot, err.*);
- continue;
} else |e| {
if (err.* == null) err.* = e;
+ try writeJsonEnvelope(writer, frame, opts.dump_mode, "", err.*);
}
+ continue;
}
var aw: std.Io.Writer.Allocating = .init(app.allocator);
@@ -425,6 +438,12 @@ fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, e
try writer.flush();
}
+fn prepareShot(arena: std.mem.Allocator, frame: *Frame, opts: FetchOpts) !screenshot.Prepared {
+ return screenshot.prepare(arena, try dumpRoot(frame, opts.selector), .{
+ .width = frame._page.getViewport().width,
+ }, frame);
+}
+
fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
const document = frame.window._document.asNode();
const sel = selector orelse return document;
diff --git a/src/main.zig b/src/main.zig
index 52169514c..55440404f 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -49,11 +49,13 @@ pub fn main(init: std.process.Init) !void {
run(gpa, main_arena, init.minimal.args) catch |err| {
if (err == error.UserCancelled) std.process.exit(130);
// error.AgentFailed: the agent thread reported the failure in-context.
+ // error.PageFailed: fetch already logged every failing url.
// lp.Agent.UserError: a user-facing message was already printed.
- if (err == error.AgentFailed or lp.Agent.isUserError(err)) std.process.exit(1);
+ if (err == error.AgentFailed or err == error.PageFailed or lp.Agent.isUserError(err)) std.process.exit(1);
+ // curl's code for --fail on an HTTP error, also already reported per url.
+ if (err == error.HttpError) std.process.exit(22);
log.fatal(.app, "exit", .{ .err = err });
- // curl's code for --fail on an HTTP error.
- std.process.exit(if (err == error.HttpError) 22 else 1);
+ std.process.exit(1);
};
}
@@ -378,6 +380,8 @@ fn fetchThread(app: *App, ft: *FetchTerminator, urls: []const [:0]const u8, fetc
lp.fetch(app, &browser, urls, fetch_opts) catch |err| {
err_out.* = err;
+ // Both are already reported per url by fetch itself.
+ if (err == error.PageFailed or err == error.HttpError) return;
log.fatal(.app, "fetch error", .{ .err = err, .url_count = urls.len });
};
}