Lightpanda: the headless browser designed for AI and automation
ops: attempt to generate an appropriate error on a bot challenge
Karl Seguin64dbff13c596
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 2b7ee4509..b2f1d4ce2 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -304,6 +304,7 @@ _type: enum { root, frame }, // only used for logs right now_req_id: u32 = 0,_navigated_options: ?NavigatedOpts = null,_http_status: ?u16 = null,+_bot_challenge: ?HttpClient.BotChallenge = null,_http_headers: std.ArrayList(HttpHeader) = .empty,_referrer: ?[]const u8 = null,@@ -816,6 +817,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo}self._http_status = null;+ self._bot_challenge = null;self._http_headers = .empty;self._referrer = null;@@ -1515,6 +1517,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.}self._http_status = transfer.responseStatus();+ self._bot_challenge = transfer.botChallenge();var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {try self._http_headers.append(self.arena, .{diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex e2e58524a..68a13942c 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -350,6 +350,16 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fetif (err.* == null) {err.* = frame._last_navigate_error;}+ if (err.* == null) {+ if (frame._bot_challenge) |challenge| {+ log.warn(.app, "bot challenge", .{+ .url = frame.url,+ .status = frame._http_status,+ .provider = challenge,+ });+ err.* = error.BotChallenge;+ }+ }if (frame._http_status) |status| {if (status >= 400) {http_error = true;diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 2665925e1..2f6f7e92d 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -4098,7 +4098,7 @@ pub const Transfer = struct {}pub fn getContentLength(self: *const Transfer) ?usize {- const cl = self.getContentLengthRawValue() orelse return null;+ const cl = self.responseHeader("content-length") orelse return null;return std.fmt.parseInt(usize, cl, 10) catch null;}@@ -4110,23 +4110,50 @@ pub const Transfer = struct {return self._body_len;}- fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {+ // First value of a response header. `name` must be lowercase.+ pub fn responseHeader(self: *const Transfer, name: [:0]const u8) ?[]const u8 {// Materialized headers (dispatch time, any source).for (self.res.headers) |hdr| {- if (std.mem.eql(u8, hdr.name, "content-length")) {+ if (std.mem.eql(u8, hdr.name, name)) {return hdr.value;}}// Mid-stream (curl's write callback): read from the live conn.if (self._conn) |c| {- const cl = c.getResponseHeader("content-length", 0) orelse return null;- return cl.value;+ const value = c.getResponseHeader(name, 0) orelse return null;+ return value.value;}return null;}+ pub fn botChallenge(self: *const Transfer) ?BotChallenge {+ const status = self.responseStatus() orelse return null;+ switch (status) {+ 403 => {+ const value = self.responseHeader("cf-mitigated") orelse return null;+ if (std.ascii.eqlIgnoreCase(value, "challenge")) {+ return .cloudflare;+ }+ },+ 429 => {+ const value = self.responseHeader("x-vercel-mitigated") orelse return null;+ if (std.ascii.eqlIgnoreCase(value, "challenge")) {+ return .vercel;+ }+ },+ 202, 405 => {+ const action = self.responseHeader("x-amzn-waf-action") orelse return null;+ if (std.ascii.eqlIgnoreCase(action, "captcha") or std.ascii.eqlIgnoreCase(action, "challenge")) {+ return .aws_waf;+ }+ },+ else => {},+ }+ return null;+ }+// Only ever called from tick, delivers buffered events to their consumer.// A buffered (completed) transfer gets exactly one deliver() and dies at// the end of it. A streaming transfer gets one deliver() per dispatched@@ -4441,6 +4468,12 @@ const Synthetic = struct {}};+pub const BotChallenge = enum {+ aws_waf,+ cloudflare,+ vercel,+};+const testing = @import("../testing.zig");// Only the transfer list, the url and the parent matter to the tests usingdiff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex 1ffe9a1e9..29f6a66c4 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -30,10 +30,11 @@ const ArenaPool = @import("../ArenaPool.zig");const http = @import("http.zig");const Robots = @import("Robots.zig");const Network = @import("Network.zig");-const Transfer = @import("HttpClient.zig").Transfer;+const HttpClient = @import("HttpClient.zig");const SingleFlight = @import("SingleFlight.zig");const log = lp.log;+const Transfer = HttpClient.Transfer;const RobotsGate = @This();@@ -129,6 +130,7 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transconst Outcome = union(enum) {decision: Robots.RobotStore.Decision,robots: Robots.Robots,+ challenged,};// The robots.txt fetch resolved: hand every waiter back to the pipeline,@@ -144,6 +146,11 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8, outcome: Outcome) voiconst decision: Robots.RobotStore.Decision = switch (outcome) {.decision => |d| d,.robots => |r| if (r.isAllowed(URL.getPathname(transfer.req.url))) .allowed else .blocked,+ .challenged => {+ lp.metrics.robots_access.incr(.deny);+ transfer.failAsync(error.BotChallenge);+ continue;+ },};if (decision == .blocked) {@@ -168,6 +175,7 @@ const RobotsContext = struct {robots_url: [:0]const u8,buffer: std.ArrayList(u8),status: u16 = 0,+ challenge: ?HttpClient.BotChallenge = null,fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));@@ -175,6 +183,7 @@ const RobotsContext = struct {log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });self.status = hdr.status;}+ self.challenge = transfer.botChallenge();lp.metrics.robots_status.incr(http.statusCategory(self.status));try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), transfer.bodyLen());return .proceed;@@ -192,6 +201,18 @@ const RobotsContext = struct {const robots_url = self.robots_url;const network = self.gate.network;+ if (self.challenge) |challenge| {+ // Nothing here can solve it, but the challenge might be gone by+ // the next request, so don't cache it.+ log.warn(.http, "robots.txt bot challenge", .{+ .url = robots_url,+ .status = self.status,+ .provider = challenge,+ });+ self.settle(.{ .outcome = .challenged, .cache = false });+ return;+ }+switch (self.status) {200 => {if (self.buffer.items.len == 0) {@@ -293,6 +314,7 @@ const RobotsContext = struct {.robots => |r| network.robot_store.put(self.robots_url, r) catch |err| {log.warn(.browser, "cache robots rules", .{ .url = self.robots_url, .err = err });},+ .challenged => {},}}}diff --git a/src/network/http.zig b/src/network/http.zigindex ab34bff33..7bd5486f6 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -931,6 +931,7 @@ pub const ErrorReason = enum {too_large,aborted,robots_blocked,+ bot_challenge,other,};@@ -959,6 +960,7 @@ pub fn errorReason(err: anyerror) ErrorReason {error.SyncWaitInterrupted,=> .aborted,error.RobotsBlocked => .robots_blocked,+ error.BotChallenge => .bot_challenge,else => .other,};}