Skip to content

Lightpanda: the headless browser designed for AI and automation

ops: attempt to generate an appropriate error on a bot challenge

Karl Seguin64dbff13c596
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 2b7ee4509..b2f1d4ce2 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -304,6 +304,7 @@ _type: enum { root, frame }, // only used for logs right now
_req_id: u32 = 0,
_navigated_options: ?NavigatedOpts = null,
_http_status: ?u16 = null,
+_bot_challenge: ?HttpClient.BotChallenge = null,
_http_headers: std.ArrayList(HttpHeader) = .empty,
_referrer: ?[]const u8 = null,
@@ -816,6 +817,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
}
self._http_status = null;
+ self._bot_challenge = null;
self._http_headers = .empty;
self._referrer = null;
@@ -1515,6 +1517,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.
}
self._http_status = transfer.responseStatus();
+ self._bot_challenge = transfer.botChallenge();
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try self._http_headers.append(self.arena, .{
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index e2e58524a..68a13942c 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -350,6 +350,16 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
if (err.* == null) {
err.* = frame._last_navigate_error;
}
+ if (err.* == null) {
+ if (frame._bot_challenge) |challenge| {
+ log.warn(.app, "bot challenge", .{
+ .url = frame.url,
+ .status = frame._http_status,
+ .provider = challenge,
+ });
+ err.* = error.BotChallenge;
+ }
+ }
if (frame._http_status) |status| {
if (status >= 400) {
http_error = true;
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 2665925e1..2f6f7e92d 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -4098,7 +4098,7 @@ pub const Transfer = struct {
}
pub fn getContentLength(self: *const Transfer) ?usize {
- const cl = self.getContentLengthRawValue() orelse return null;
+ const cl = self.responseHeader("content-length") orelse return null;
return std.fmt.parseInt(usize, cl, 10) catch null;
}
@@ -4110,23 +4110,50 @@ pub const Transfer = struct {
return self._body_len;
}
- fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {
+ // First value of a response header. `name` must be lowercase.
+ pub fn responseHeader(self: *const Transfer, name: [:0]const u8) ?[]const u8 {
// Materialized headers (dispatch time, any source).
for (self.res.headers) |hdr| {
- if (std.mem.eql(u8, hdr.name, "content-length")) {
+ if (std.mem.eql(u8, hdr.name, name)) {
return hdr.value;
}
}
// Mid-stream (curl's write callback): read from the live conn.
if (self._conn) |c| {
- const cl = c.getResponseHeader("content-length", 0) orelse return null;
- return cl.value;
+ const value = c.getResponseHeader(name, 0) orelse return null;
+ return value.value;
}
return null;
}
+ pub fn botChallenge(self: *const Transfer) ?BotChallenge {
+ const status = self.responseStatus() orelse return null;
+ switch (status) {
+ 403 => {
+ const value = self.responseHeader("cf-mitigated") orelse return null;
+ if (std.ascii.eqlIgnoreCase(value, "challenge")) {
+ return .cloudflare;
+ }
+ },
+ 429 => {
+ const value = self.responseHeader("x-vercel-mitigated") orelse return null;
+ if (std.ascii.eqlIgnoreCase(value, "challenge")) {
+ return .vercel;
+ }
+ },
+ 202, 405 => {
+ const action = self.responseHeader("x-amzn-waf-action") orelse return null;
+ if (std.ascii.eqlIgnoreCase(action, "captcha") or std.ascii.eqlIgnoreCase(action, "challenge")) {
+ return .aws_waf;
+ }
+ },
+ else => {},
+ }
+ return null;
+ }
+
// Only ever called from tick, delivers buffered events to their consumer.
// A buffered (completed) transfer gets exactly one deliver() and dies at
// the end of it. A streaming transfer gets one deliver() per dispatched
@@ -4441,6 +4468,12 @@ const Synthetic = struct {
}
};
+pub const BotChallenge = enum {
+ aws_waf,
+ cloudflare,
+ vercel,
+};
+
const testing = @import("../testing.zig");
// Only the transfer list, the url and the parent matter to the tests using
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index 1ffe9a1e9..29f6a66c4 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -30,10 +30,11 @@ const ArenaPool = @import("../ArenaPool.zig");
const http = @import("http.zig");
const Robots = @import("Robots.zig");
const Network = @import("Network.zig");
-const Transfer = @import("HttpClient.zig").Transfer;
+const HttpClient = @import("HttpClient.zig");
const SingleFlight = @import("SingleFlight.zig");
const log = lp.log;
+const Transfer = HttpClient.Transfer;
const RobotsGate = @This();
@@ -129,6 +130,7 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans
const Outcome = union(enum) {
decision: Robots.RobotStore.Decision,
robots: Robots.Robots,
+ challenged,
};
// The robots.txt fetch resolved: hand every waiter back to the pipeline,
@@ -144,6 +146,11 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8, outcome: Outcome) voi
const decision: Robots.RobotStore.Decision = switch (outcome) {
.decision => |d| d,
.robots => |r| if (r.isAllowed(URL.getPathname(transfer.req.url))) .allowed else .blocked,
+ .challenged => {
+ lp.metrics.robots_access.incr(.deny);
+ transfer.failAsync(error.BotChallenge);
+ continue;
+ },
};
if (decision == .blocked) {
@@ -168,6 +175,7 @@ const RobotsContext = struct {
robots_url: [:0]const u8,
buffer: std.ArrayList(u8),
status: u16 = 0,
+ challenge: ?HttpClient.BotChallenge = null,
fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {
const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));
@@ -175,6 +183,7 @@ const RobotsContext = struct {
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
self.status = hdr.status;
}
+ self.challenge = transfer.botChallenge();
lp.metrics.robots_status.incr(http.statusCategory(self.status));
try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), transfer.bodyLen());
return .proceed;
@@ -192,6 +201,18 @@ const RobotsContext = struct {
const robots_url = self.robots_url;
const network = self.gate.network;
+ if (self.challenge) |challenge| {
+ // Nothing here can solve it, but the challenge might be gone by
+ // the next request, so don't cache it.
+ log.warn(.http, "robots.txt bot challenge", .{
+ .url = robots_url,
+ .status = self.status,
+ .provider = challenge,
+ });
+ self.settle(.{ .outcome = .challenged, .cache = false });
+ return;
+ }
+
switch (self.status) {
200 => {
if (self.buffer.items.len == 0) {
@@ -293,6 +314,7 @@ const RobotsContext = struct {
.robots => |r| network.robot_store.put(self.robots_url, r) catch |err| {
log.warn(.browser, "cache robots rules", .{ .url = self.robots_url, .err = err });
},
+ .challenged => {},
}
}
}
diff --git a/src/network/http.zig b/src/network/http.zig
index ab34bff33..7bd5486f6 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -931,6 +931,7 @@ pub const ErrorReason = enum {
too_large,
aborted,
robots_blocked,
+ bot_challenge,
other,
};
@@ -959,6 +960,7 @@ pub fn errorReason(err: anyerror) ErrorReason {
error.SyncWaitInterrupted,
=> .aborted,
error.RobotsBlocked => .robots_blocked,
+ error.BotChallenge => .bot_challenge,
else => .other,
};
}