Skip to content

Lightpanda: the headless browser designed for AI and automation

better handling of allowed and disallowed robots cases

Muki Kiboigo7cb659064257
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index cc3cf949e..02c72c588 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -92,7 +92,8 @@ pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
pub const RobotStore = struct {
const RobotsEntry = union(enum) {
present: Robots,
- absent,
+ allowed,
+ disallowed,
};
pub const RobotsMap = std.HashMapUnmanaged([]const u8, RobotsEntry, struct {
@@ -141,7 +142,7 @@ pub const RobotStore = struct {
switch (entry.value_ptr.*) {
.present => |*robots| robots.deinit(self.allocator),
- .absent => {},
+ .allowed, .disallowed => {},
}
}
@@ -175,16 +176,26 @@ pub const RobotStore = struct {
const entry = self.map.get(url) orelse return null;
return switch (entry) {
.present => |robots| robots.content_signals,
- .absent => null,
+ .allowed, .disallowed => null,
};
}
- pub fn putAbsent(self: *RobotStore, url: []const u8) !void {
+ /// This URL has no restrictions on crawling.
+ pub fn putAllowed(self: *RobotStore, url: []const u8) !void {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
const duped = try self.allocator.dupe(u8, url);
- try self.map.put(self.allocator, duped, .absent);
+ try self.map.put(self.allocator, duped, .allowed);
+ }
+
+ /// This URL is fully restricted from crawling.
+ pub fn putDisallowed(self: *RobotStore, url: []const u8) !void {
+ self.mutex.lockUncancelable(lp.io);
+ defer self.mutex.unlock(lp.io);
+
+ const duped = try self.allocator.dupe(u8, url);
+ try self.map.put(self.allocator, duped, .disallowed);
}
};
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index 0c91191e8..fd787d53f 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -53,7 +53,11 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
if (self.network.robot_store.get(robots_url)) |robot_entry| {
switch (robot_entry) {
- .absent => return .allowed,
+ .allowed => return .allowed,
+ .disallowed => {
+ log.warn(.http, "blocked by robots", .{ .url = url });
+ return .blocked;
+ },
.present => |robots| {
if (robots.isAllowed(URL.getPathname(url))) {
return .allowed;
@@ -142,7 +146,8 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
transfer.unpark();
const allowed = if (robot_entry) |entry| switch (entry) {
- .absent => true,
+ .allowed => true,
+ .disallowed => false,
.present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),
} else true;
@@ -201,26 +206,41 @@ const RobotsContext = struct {
const robots: ?Robots = network.robot_store.robotsFromBytes(
network.config.http_headers.user_agent,
self.buffer.items,
- ) catch blk: {
- log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
- try network.robot_store.putAbsent(robots_url);
+ ) catch |err| blk: {
+ // We only return an error if an allocation or something fails.
+ // Our parser does already leniently handle malformed input and takes whichever rules it can parse.
+ // On this case of an allocation failure, it is our fault so we put it as disallowed.
+ log.warn(.browser, "error while parsing robots.txt", .{ .robots_url = robots_url, .err = err });
+ try network.robot_store.putDisallowed(robots_url);
break :blk null;
};
if (robots) |r| {
try network.robot_store.put(robots_url, r);
}
+ } else {
+ // Empty robots.txt means we can short-circuit the allowed path.
+ try network.robot_store.putAllowed(robots_url);
}
},
- 404 => {
- log.debug(.http, "robots not found", .{ .url = robots_url });
- try network.robot_store.putAbsent(robots_url);
+ // RFC9309: Unavailable (400-499) means that we may access any resources on the server.
+ 400...499 => {
+ log.debug(.http, "robots.txt unavailable", .{ .url = robots_url });
+ try network.robot_store.putAllowed(robots_url);
+ },
+ // RFC9309: Unreachable (500-599) means that we are completely disallowed.
+ 500...599 => {
+ log.warn(.http, "robots.txt unreachable", .{
+ .url = robots_url,
+ .status = self.status,
+ });
+ try network.robot_store.putDisallowed(robots_url);
},
else => {
log.debug(.http, "unexpected status on robots", .{
.url = robots_url,
.status = self.status,
});
- try network.robot_store.putAbsent(robots_url);
+ try network.robot_store.putDisallowed(robots_url);
},
}