Lightpanda: the headless browser designed for AI and automation
better handling of allowed and disallowed robots cases
Muki Kiboigo7cb659064257
diff --git a/src/network/Robots.zig b/src/network/Robots.zigindex cc3cf949e..02c72c588 100644--- a/src/network/Robots.zig+++ b/src/network/Robots.zig@@ -92,7 +92,8 @@ pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };pub const RobotStore = struct {const RobotsEntry = union(enum) {present: Robots,- absent,+ allowed,+ disallowed,};pub const RobotsMap = std.HashMapUnmanaged([]const u8, RobotsEntry, struct {@@ -141,7 +142,7 @@ pub const RobotStore = struct {switch (entry.value_ptr.*) {.present => |*robots| robots.deinit(self.allocator),- .absent => {},+ .allowed, .disallowed => {},}}@@ -175,16 +176,26 @@ pub const RobotStore = struct {const entry = self.map.get(url) orelse return null;return switch (entry) {.present => |robots| robots.content_signals,- .absent => null,+ .allowed, .disallowed => null,};}- pub fn putAbsent(self: *RobotStore, url: []const u8) !void {+ /// This URL has no restrictions on crawling.+ pub fn putAllowed(self: *RobotStore, url: []const u8) !void {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);const duped = try self.allocator.dupe(u8, url);- try self.map.put(self.allocator, duped, .absent);+ try self.map.put(self.allocator, duped, .allowed);+ }++ /// This URL is fully restricted from crawling.+ pub fn putDisallowed(self: *RobotStore, url: []const u8) !void {+ self.mutex.lockUncancelable(lp.io);+ defer self.mutex.unlock(lp.io);++ const duped = try self.allocator.dupe(u8, url);+ try self.map.put(self.allocator, duped, .disallowed);}};diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex 0c91191e8..fd787d53f 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -53,7 +53,11 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {if (self.network.robot_store.get(robots_url)) |robot_entry| {switch (robot_entry) {- .absent => return .allowed,+ .allowed => return .allowed,+ .disallowed => {+ log.warn(.http, "blocked by robots", .{ .url = url });+ return .blocked;+ },.present => |robots| {if (robots.isAllowed(URL.getPathname(url))) {return .allowed;@@ -142,7 +146,8 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {transfer.unpark();const allowed = if (robot_entry) |entry| switch (entry) {- .absent => true,+ .allowed => true,+ .disallowed => false,.present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),} else true;@@ -201,26 +206,41 @@ const RobotsContext = struct {const robots: ?Robots = network.robot_store.robotsFromBytes(network.config.http_headers.user_agent,self.buffer.items,- ) catch blk: {- log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });- try network.robot_store.putAbsent(robots_url);+ ) catch |err| blk: {+ // We only return an error if an allocation or something fails.+ // Our parser does already leniently handle malformed input and takes whichever rules it can parse.+ // On this case of an allocation failure, it is our fault so we put it as disallowed.+ log.warn(.browser, "error while parsing robots.txt", .{ .robots_url = robots_url, .err = err });+ try network.robot_store.putDisallowed(robots_url);break :blk null;};if (robots) |r| {try network.robot_store.put(robots_url, r);}+ } else {+ // Empty robots.txt means we can short-circuit the allowed path.+ try network.robot_store.putAllowed(robots_url);}},- 404 => {- log.debug(.http, "robots not found", .{ .url = robots_url });- try network.robot_store.putAbsent(robots_url);+ // RFC9309: Unavailable (400-499) means that we may access any resources on the server.+ 400...499 => {+ log.debug(.http, "robots.txt unavailable", .{ .url = robots_url });+ try network.robot_store.putAllowed(robots_url);+ },+ // RFC9309: Unreachable (500-599) means that we are completely disallowed.+ 500...599 => {+ log.warn(.http, "robots.txt unreachable", .{+ .url = robots_url,+ .status = self.status,+ });+ try network.robot_store.putDisallowed(robots_url);},else => {log.debug(.http, "unexpected status on robots", .{.url = robots_url,.status = self.status,});- try network.robot_store.putAbsent(robots_url);+ try network.robot_store.putDisallowed(robots_url);},}