Skip to content

Lightpanda: the headless browser designed for AI and automation

robots: limit robot cache to 24 hours

Karl Seguinec4b27533216
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index f303db173..4eb41068f 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -89,10 +89,18 @@ pub const Robots = @This();
pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
pub const RobotStore = struct {
- const RobotsEntry = union(enum) {
- present: Robots,
- allowed,
- disallowed,
+ //a cached robots.txt SHOULD NOT be used for more than 24h.
+ const max_age_s = 24 * 60 * 60;
+
+ const RobotsEntry = struct {
+ result: Result,
+ expires_at: u64,
+
+ const Result = union(enum) {
+ present: Robots,
+ allowed,
+ disallowed,
+ };
};
allocator: std.mem.Allocator,
@@ -117,7 +125,7 @@ pub const RobotStore = struct {
}
fn freeEntry(self: *RobotStore, entry: *RobotsEntry) void {
- switch (entry.*) {
+ switch (entry.result) {
.present => |*robots| robots.deinit(self.allocator),
.allowed, .disallowed => {},
}
@@ -129,8 +137,8 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const entry = self.map.get(url) orelse return null;
- return switch (entry.*) {
+ const entry = self.getLive(url) orelse return null;
+ return switch (entry.result) {
.allowed => .allowed,
.disallowed => .blocked,
.present => |robots| if (robots.isAllowed(path)) .allowed else .blocked,
@@ -159,8 +167,8 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const entry = self.map.get(url) orelse return null;
- const signals = switch (entry.*) {
+ const entry = self.getLive(url) orelse return null;
+ const signals = switch (entry.result) {
.present => |robots| robots.content_signals,
.allowed, .disallowed => return null,
};
@@ -189,7 +197,22 @@ pub const RobotStore = struct {
_ = try self.insert(url, .disallowed);
}
- fn insert(self: *RobotStore, url: []const u8, entry: RobotsEntry) !bool {
+ // Caller holds the mutex.
+ fn getLive(self: *RobotStore, url: []const u8) ?*RobotsEntry {
+ const entry = self.map.get(url) orelse return null;
+ if (entry.expires_at > lp.datetime.timestamp(.real)) {
+ return entry;
+ }
+ var expired = self.map.remove(url).?;
+ self.freeEntry(&expired);
+ return null;
+ }
+
+ fn insert(self: *RobotStore, url: []const u8, result: RobotsEntry.Result) !bool {
+ const entry: RobotsEntry = .{
+ .result = result,
+ .expires_at = lp.datetime.timestamp(.real) + max_age_s,
+ };
switch (try self.map.insert(url, entry)) {
.exists => return false,
.inserted => |evicted| {
@@ -1272,6 +1295,29 @@ test "Robots: RobotStore.getContentSignals round-trips" {
try std.testing.expectEqual(null, try store.getContentSignals(allocator, "https://other.com/robots.txt"));
}
+test "Robots: RobotStore drops expired entries" {
+ const allocator = std.testing.allocator;
+
+ var store = RobotStore.init(allocator, 1000);
+ defer store.deinit();
+
+ const url = "https://example.com/robots.txt";
+ const robots = try store.robotsFromBytes("MyBot",
+ \\User-agent: *
+ \\Disallow: /private
+ \\
+ );
+ try store.put(url, robots);
+ try std.testing.expectEqual(.blocked, store.checkPath(url, "/private").?);
+
+ store.map.get(url).?.expires_at = lp.datetime.timestamp(.real) - 1;
+ try std.testing.expectEqual(null, store.checkPath(url, "/private"));
+
+ // The refetch's result replaces it.
+ try store.putAllowed(url);
+ try std.testing.expectEqual(.allowed, store.checkPath(url, "/private").?);
+}
+
fn testMatch(pattern: []const u8, path: []const u8) bool {
comptime if (!lp.IS_TEST) unreachable;