Lightpanda: the headless browser designed for AI and automation
add an eviction system for RobotsStore
Muki Kiboigo9510e1361cfe
diff --git a/src/network/EvictionQueue.zig b/src/network/EvictionQueue.zignew file mode 100644index 000000000..a4fe35d78--- /dev/null+++ b/src/network/EvictionQueue.zig@@ -0,0 +1,99 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");+const Allocator = std.mem.Allocator;++pub fn EvictionQueue(comptime K: type) type {+ return struct {+ const Self = @This();++ const Item = struct {+ seq: u64,+ key: K,+ };++ fn olderFirst(_: void, a: Item, b: Item) std.math.Order {+ return std.math.order(a.seq, b.seq);+ }++ const Heap = std.PriorityQueue(Item, void, olderFirst);++ allocator: Allocator,+ capacity: usize,+ next_seq: u64 = 0,+ heap: Heap = .empty,++ pub fn init(allocator: Allocator, capacity: usize) Self {+ return .{ .allocator = allocator, .capacity = capacity };+ }++ pub fn deinit(self: *Self) void {+ self.heap.deinit(self.allocator);+ }++ pub fn count(self: *const Self) usize {+ return self.heap.count();+ }++ // Record a newly-inserted key. If this pushes the queue over+ // capacity, returns the oldest key so the caller can evict it from+ // its own map. Callers must only call this once per new key -- it+ // does not check for duplicates.+ pub fn insert(self: *Self, key: K) !?K {+ const seq = self.next_seq;+ self.next_seq += 1;++ try self.heap.push(self.allocator, .{ .seq = seq, .key = key });+ if (self.heap.count() <= self.capacity) {+ return null;+ }+ return self.heap.pop().?.key;+ }+ };+}++const testing = @import("../testing.zig");++test "EvictionQueue: no eviction under capacity" {+ var q = EvictionQueue([]const u8).init(testing.allocator, 3);+ defer q.deinit();++ try testing.expectEqual(null, try q.insert("a"));+ try testing.expectEqual(null, try q.insert("b"));+ try testing.expectEqual(null, try q.insert("c"));+ try testing.expectEqual(3, q.count());+}++test "EvictionQueue: evicts oldest key once over capacity" {+ var q = EvictionQueue([]const u8).init(testing.allocator, 2);+ defer q.deinit();++ try testing.expectEqual(null, try q.insert("a"));+ try testing.expectEqual(null, try q.insert("b"));++ const evicted = try q.insert("c");+ try testing.expect(evicted != null);+ try testing.expectString("a", evicted.?);+ try testing.expectEqual(2, q.count());++ const evicted2 = try q.insert("d");+ try testing.expect(evicted2 != null);+ try testing.expectString("b", evicted2.?);+ try testing.expectEqual(2, q.count());+}diff --git a/src/network/Robots.zig b/src/network/Robots.zigindex 5ade56263..86c3675f6 100644--- a/src/network/Robots.zig+++ b/src/network/Robots.zig@@ -19,6 +19,8 @@const std = @import("std");const lp = @import("lightpanda");+const EvictionQueue = @import("EvictionQueue.zig").EvictionQueue;+const log = lp.log;const CompiledPattern = struct {@@ -86,10 +88,9 @@ pub const ContentSignal = struct {pub const Robots = @This();pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };-// Think twice before deleting/freeing any entries from the map. Readers, e.g.-// get and getContentSignals, receive values from the map, and if another thread-// was to delete / free those values while in use, UAF.pub const RobotStore = struct {+ pub const DEFAULT_CAPACITY = 1_000;+const RobotsEntry = union(enum) {present: Robots,allowed,@@ -100,10 +101,19 @@ pub const RobotStore = struct {allocator: std.mem.Allocator,map: RobotsMap,+ evictions: EvictionQueue([]const u8),mutex: std.Io.Mutex = .init,pub fn init(allocator: std.mem.Allocator) RobotStore {- return .{ .allocator = allocator, .map = .empty };+ return .initCapacity(allocator, DEFAULT_CAPACITY);+ }++ pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) RobotStore {+ return .{+ .allocator = allocator,+ .map = .empty,+ .evictions = .init(allocator, capacity),+ };}pub fn deinit(self: *RobotStore) void {@@ -114,21 +124,32 @@ pub const RobotStore = struct {while (iter.next()) |entry| {self.allocator.free(entry.key_ptr.*);-- switch (entry.value_ptr.*) {- .present => |*robots| robots.deinit(self.allocator),- .allowed, .disallowed => {},- }+ self.freeEntry(entry.value_ptr);}self.map.deinit(self.allocator);+ self.evictions.deinit();+ }++ fn freeEntry(self: *RobotStore, entry: *RobotsEntry) void {+ switch (entry.*) {+ .present => |*robots| robots.deinit(self.allocator),+ .allowed, .disallowed => {},+ }}- pub fn get(self: *RobotStore, url: []const u8) ?RobotsEntry {+ pub const Decision = enum { allowed, blocked };++ pub fn checkPath(self: *RobotStore, url: []const u8, path: []const u8) ?Decision {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);- return self.map.get(url);+ const entry = self.map.get(url) orelse return null;+ return switch (entry) {+ .allowed => .allowed,+ .disallowed => .blocked,+ .present => |robots| if (robots.isAllowed(path)) .allowed else .blocked,+ };}pub fn robotsFromBytes(self: *RobotStore, user_agent: []const u8, bytes: []const u8) !Robots {@@ -152,16 +173,28 @@ pub const RobotStore = struct {discarded.deinit(self.allocator);}- // The returned slice is owned by the store- pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal {+ pub fn getContentSignals(+ self: *RobotStore,+ allocator: std.mem.Allocator,+ url: []const u8,+ ) !?[]const ContentSignal {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);const entry = self.map.get(url) orelse return null;- return switch (entry) {+ const signals = switch (entry) {.present => |robots| robots.content_signals,- .allowed, .disallowed => null,+ .allowed, .disallowed => return null,};++ const out = try allocator.alloc(ContentSignal, signals.len);+ for (signals, 0..) |signal, i| {+ out[i] = .{+ .name = try allocator.dupe(u8, signal.name),+ .value = try allocator.dupe(u8, signal.value),+ };+ }+ return out;}/// This URL has no restrictions on crawling.@@ -201,6 +234,15 @@ pub const RobotStore = struct {}errdefer _ = self.map.remove(url);gop.key_ptr.* = try self.allocator.dupe(u8, url);++ if (try self.evictions.insert(gop.key_ptr.*)) |evict_key| {+ if (self.map.fetchRemove(evict_key)) |kv| {+ self.allocator.free(kv.key);+ var entry = kv.value;+ self.freeEntry(&entry);+ }+ }+return gop.value_ptr;}};@@ -1257,13 +1299,20 @@ test "Robots: RobotStore.getContentSignals round-trips" {);try store.put("https://example.com/robots.txt", robots);- const signals = store.getContentSignals("https://example.com/robots.txt").?;+ const signals = (try store.getContentSignals(allocator, "https://example.com/robots.txt")).?;+ defer {+ for (signals) |signal| {+ allocator.free(signal.name);+ allocator.free(signal.value);+ }+ allocator.free(signals);+ }try std.testing.expectEqual(1, signals.len);try std.testing.expectEqualStrings("ai-train", signals[0].name);try std.testing.expectEqualStrings("no", signals[0].value);// Unknown host has no stored robots.- try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt"));+ try std.testing.expectEqual(null, try store.getContentSignals(allocator, "https://other.com/robots.txt"));}fn testMatch(pattern: []const u8, path: []const u8) bool {diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex ace25406c..9507c1165 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -50,17 +50,10 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {const url = transfer.req.url;const robots_url = try URL.getRobotsUrl(transfer.arena.allocator(), url);- if (self.network.robot_store.get(robots_url)) |robot_entry| {- switch (robot_entry) {+ if (self.network.robot_store.checkPath(robots_url, URL.getPathname(url))) |decision| {+ switch (decision) {.allowed => return .allowed,- .disallowed => {- log.warn(.http, "blocked by robots", .{ .url = url });- return .blocked;- },- .present => |robots| {- if (robots.isAllowed(URL.getPathname(url))) {- return .allowed;- }+ .blocked => {log.warn(.http, "blocked by robots", .{ .url = url });return .blocked;},@@ -141,17 +134,11 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {var queued = self.single_flight.take(robots_url) orelse return;defer queued.deinit(self.single_flight.allocator);- const robot_entry = self.network.robot_store.get(robots_url);for (queued.items) |transfer| {transfer.unpark();- const allowed = if (robot_entry) |entry| switch (entry) {- .allowed => true,- .disallowed => false,- .present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),- } else true;-- if (!allowed) {+ const decision = self.network.robot_store.checkPath(robots_url, URL.getPathname(transfer.req.url));+ if (decision == .blocked) {lp.metrics.robots_access.incr(.deny);log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });transfer.failAsync(error.RobotsBlocked);diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zigindex 9f9e15402..ea554b832 100644--- a/src/server/cdp/domains/lp.zig+++ b/src/server/cdp/domains/lp.zig@@ -306,7 +306,7 @@ fn getContentSignal(cmd: anytype) !void {return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{});};- const signals = network.robot_store.getContentSignals(robots_url);+ const signals = try network.robot_store.getContentSignals(cmd.arena, robots_url);return cmd.sendResult(.{.available = signals != null,.contentSignals = signals orelse empty,