Skip to content

Lightpanda: the headless browser designed for AI and automation

add an eviction system for RobotsStore

Muki Kiboigo9510e1361cfe
diff --git a/src/network/EvictionQueue.zig b/src/network/EvictionQueue.zig
new file mode 100644
index 000000000..a4fe35d78
--- /dev/null
+++ b/src/network/EvictionQueue.zig
@@ -0,0 +1,99 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+
+pub fn EvictionQueue(comptime K: type) type {
+ return struct {
+ const Self = @This();
+
+ const Item = struct {
+ seq: u64,
+ key: K,
+ };
+
+ fn olderFirst(_: void, a: Item, b: Item) std.math.Order {
+ return std.math.order(a.seq, b.seq);
+ }
+
+ const Heap = std.PriorityQueue(Item, void, olderFirst);
+
+ allocator: Allocator,
+ capacity: usize,
+ next_seq: u64 = 0,
+ heap: Heap = .empty,
+
+ pub fn init(allocator: Allocator, capacity: usize) Self {
+ return .{ .allocator = allocator, .capacity = capacity };
+ }
+
+ pub fn deinit(self: *Self) void {
+ self.heap.deinit(self.allocator);
+ }
+
+ pub fn count(self: *const Self) usize {
+ return self.heap.count();
+ }
+
+ // Record a newly-inserted key. If this pushes the queue over
+ // capacity, returns the oldest key so the caller can evict it from
+ // its own map. Callers must only call this once per new key -- it
+ // does not check for duplicates.
+ pub fn insert(self: *Self, key: K) !?K {
+ const seq = self.next_seq;
+ self.next_seq += 1;
+
+ try self.heap.push(self.allocator, .{ .seq = seq, .key = key });
+ if (self.heap.count() <= self.capacity) {
+ return null;
+ }
+ return self.heap.pop().?.key;
+ }
+ };
+}
+
+const testing = @import("../testing.zig");
+
+test "EvictionQueue: no eviction under capacity" {
+ var q = EvictionQueue([]const u8).init(testing.allocator, 3);
+ defer q.deinit();
+
+ try testing.expectEqual(null, try q.insert("a"));
+ try testing.expectEqual(null, try q.insert("b"));
+ try testing.expectEqual(null, try q.insert("c"));
+ try testing.expectEqual(3, q.count());
+}
+
+test "EvictionQueue: evicts oldest key once over capacity" {
+ var q = EvictionQueue([]const u8).init(testing.allocator, 2);
+ defer q.deinit();
+
+ try testing.expectEqual(null, try q.insert("a"));
+ try testing.expectEqual(null, try q.insert("b"));
+
+ const evicted = try q.insert("c");
+ try testing.expect(evicted != null);
+ try testing.expectString("a", evicted.?);
+ try testing.expectEqual(2, q.count());
+
+ const evicted2 = try q.insert("d");
+ try testing.expect(evicted2 != null);
+ try testing.expectString("b", evicted2.?);
+ try testing.expectEqual(2, q.count());
+}
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index 5ade56263..86c3675f6 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -19,6 +19,8 @@
const std = @import("std");
const lp = @import("lightpanda");
+const EvictionQueue = @import("EvictionQueue.zig").EvictionQueue;
+
const log = lp.log;
const CompiledPattern = struct {
@@ -86,10 +88,9 @@ pub const ContentSignal = struct {
pub const Robots = @This();
pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
-// Think twice before deleting/freeing any entries from the map. Readers, e.g.
-// get and getContentSignals, receive values from the map, and if another thread
-// was to delete / free those values while in use, UAF.
pub const RobotStore = struct {
+ pub const DEFAULT_CAPACITY = 1_000;
+
const RobotsEntry = union(enum) {
present: Robots,
allowed,
@@ -100,10 +101,19 @@ pub const RobotStore = struct {
allocator: std.mem.Allocator,
map: RobotsMap,
+ evictions: EvictionQueue([]const u8),
mutex: std.Io.Mutex = .init,
pub fn init(allocator: std.mem.Allocator) RobotStore {
- return .{ .allocator = allocator, .map = .empty };
+ return .initCapacity(allocator, DEFAULT_CAPACITY);
+ }
+
+ pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) RobotStore {
+ return .{
+ .allocator = allocator,
+ .map = .empty,
+ .evictions = .init(allocator, capacity),
+ };
}
pub fn deinit(self: *RobotStore) void {
@@ -114,21 +124,32 @@ pub const RobotStore = struct {
while (iter.next()) |entry| {
self.allocator.free(entry.key_ptr.*);
-
- switch (entry.value_ptr.*) {
- .present => |*robots| robots.deinit(self.allocator),
- .allowed, .disallowed => {},
- }
+ self.freeEntry(entry.value_ptr);
}
self.map.deinit(self.allocator);
+ self.evictions.deinit();
+ }
+
+ fn freeEntry(self: *RobotStore, entry: *RobotsEntry) void {
+ switch (entry.*) {
+ .present => |*robots| robots.deinit(self.allocator),
+ .allowed, .disallowed => {},
+ }
}
- pub fn get(self: *RobotStore, url: []const u8) ?RobotsEntry {
+ pub const Decision = enum { allowed, blocked };
+
+ pub fn checkPath(self: *RobotStore, url: []const u8, path: []const u8) ?Decision {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- return self.map.get(url);
+ const entry = self.map.get(url) orelse return null;
+ return switch (entry) {
+ .allowed => .allowed,
+ .disallowed => .blocked,
+ .present => |robots| if (robots.isAllowed(path)) .allowed else .blocked,
+ };
}
pub fn robotsFromBytes(self: *RobotStore, user_agent: []const u8, bytes: []const u8) !Robots {
@@ -152,16 +173,28 @@ pub const RobotStore = struct {
discarded.deinit(self.allocator);
}
- // The returned slice is owned by the store
- pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal {
+ pub fn getContentSignals(
+ self: *RobotStore,
+ allocator: std.mem.Allocator,
+ url: []const u8,
+ ) !?[]const ContentSignal {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
const entry = self.map.get(url) orelse return null;
- return switch (entry) {
+ const signals = switch (entry) {
.present => |robots| robots.content_signals,
- .allowed, .disallowed => null,
+ .allowed, .disallowed => return null,
};
+
+ const out = try allocator.alloc(ContentSignal, signals.len);
+ for (signals, 0..) |signal, i| {
+ out[i] = .{
+ .name = try allocator.dupe(u8, signal.name),
+ .value = try allocator.dupe(u8, signal.value),
+ };
+ }
+ return out;
}
/// This URL has no restrictions on crawling.
@@ -201,6 +234,15 @@ pub const RobotStore = struct {
}
errdefer _ = self.map.remove(url);
gop.key_ptr.* = try self.allocator.dupe(u8, url);
+
+ if (try self.evictions.insert(gop.key_ptr.*)) |evict_key| {
+ if (self.map.fetchRemove(evict_key)) |kv| {
+ self.allocator.free(kv.key);
+ var entry = kv.value;
+ self.freeEntry(&entry);
+ }
+ }
+
return gop.value_ptr;
}
};
@@ -1257,13 +1299,20 @@ test "Robots: RobotStore.getContentSignals round-trips" {
);
try store.put("https://example.com/robots.txt", robots);
- const signals = store.getContentSignals("https://example.com/robots.txt").?;
+ const signals = (try store.getContentSignals(allocator, "https://example.com/robots.txt")).?;
+ defer {
+ for (signals) |signal| {
+ allocator.free(signal.name);
+ allocator.free(signal.value);
+ }
+ allocator.free(signals);
+ }
try std.testing.expectEqual(1, signals.len);
try std.testing.expectEqualStrings("ai-train", signals[0].name);
try std.testing.expectEqualStrings("no", signals[0].value);
// Unknown host has no stored robots.
- try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt"));
+ try std.testing.expectEqual(null, try store.getContentSignals(allocator, "https://other.com/robots.txt"));
}
fn testMatch(pattern: []const u8, path: []const u8) bool {
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index ace25406c..9507c1165 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -50,17 +50,10 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
const url = transfer.req.url;
const robots_url = try URL.getRobotsUrl(transfer.arena.allocator(), url);
- if (self.network.robot_store.get(robots_url)) |robot_entry| {
- switch (robot_entry) {
+ if (self.network.robot_store.checkPath(robots_url, URL.getPathname(url))) |decision| {
+ switch (decision) {
.allowed => return .allowed,
- .disallowed => {
- log.warn(.http, "blocked by robots", .{ .url = url });
- return .blocked;
- },
- .present => |robots| {
- if (robots.isAllowed(URL.getPathname(url))) {
- return .allowed;
- }
+ .blocked => {
log.warn(.http, "blocked by robots", .{ .url = url });
return .blocked;
},
@@ -141,17 +134,11 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
var queued = self.single_flight.take(robots_url) orelse return;
defer queued.deinit(self.single_flight.allocator);
- const robot_entry = self.network.robot_store.get(robots_url);
for (queued.items) |transfer| {
transfer.unpark();
- const allowed = if (robot_entry) |entry| switch (entry) {
- .allowed => true,
- .disallowed => false,
- .present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),
- } else true;
-
- if (!allowed) {
+ const decision = self.network.robot_store.checkPath(robots_url, URL.getPathname(transfer.req.url));
+ if (decision == .blocked) {
lp.metrics.robots_access.incr(.deny);
log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });
transfer.failAsync(error.RobotsBlocked);
diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zig
index 9f9e15402..ea554b832 100644
--- a/src/server/cdp/domains/lp.zig
+++ b/src/server/cdp/domains/lp.zig
@@ -306,7 +306,7 @@ fn getContentSignal(cmd: anytype) !void {
return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{});
};
- const signals = network.robot_store.getContentSignals(robots_url);
+ const signals = try network.robot_store.getContentSignals(cmd.arena, robots_url);
return cmd.sendResult(.{
.available = signals != null,
.contentSignals = signals orelse empty,