Skip to content

Lightpanda: the headless browser designed for AI and automation

add robot store entry limit option

Muki Kiboigo9fa9e5ae0d80
diff --git a/src/Config.zig b/src/Config.zig
index 7eb5bf809..c15d51314 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -253,6 +253,7 @@ pub const ExperimentalFeatures = packed struct(u2) {
/// Common CLI args.
const CommonOptions = .{
.{ .name = "obey_robots", .type = bool },
+ .{ .name = "robot_store_entry_limit", .type = ?u32, .default = 1000 },
.{ .name = "proxy_bearer_token", .type = ?[:0]const u8 },
.{ .name = "http_proxy", .type = ?[:0]const u8 },
.{ .name = "http_max_concurrent", .type = ?u8 },
@@ -563,6 +564,13 @@ pub fn obeyRobots(self: *const Config) bool {
};
}
+pub fn robotStoreEntryLimit(self: *const Config) u32 {
+ return switch (self.mode) {
+ inline .serve, .fetch, .mcp, .agent => |opts| opts.robot_store_entry_limit.?,
+ else => 1000,
+ };
+}
+
pub fn httpVersion(self: *const Config) HttpVersion {
return switch (self.mode) {
inline .serve, .fetch, .mcp, .agent => |opts| opts.http_version,
diff --git a/src/help.zon b/src/help.zon
index c6aba3f2a..576741170 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -489,6 +489,9 @@
\\ --obey-robots
\\ Fetches and obeys robots.txt of the target page.
\\ Defaults to false.
+ \\ --robot-store-entry-limit <INT>
+ \\ Maximum number of entries kept in the RobotStore.
+ \\ Defaults to 1000.
\\ --proxy-bearer-token <TOKEN>
\\ Token sent for bearer authentication with the proxy:
\\ Proxy-Authorization: Bearer <token>.
diff --git a/src/network/Network.zig b/src/network/Network.zig
index 71ba1fe3f..f6593c008 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -121,7 +121,7 @@ pub fn init(app: *App) !Network {
.connections = connections,
.cache = cache,
- .robot_store = RobotStore.init(allocator),
+ .robot_store = RobotStore.init(allocator, config.robotStoreEntryLimit()),
.web_bot_auth = web_bot_auth,
.rate_limiter = if (config.httpNavDelay()) |ms| RateLimiter.init(allocator, ms, config.httpNavBurst()) else null,
.adblocker = adblocker,
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index 86c3675f6..d9990a4b4 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -89,8 +89,6 @@ pub const Robots = @This();
pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
pub const RobotStore = struct {
- pub const DEFAULT_CAPACITY = 1_000;
-
const RobotsEntry = union(enum) {
present: Robots,
allowed,
@@ -104,8 +102,8 @@ pub const RobotStore = struct {
evictions: EvictionQueue([]const u8),
mutex: std.Io.Mutex = .init,
- pub fn init(allocator: std.mem.Allocator) RobotStore {
- return .initCapacity(allocator, DEFAULT_CAPACITY);
+ pub fn init(allocator: std.mem.Allocator, capacity: u32) RobotStore {
+ return .initCapacity(allocator, capacity);
}
pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) RobotStore {
@@ -1289,7 +1287,7 @@ test "Robots: content-signal prefers specific user-agent over wildcard" {
test "Robots: RobotStore.getContentSignals round-trips" {
const allocator = std.testing.allocator;
- var store = RobotStore.init(allocator);
+ var store = RobotStore.init(allocator, 1000);
defer store.deinit();
const robots = try store.robotsFromBytes("MyBot",