Lightpanda: the headless browser designed for AI and automation
add robot store entry limit option
Muki Kiboigo9fa9e5ae0d80
diff --git a/src/Config.zig b/src/Config.zigindex 7eb5bf809..c15d51314 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -253,6 +253,7 @@ pub const ExperimentalFeatures = packed struct(u2) {/// Common CLI args.const CommonOptions = .{.{ .name = "obey_robots", .type = bool },+ .{ .name = "robot_store_entry_limit", .type = ?u32, .default = 1000 },.{ .name = "proxy_bearer_token", .type = ?[:0]const u8 },.{ .name = "http_proxy", .type = ?[:0]const u8 },.{ .name = "http_max_concurrent", .type = ?u8 },@@ -563,6 +564,13 @@ pub fn obeyRobots(self: *const Config) bool {};}+pub fn robotStoreEntryLimit(self: *const Config) u32 {+ return switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.robot_store_entry_limit.?,+ else => 1000,+ };+}+pub fn httpVersion(self: *const Config) HttpVersion {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.http_version,diff --git a/src/help.zon b/src/help.zonindex c6aba3f2a..576741170 100644--- a/src/help.zon+++ b/src/help.zon@@ -489,6 +489,9 @@\\ --obey-robots\\ Fetches and obeys robots.txt of the target page.\\ Defaults to false.+ \\ --robot-store-entry-limit <INT>+ \\ Maximum number of entries kept in the RobotStore.+ \\ Defaults to 1000.\\ --proxy-bearer-token <TOKEN>\\ Token sent for bearer authentication with the proxy:\\ Proxy-Authorization: Bearer <token>.diff --git a/src/network/Network.zig b/src/network/Network.zigindex 71ba1fe3f..f6593c008 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -121,7 +121,7 @@ pub fn init(app: *App) !Network {.connections = connections,.cache = cache,- .robot_store = RobotStore.init(allocator),+ .robot_store = RobotStore.init(allocator, config.robotStoreEntryLimit()),.web_bot_auth = web_bot_auth,.rate_limiter = if (config.httpNavDelay()) |ms| RateLimiter.init(allocator, ms, config.httpNavBurst()) else null,.adblocker = adblocker,diff --git a/src/network/Robots.zig b/src/network/Robots.zigindex 86c3675f6..d9990a4b4 100644--- a/src/network/Robots.zig+++ b/src/network/Robots.zig@@ -89,8 +89,6 @@ pub const Robots = @This();pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };pub const RobotStore = struct {- pub const DEFAULT_CAPACITY = 1_000;-const RobotsEntry = union(enum) {present: Robots,allowed,@@ -104,8 +102,8 @@ pub const RobotStore = struct {evictions: EvictionQueue([]const u8),mutex: std.Io.Mutex = .init,- pub fn init(allocator: std.mem.Allocator) RobotStore {- return .initCapacity(allocator, DEFAULT_CAPACITY);+ pub fn init(allocator: std.mem.Allocator, capacity: u32) RobotStore {+ return .initCapacity(allocator, capacity);}pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) RobotStore {@@ -1289,7 +1287,7 @@ test "Robots: content-signal prefers specific user-agent over wildcard" {test "Robots: RobotStore.getContentSignals round-trips" {const allocator = std.testing.allocator;- var store = RobotStore.init(allocator);+ var store = RobotStore.init(allocator, 1000);defer store.deinit();const robots = try store.robotsFromBytes("MyBot",