Lightpanda: the headless browser designed for AI and automation
use ClockCache as the map in RobotStore
Muki Kiboigo74789d3af0ae
diff --git a/src/network/ClockCache.zig b/src/network/ClockCache.zigindex 86a0abf7a..84c41f310 100644--- a/src/network/ClockCache.zig+++ b/src/network/ClockCache.zig@@ -19,14 +19,23 @@const std = @import("std");const Allocator = std.mem.Allocator;-pub fn ClockCache(comptime K: type) type {+pub fn Entry(comptime V: type) type {+ return struct {+ value: V,+ referenced: bool,+ };+}++pub fn ClockCache(comptime V: type) type {return struct {const Self = @This();- const Map = if (K == []const u8)- std.array_hash_map.String(bool)- else- std.array_hash_map.Auto(K, bool);+ const Map = std.array_hash_map.String(Entry(V));++ pub const InsertResult = union(enum) {+ exists,+ inserted: ?V,+ };allocator: Allocator,capacity: usize,@@ -39,34 +48,50 @@ pub fn ClockCache(comptime K: type) type {}pub fn deinit(self: *Self) void {+ for (self.map.keys()) |key| {+ self.allocator.free(key);+ }self.map.deinit(self.allocator);}- pub fn insert(self: *Self, key: K) !?K {- try self.map.put(self.allocator, key, true);+ pub fn entries(self: *Self) []Entry(V) {+ return self.map.values();+ }- if (self.map.count() <= self.capacity) return null;- return self.evictOne();+ pub fn get(self: *Self, key: []const u8) ?*V {+ const entry = self.map.getPtr(key) orelse return null;+ entry.referenced = true;+ return &entry.value;}- pub fn touch(self: *Self, key: K) void {- if (self.map.getPtr(key)) |referenced| referenced.* = true;+ pub fn insert(self: *Self, key: []const u8, value: V) !InsertResult {+ const gop = try self.map.getOrPut(self.allocator, key);+ if (gop.found_existing) return .exists;++ errdefer self.map.swapRemoveAt(gop.index);+ gop.key_ptr.* = try self.allocator.dupe(u8, key);+ gop.value_ptr.* = .{ .value = value, .referenced = true };++ if (self.map.count() <= self.capacity) return .{ .inserted = null };+ return .{ .inserted = self.evictOne() };}- fn evictOne(self: *Self) ?K {- const referenced = self.map.values();+ fn evictOne(self: *Self) V {+ const items = self.map.values();while (true) {- if (self.hand >= referenced.len) self.hand = 0;+ if (self.hand >= items.len) self.hand = 0;- if (referenced[self.hand]) {- referenced[self.hand] = false;+ if (items[self.hand].referenced) {+ items[self.hand].referenced = false;self.hand += 1;continue;}const key = self.map.keys()[self.hand];+ const value = items[self.hand].value;self.map.swapRemoveAt(self.hand);- return key;+ self.allocator.free(key);+ return value;}}};@@ -74,46 +99,60 @@ pub fn ClockCache(comptime K: type) type {const testing = @import("../testing.zig");+fn evictedOf(comptime V: type, r: ClockCache(V).InsertResult) ?V {+ return switch (r) {+ .exists => unreachable,+ .inserted => |v| v,+ };+}+test "ClockCache: no eviction under capacity" {- var q = ClockCache([]const u8).init(testing.allocator, 3);+ var q = ClockCache(u32).init(testing.allocator, 3);defer q.deinit();- try testing.expectEqual(null, try q.insert("a"));- try testing.expectEqual(null, try q.insert("b"));- try testing.expectEqual(null, try q.insert("c"));+ try testing.expectEqual(null, evictedOf(u32, try q.insert("a", 1)));+ try testing.expectEqual(null, evictedOf(u32, try q.insert("b", 2)));+ try testing.expectEqual(null, evictedOf(u32, try q.insert("c", 3)));try testing.expectEqual(3, q.map.count());}test "ClockCache: evicts once over capacity" {- var q = ClockCache([]const u8).init(testing.allocator, 2);+ var q = ClockCache(u32).init(testing.allocator, 2);defer q.deinit();- try testing.expectEqual(null, try q.insert("a"));- try testing.expectEqual(null, try q.insert("b"));+ try testing.expectEqual(null, evictedOf(u32, try q.insert("a", 1)));+ try testing.expectEqual(null, evictedOf(u32, try q.insert("b", 2)));- const evicted = try q.insert("c");- try testing.expect(evicted != null);- try testing.expectString("a", evicted.?);+ const evicted = evictedOf(u32, try q.insert("c", 3));+ try testing.expectEqual(1, evicted.?);+ try testing.expect(q.get("a") == null);try testing.expectEqual(2, q.map.count());}test "ClockCache: touch protects a key from eviction" {- var q = ClockCache([]const u8).init(testing.allocator, 2);+ var q = ClockCache(u32).init(testing.allocator, 2);defer q.deinit();- try testing.expectEqual(null, try q.insert("a"));- try testing.expectEqual(null, try q.insert("b"));+ _ = try q.insert("a", 1);+ _ = try q.insert("b", 2);- const evicted1 = try q.insert("c");- try testing.expect(evicted1 != null);- try testing.expectString("a", evicted1.?);- try testing.expect(q.map.contains("b"));- try testing.expect(q.map.contains("c"));+ const evicted1 = evictedOf(u32, try q.insert("c", 3));+ try testing.expectEqual(1, evicted1.?);+ try testing.expect(q.get("b") != null);+ try testing.expect(q.get("c") != null);- q.touch("b");- const evicted2 = try q.insert("d");- try testing.expect(evicted2 != null);- try testing.expectString("c", evicted2.?);- try testing.expect(q.map.contains("b"));+ _ = q.get("b");+ const evicted2 = evictedOf(u32, try q.insert("d", 4));+ try testing.expectEqual(3, evicted2.?);+ try testing.expect(q.get("b") != null);try testing.expectEqual(2, q.map.count());}++test "ClockCache: insert does not overwrite" {+ var q = ClockCache(u32).init(testing.allocator, 2);+ defer q.deinit();++ _ = try q.insert("a", 1);+ try testing.expect((try q.insert("a", 99)) == .exists);+ try testing.expectEqual(1, q.get("a").?.*);+}diff --git a/src/network/Robots.zig b/src/network/Robots.zigindex f92cc9aba..33d0fcfba 100644--- a/src/network/Robots.zig+++ b/src/network/Robots.zig@@ -95,18 +95,14 @@ pub const RobotStore = struct {disallowed,};- const RobotsMap = @import("Network.zig").HostHashMap(RobotsEntry);-allocator: std.mem.Allocator,- map: RobotsMap,- evictions: ClockCache([]const u8),+ map: ClockCache(RobotsEntry),mutex: std.Io.Mutex = .init,pub fn init(allocator: std.mem.Allocator, capacity: u32) RobotStore {return .{.allocator = allocator,- .map = .empty,- .evictions = .init(allocator, capacity),+ .map = .init(allocator, capacity),};}@@ -114,15 +110,10 @@ pub const RobotStore = struct {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);- var iter = self.map.iterator();-- while (iter.next()) |entry| {- self.allocator.free(entry.key_ptr.*);- self.freeEntry(entry.value_ptr);+ for (self.map.entries()) |*entry| {+ self.freeEntry(&entry.value);}-- self.map.deinit(self.allocator);- self.evictions.deinit();+ self.map.deinit();}fn freeEntry(self: *RobotStore, entry: *RobotsEntry) void {@@ -138,10 +129,8 @@ pub const RobotStore = struct {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);- const kv = self.map.getEntry(url) orelse return null;- self.evictions.touch(kv.key_ptr.*);-- return switch (kv.value_ptr.*) {+ const entry = self.map.get(url) orelse return null;+ return switch (entry.*) {.allowed => .allowed,.disallowed => .blocked,.present => |robots| if (robots.isAllowed(path)) .allowed else .blocked,@@ -156,15 +145,10 @@ pub const RobotStore = struct {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);- if (try self.putKey(url)) |value_ptr| {- // first time seeing this url, store the value- value_ptr.* = .{ .present = robots };- return;- }+ if (try self.insert(url, .{ .present = robots })) return;// cannot overwrite an existing value, if it was `present`, we'd have// to free the value but it might be being used.-var discarded = robots;discarded.deinit(self.allocator);}@@ -178,7 +162,7 @@ pub const RobotStore = struct {defer self.mutex.unlock(lp.io);const entry = self.map.get(url) orelse return null;- const signals = switch (entry) {+ const signals = switch (entry.*) {.present => |robots| robots.content_signals,.allowed, .disallowed => return null,};@@ -197,50 +181,28 @@ pub const RobotStore = struct {pub fn putAllowed(self: *RobotStore, url: []const u8) !void {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);-- if (try self.putKey(url)) |value_ptr| {- // first time seeing this url, store the value- value_ptr.* = .allowed;- }- // cannot overwrite an existing value, if it was `present`, we'd have- // to free the value but it might be being used.+ _ = try self.insert(url, .allowed);}/// This URL is fully restricted from crawling.pub fn putDisallowed(self: *RobotStore, url: []const u8) !void {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);-- if (try self.putKey(url)) |value_ptr| {- // first time seeing this url, store the value- value_ptr.* = .disallowed;- }- // cannot overwrite an existing value, if it was `present`, we'd have- // to free the value but it might be being used.+ _ = try self.insert(url, .disallowed);}- // The RobotStore is shared across Browsers. Two rowsers can request the- // same robots URL at the same time, and they'll race here. First one wins.- // Caller holds the mutex.- fn putKey(self: *RobotStore, url: []const u8) !?*RobotsEntry {- const gop = try self.map.getOrPut(self.allocator, url);- if (gop.found_existing) {- // already have a value, caller should not overwrite- return null;- }- errdefer _ = self.map.remove(url);- gop.key_ptr.* = try self.allocator.dupe(u8, url);-- if (try self.evictions.insert(gop.key_ptr.*)) |evict_key| {- if (self.map.fetchRemove(evict_key)) |kv| {- lp.metrics.robots_evictions.incr();- self.allocator.free(kv.key);- var entry = kv.value;- self.freeEntry(&entry);- }+ fn insert(self: *RobotStore, url: []const u8, entry: RobotsEntry) !bool {+ switch (try self.map.insert(url, entry)) {+ .exists => return false,+ .inserted => |evicted| {+ if (evicted) |value| {+ lp.metrics.robots_evictions.incr();+ var e = value;+ self.freeEntry(&e);+ }+ return true;+ },}-- return gop.value_ptr;}};