Skip to content

Lightpanda: the headless browser designed for AI and automation

use ClockCache as the map in RobotStore

Muki Kiboigo74789d3af0ae
diff --git a/src/network/ClockCache.zig b/src/network/ClockCache.zig
index 86a0abf7a..84c41f310 100644
--- a/src/network/ClockCache.zig
+++ b/src/network/ClockCache.zig
@@ -19,14 +19,23 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
-pub fn ClockCache(comptime K: type) type {
+pub fn Entry(comptime V: type) type {
+ return struct {
+ value: V,
+ referenced: bool,
+ };
+}
+
+pub fn ClockCache(comptime V: type) type {
return struct {
const Self = @This();
- const Map = if (K == []const u8)
- std.array_hash_map.String(bool)
- else
- std.array_hash_map.Auto(K, bool);
+ const Map = std.array_hash_map.String(Entry(V));
+
+ pub const InsertResult = union(enum) {
+ exists,
+ inserted: ?V,
+ };
allocator: Allocator,
capacity: usize,
@@ -39,34 +48,50 @@ pub fn ClockCache(comptime K: type) type {
}
pub fn deinit(self: *Self) void {
+ for (self.map.keys()) |key| {
+ self.allocator.free(key);
+ }
self.map.deinit(self.allocator);
}
- pub fn insert(self: *Self, key: K) !?K {
- try self.map.put(self.allocator, key, true);
+ pub fn entries(self: *Self) []Entry(V) {
+ return self.map.values();
+ }
- if (self.map.count() <= self.capacity) return null;
- return self.evictOne();
+ pub fn get(self: *Self, key: []const u8) ?*V {
+ const entry = self.map.getPtr(key) orelse return null;
+ entry.referenced = true;
+ return &entry.value;
}
- pub fn touch(self: *Self, key: K) void {
- if (self.map.getPtr(key)) |referenced| referenced.* = true;
+ pub fn insert(self: *Self, key: []const u8, value: V) !InsertResult {
+ const gop = try self.map.getOrPut(self.allocator, key);
+ if (gop.found_existing) return .exists;
+
+ errdefer self.map.swapRemoveAt(gop.index);
+ gop.key_ptr.* = try self.allocator.dupe(u8, key);
+ gop.value_ptr.* = .{ .value = value, .referenced = true };
+
+ if (self.map.count() <= self.capacity) return .{ .inserted = null };
+ return .{ .inserted = self.evictOne() };
}
- fn evictOne(self: *Self) ?K {
- const referenced = self.map.values();
+ fn evictOne(self: *Self) V {
+ const items = self.map.values();
while (true) {
- if (self.hand >= referenced.len) self.hand = 0;
+ if (self.hand >= items.len) self.hand = 0;
- if (referenced[self.hand]) {
- referenced[self.hand] = false;
+ if (items[self.hand].referenced) {
+ items[self.hand].referenced = false;
self.hand += 1;
continue;
}
const key = self.map.keys()[self.hand];
+ const value = items[self.hand].value;
self.map.swapRemoveAt(self.hand);
- return key;
+ self.allocator.free(key);
+ return value;
}
}
};
@@ -74,46 +99,60 @@ pub fn ClockCache(comptime K: type) type {
const testing = @import("../testing.zig");
+fn evictedOf(comptime V: type, r: ClockCache(V).InsertResult) ?V {
+ return switch (r) {
+ .exists => unreachable,
+ .inserted => |v| v,
+ };
+}
+
test "ClockCache: no eviction under capacity" {
- var q = ClockCache([]const u8).init(testing.allocator, 3);
+ var q = ClockCache(u32).init(testing.allocator, 3);
defer q.deinit();
- try testing.expectEqual(null, try q.insert("a"));
- try testing.expectEqual(null, try q.insert("b"));
- try testing.expectEqual(null, try q.insert("c"));
+ try testing.expectEqual(null, evictedOf(u32, try q.insert("a", 1)));
+ try testing.expectEqual(null, evictedOf(u32, try q.insert("b", 2)));
+ try testing.expectEqual(null, evictedOf(u32, try q.insert("c", 3)));
try testing.expectEqual(3, q.map.count());
}
test "ClockCache: evicts once over capacity" {
- var q = ClockCache([]const u8).init(testing.allocator, 2);
+ var q = ClockCache(u32).init(testing.allocator, 2);
defer q.deinit();
- try testing.expectEqual(null, try q.insert("a"));
- try testing.expectEqual(null, try q.insert("b"));
+ try testing.expectEqual(null, evictedOf(u32, try q.insert("a", 1)));
+ try testing.expectEqual(null, evictedOf(u32, try q.insert("b", 2)));
- const evicted = try q.insert("c");
- try testing.expect(evicted != null);
- try testing.expectString("a", evicted.?);
+ const evicted = evictedOf(u32, try q.insert("c", 3));
+ try testing.expectEqual(1, evicted.?);
+ try testing.expect(q.get("a") == null);
try testing.expectEqual(2, q.map.count());
}
test "ClockCache: touch protects a key from eviction" {
- var q = ClockCache([]const u8).init(testing.allocator, 2);
+ var q = ClockCache(u32).init(testing.allocator, 2);
defer q.deinit();
- try testing.expectEqual(null, try q.insert("a"));
- try testing.expectEqual(null, try q.insert("b"));
+ _ = try q.insert("a", 1);
+ _ = try q.insert("b", 2);
- const evicted1 = try q.insert("c");
- try testing.expect(evicted1 != null);
- try testing.expectString("a", evicted1.?);
- try testing.expect(q.map.contains("b"));
- try testing.expect(q.map.contains("c"));
+ const evicted1 = evictedOf(u32, try q.insert("c", 3));
+ try testing.expectEqual(1, evicted1.?);
+ try testing.expect(q.get("b") != null);
+ try testing.expect(q.get("c") != null);
- q.touch("b");
- const evicted2 = try q.insert("d");
- try testing.expect(evicted2 != null);
- try testing.expectString("c", evicted2.?);
- try testing.expect(q.map.contains("b"));
+ _ = q.get("b");
+ const evicted2 = evictedOf(u32, try q.insert("d", 4));
+ try testing.expectEqual(3, evicted2.?);
+ try testing.expect(q.get("b") != null);
try testing.expectEqual(2, q.map.count());
}
+
+test "ClockCache: insert does not overwrite" {
+ var q = ClockCache(u32).init(testing.allocator, 2);
+ defer q.deinit();
+
+ _ = try q.insert("a", 1);
+ try testing.expect((try q.insert("a", 99)) == .exists);
+ try testing.expectEqual(1, q.get("a").?.*);
+}
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index f92cc9aba..33d0fcfba 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -95,18 +95,14 @@ pub const RobotStore = struct {
disallowed,
};
- const RobotsMap = @import("Network.zig").HostHashMap(RobotsEntry);
-
allocator: std.mem.Allocator,
- map: RobotsMap,
- evictions: ClockCache([]const u8),
+ map: ClockCache(RobotsEntry),
mutex: std.Io.Mutex = .init,
pub fn init(allocator: std.mem.Allocator, capacity: u32) RobotStore {
return .{
.allocator = allocator,
- .map = .empty,
- .evictions = .init(allocator, capacity),
+ .map = .init(allocator, capacity),
};
}
@@ -114,15 +110,10 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- var iter = self.map.iterator();
-
- while (iter.next()) |entry| {
- self.allocator.free(entry.key_ptr.*);
- self.freeEntry(entry.value_ptr);
+ for (self.map.entries()) |*entry| {
+ self.freeEntry(&entry.value);
}
-
- self.map.deinit(self.allocator);
- self.evictions.deinit();
+ self.map.deinit();
}
fn freeEntry(self: *RobotStore, entry: *RobotsEntry) void {
@@ -138,10 +129,8 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const kv = self.map.getEntry(url) orelse return null;
- self.evictions.touch(kv.key_ptr.*);
-
- return switch (kv.value_ptr.*) {
+ const entry = self.map.get(url) orelse return null;
+ return switch (entry.*) {
.allowed => .allowed,
.disallowed => .blocked,
.present => |robots| if (robots.isAllowed(path)) .allowed else .blocked,
@@ -156,15 +145,10 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- if (try self.putKey(url)) |value_ptr| {
- // first time seeing this url, store the value
- value_ptr.* = .{ .present = robots };
- return;
- }
+ if (try self.insert(url, .{ .present = robots })) return;
// cannot overwrite an existing value, if it was `present`, we'd have
// to free the value but it might be being used.
-
var discarded = robots;
discarded.deinit(self.allocator);
}
@@ -178,7 +162,7 @@ pub const RobotStore = struct {
defer self.mutex.unlock(lp.io);
const entry = self.map.get(url) orelse return null;
- const signals = switch (entry) {
+ const signals = switch (entry.*) {
.present => |robots| robots.content_signals,
.allowed, .disallowed => return null,
};
@@ -197,50 +181,28 @@ pub const RobotStore = struct {
pub fn putAllowed(self: *RobotStore, url: []const u8) !void {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
-
- if (try self.putKey(url)) |value_ptr| {
- // first time seeing this url, store the value
- value_ptr.* = .allowed;
- }
- // cannot overwrite an existing value, if it was `present`, we'd have
- // to free the value but it might be being used.
+ _ = try self.insert(url, .allowed);
}
/// This URL is fully restricted from crawling.
pub fn putDisallowed(self: *RobotStore, url: []const u8) !void {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
-
- if (try self.putKey(url)) |value_ptr| {
- // first time seeing this url, store the value
- value_ptr.* = .disallowed;
- }
- // cannot overwrite an existing value, if it was `present`, we'd have
- // to free the value but it might be being used.
+ _ = try self.insert(url, .disallowed);
}
- // The RobotStore is shared across Browsers. Two rowsers can request the
- // same robots URL at the same time, and they'll race here. First one wins.
- // Caller holds the mutex.
- fn putKey(self: *RobotStore, url: []const u8) !?*RobotsEntry {
- const gop = try self.map.getOrPut(self.allocator, url);
- if (gop.found_existing) {
- // already have a value, caller should not overwrite
- return null;
- }
- errdefer _ = self.map.remove(url);
- gop.key_ptr.* = try self.allocator.dupe(u8, url);
-
- if (try self.evictions.insert(gop.key_ptr.*)) |evict_key| {
- if (self.map.fetchRemove(evict_key)) |kv| {
- lp.metrics.robots_evictions.incr();
- self.allocator.free(kv.key);
- var entry = kv.value;
- self.freeEntry(&entry);
- }
+ fn insert(self: *RobotStore, url: []const u8, entry: RobotsEntry) !bool {
+ switch (try self.map.insert(url, entry)) {
+ .exists => return false,
+ .inserted => |evicted| {
+ if (evicted) |value| {
+ lp.metrics.robots_evictions.incr();
+ var e = value;
+ self.freeEntry(&e);
+ }
+ return true;
+ },
}
-
- return gop.value_ptr;
}
};