Skip to content

Lightpanda: the headless browser designed for AI and automation

mem: fix robot store key leak

Karl Seguin7a4e2f62bd58
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index 81cb68ffc..5ade56263 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -139,8 +139,17 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const duped = try self.allocator.dupe(u8, url);
- try self.map.put(self.allocator, duped, .{ .present = robots });
+ if (try self.putKey(url)) |value_ptr| {
+ // first time seeing this url, store the value
+ value_ptr.* = .{ .present = robots };
+ return;
+ }
+
+ // cannot overwrite an existing value, if it was `present`, we'd have
+ // to free the value but it might be being used.
+
+ var discarded = robots;
+ discarded.deinit(self.allocator);
}
// The returned slice is owned by the store
@@ -160,8 +169,12 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const duped = try self.allocator.dupe(u8, url);
- try self.map.put(self.allocator, duped, .allowed);
+ if (try self.putKey(url)) |value_ptr| {
+ // first time seeing this url, store the value
+ value_ptr.* = .allowed;
+ }
+ // cannot overwrite an existing value, if it was `present`, we'd have
+ // to free the value but it might be being used.
}
/// This URL is fully restricted from crawling.
@@ -169,8 +182,26 @@ pub const RobotStore = struct {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
- const duped = try self.allocator.dupe(u8, url);
- try self.map.put(self.allocator, duped, .disallowed);
+ if (try self.putKey(url)) |value_ptr| {
+ // first time seeing this url, store the value
+ value_ptr.* = .disallowed;
+ }
+ // cannot overwrite an existing value, if it was `present`, we'd have
+ // to free the value but it might be being used.
+ }
+
+ // The RobotStore is shared across Browsers. Two rowsers can request the
+ // same robots URL at the same time, and they'll race here. First one wins.
+ // Caller holds the mutex.
+ fn putKey(self: *RobotStore, url: []const u8) !?*RobotsEntry {
+ const gop = try self.map.getOrPut(self.allocator, url);
+ if (gop.found_existing) {
+ // already have a value, caller should not overwrite
+ return null;
+ }
+ errdefer _ = self.map.remove(url);
+ gop.key_ptr.* = try self.allocator.dupe(u8, url);
+ return gop.value_ptr;
}
};
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index cf9e855d1..ace25406c 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -214,6 +214,7 @@ const RobotsContext = struct {
};
if (robots) |r| {
try network.robot_store.put(robots_url, r);
+ // BE CAREFUL: robots can be invalidated after this call
}
} else {
// Empty robots.txt means we can short-circuit the allowed path.