Skip to content

Lightpanda: the headless browser designed for AI and automation

use SingleFlight in RobotsGate

Muki Kiboigo82e9496409ab
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 8324ef7db..36b9d98e7 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -222,7 +222,11 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.serve_mode = network.config.mode == .serve,
.obey_robots = network.config.obeyRobots(),
- .robots = .{ .allocator = allocator, .network = network },
+ .robots = .{
+ .allocator = allocator,
+ .network = network,
+ .single_flight = .init(allocator),
+ },
.url_blocklist = url_blocklist,
.arena_pool = &network.app.arena_pool,
};
@@ -422,7 +426,7 @@ pub fn abort(self: *Client) void {
std.debug.assert(self.ws_dispatch_queue.first == null);
// - self.robots.pending : each robots fetch's shutdown_callback
// drops its entry; parked waiters unlink in their own deinit.
- std.debug.assert(self.robots.pending.count() == 0);
+ std.debug.assert(self.robots.single_flight.count() == 0);
}
}
@@ -3386,7 +3390,11 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
client.cache = null;
client.serve_mode = false;
client.obey_robots = false;
- client.robots = .{ .allocator = testing.allocator, .network = undefined };
+ client.robots = .{
+ .allocator = testing.allocator,
+ .network = undefined,
+ .single_flight = .init(testing.allocator),
+ };
client.url_blocklist = null;
}
@@ -3578,8 +3586,8 @@ test "HttpClient: fulfillIntercepted survives a done_callback that tears down th
}
test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {
- // Regression: RobotsGate.pending kept a raw *Transfer with nothing
- // removing it when a parked transfer was aborted out-of-band
+ // Regression: RobotsGate's single-flight map kept a raw *Transfer with
+ // nothing removing it when a parked transfer was aborted out-of-band
// (xhr.abort(), owner teardown). The robots.txt resolution would then
// unpark freed memory.
var pool = ArenaPool.init(testing.allocator, .{});
@@ -3620,18 +3628,19 @@ test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {
try waiting.append(testing.allocator, transfer);
transfer.park(.robots);
}
- try client.robots.pending.putNoClobber(testing.allocator, robots_url, waiting);
- const t1 = client.robots.pending.get(robots_url).?.items[0];
- const t2 = client.robots.pending.get(robots_url).?.items[1];
+ try client.robots.single_flight.pending.putNoClobber(testing.allocator, robots_url, waiting);
+
+ const t1 = client.robots.single_flight.pending.get(robots_url).?.items[0];
+ const t2 = client.robots.single_flight.pending.get(robots_url).?.items[1];
t1.abort(error.Abort);
- try testing.expectEqual(1, client.robots.pending.get(robots_url).?.items.len);
- try testing.expect(client.robots.pending.get(robots_url).?.items[0] == t2);
+ try testing.expectEqual(1, client.robots.single_flight.pending.get(robots_url).?.items.len);
+ try testing.expect(client.robots.single_flight.pending.get(robots_url).?.items[0] == t2);
try testing.expectEqual(1, client.transfers.count());
t2.abort(error.Abort);
- try testing.expectEqual(0, client.robots.pending.get(robots_url).?.items.len);
+ try testing.expectEqual(0, client.robots.single_flight.pending.get(robots_url).?.items.len);
try testing.expectEqual(0, client.transfers.count());
}
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index 552132a89..fdbb68b31 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -31,6 +31,7 @@ const http = @import("http.zig");
const Robots = @import("Robots.zig");
const Network = @import("Network.zig");
const Transfer = @import("HttpClient.zig").Transfer;
+const SingleFlight = @import("SingleFlight.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -39,16 +40,12 @@ const RobotsGate = @This();
network: *Network,
allocator: Allocator,
-pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,
+single_flight: SingleFlight,
pub const Result = enum { allowed, blocked, pending };
pub fn deinit(self: *RobotsGate) void {
- var it = self.pending.iterator();
- while (it.next()) |entry| {
- entry.value_ptr.deinit(self.allocator);
- }
- self.pending.deinit(self.allocator);
+ self.single_flight.deinit();
}
pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
@@ -77,25 +74,10 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
// stays: the in-flight fetch owns it (the key lives on the fetch's context
// arena) and still resolves the remaining waiters.
pub fn remove(self: *RobotsGate, transfer: *Transfer) void {
- var it = self.pending.valueIterator();
- while (it.next()) |waiting| {
- for (waiting.items, 0..) |t, i| {
- if (t == transfer) {
- _ = waiting.swapRemove(i);
- return;
- }
- }
- }
+ self.single_flight.remove(transfer);
}
fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {
- if (self.pending.getPtr(robots_url)) |waiting| {
- // A fetch for this robots.txt is already in flight, queue behind it.
- try waiting.append(self.allocator, transfer);
- transfer.park(.robots);
- return;
- }
-
const client = transfer.client;
// The context, the response buffer and the pending-map key live on
@@ -107,6 +89,15 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans
errdefer arena.release();
const owned_url = try arena.dupeZ(u8, robots_url);
+ const result = try self.single_flight.enter(owned_url, transfer, .robots);
+ if (result == .queued) {
+ // Someone else already owns an in-flight fetch for this key
+ // (under its own, earlier-allocated owned_url). This arena was
+ // never handed to a fetch, so release it ourselves.
+ arena.release();
+ return;
+ }
+
const robots_ctx = try arena.create(RobotsContext);
robots_ctx.* = .{
.gate = self,
@@ -116,16 +107,6 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans
.robots_url = owned_url,
};
- var waiting: std.ArrayList(*Transfer) = .empty;
- try waiting.append(self.allocator, transfer);
- errdefer waiting.deinit(self.allocator);
-
- try self.pending.putNoClobber(self.allocator, owned_url, waiting);
- errdefer _ = self.pending.remove(owned_url);
-
- transfer.park(.robots);
- errdefer transfer.unpark();
-
log.debug(.browser, "fetching robots.txt", .{ .robots_url = owned_url });
// Only the parent's frame/loader ids (CDP correlation) and notification
@@ -161,11 +142,11 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans
// each judged against its own path. No store entry (fetch failed, or a 200
// whose body never got parsed) fails open.
fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
- var queued = self.pending.fetchRemove(robots_url) orelse return;
- defer queued.value.deinit(self.allocator);
+ var queued = self.single_flight.take(robots_url) orelse return;
+ defer queued.deinit(self.allocator);
const robot_entry = self.network.robot_store.get(robots_url);
- for (queued.value.items) |transfer| {
+ for (queued.items) |transfer| {
transfer.unpark();
const allowed = if (robot_entry) |entry| switch (entry) {
@@ -189,15 +170,6 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
}
}
-// shutdown_callback fires when the fetch is kill()'d. The fetch is
-// ownerless, so that only happens at client-wide teardown (Client.abort),
-// where every waiter is being kill()'d by the same loop; their deinit
-// finds no gate entry left (or unlinks itself first) and no-ops.
-fn flushPendingShutdown(self: *RobotsGate, robots_url: []const u8) void {
- var pending = self.pending.fetchRemove(robots_url) orelse return;
- pending.value.deinit(self.allocator);
-}
-
const RobotsContext = struct {
gate: *RobotsGate,
arena: *lp.Arena,
@@ -278,7 +250,7 @@ const RobotsContext = struct {
log.debug(.http, "robots fetch shutdown", .{});
const gate = self.gate;
const arena = self.arena;
- gate.flushPendingShutdown(self.robots_url);
+ gate.single_flight.discard(self.robots_url);
arena.release();
}