Lightpanda: the headless browser designed for AI and automation
use SingleFlight in RobotsGate
Muki Kiboigo82e9496409ab
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 8324ef7db..36b9d98e7 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -222,7 +222,11 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).serve_mode = network.config.mode == .serve,.obey_robots = network.config.obeyRobots(),- .robots = .{ .allocator = allocator, .network = network },+ .robots = .{+ .allocator = allocator,+ .network = network,+ .single_flight = .init(allocator),+ },.url_blocklist = url_blocklist,.arena_pool = &network.app.arena_pool,};@@ -422,7 +426,7 @@ pub fn abort(self: *Client) void {std.debug.assert(self.ws_dispatch_queue.first == null);// - self.robots.pending : each robots fetch's shutdown_callback// drops its entry; parked waiters unlink in their own deinit.- std.debug.assert(self.robots.pending.count() == 0);+ std.debug.assert(self.robots.single_flight.count() == 0);}}@@ -3386,7 +3390,11 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {client.cache = null;client.serve_mode = false;client.obey_robots = false;- client.robots = .{ .allocator = testing.allocator, .network = undefined };+ client.robots = .{+ .allocator = testing.allocator,+ .network = undefined,+ .single_flight = .init(testing.allocator),+ };client.url_blocklist = null;}@@ -3578,8 +3586,8 @@ test "HttpClient: fulfillIntercepted survives a done_callback that tears down th}test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {- // Regression: RobotsGate.pending kept a raw *Transfer with nothing- // removing it when a parked transfer was aborted out-of-band+ // Regression: RobotsGate's single-flight map kept a raw *Transfer with+ // nothing removing it when a parked transfer was aborted out-of-band// (xhr.abort(), owner teardown). The robots.txt resolution would then// unpark freed memory.var pool = ArenaPool.init(testing.allocator, .{});@@ -3620,18 +3628,19 @@ test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {try waiting.append(testing.allocator, transfer);transfer.park(.robots);}- try client.robots.pending.putNoClobber(testing.allocator, robots_url, waiting);- const t1 = client.robots.pending.get(robots_url).?.items[0];- const t2 = client.robots.pending.get(robots_url).?.items[1];+ try client.robots.single_flight.pending.putNoClobber(testing.allocator, robots_url, waiting);++ const t1 = client.robots.single_flight.pending.get(robots_url).?.items[0];+ const t2 = client.robots.single_flight.pending.get(robots_url).?.items[1];t1.abort(error.Abort);- try testing.expectEqual(1, client.robots.pending.get(robots_url).?.items.len);- try testing.expect(client.robots.pending.get(robots_url).?.items[0] == t2);+ try testing.expectEqual(1, client.robots.single_flight.pending.get(robots_url).?.items.len);+ try testing.expect(client.robots.single_flight.pending.get(robots_url).?.items[0] == t2);try testing.expectEqual(1, client.transfers.count());t2.abort(error.Abort);- try testing.expectEqual(0, client.robots.pending.get(robots_url).?.items.len);+ try testing.expectEqual(0, client.robots.single_flight.pending.get(robots_url).?.items.len);try testing.expectEqual(0, client.transfers.count());}diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex 552132a89..fdbb68b31 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -31,6 +31,7 @@ const http = @import("http.zig");const Robots = @import("Robots.zig");const Network = @import("Network.zig");const Transfer = @import("HttpClient.zig").Transfer;+const SingleFlight = @import("SingleFlight.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -39,16 +40,12 @@ const RobotsGate = @This();network: *Network,allocator: Allocator,-pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,+single_flight: SingleFlight,pub const Result = enum { allowed, blocked, pending };pub fn deinit(self: *RobotsGate) void {- var it = self.pending.iterator();- while (it.next()) |entry| {- entry.value_ptr.deinit(self.allocator);- }- self.pending.deinit(self.allocator);+ self.single_flight.deinit();}pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {@@ -77,25 +74,10 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {// stays: the in-flight fetch owns it (the key lives on the fetch's context// arena) and still resolves the remaining waiters.pub fn remove(self: *RobotsGate, transfer: *Transfer) void {- var it = self.pending.valueIterator();- while (it.next()) |waiting| {- for (waiting.items, 0..) |t, i| {- if (t == transfer) {- _ = waiting.swapRemove(i);- return;- }- }- }+ self.single_flight.remove(transfer);}fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {- if (self.pending.getPtr(robots_url)) |waiting| {- // A fetch for this robots.txt is already in flight, queue behind it.- try waiting.append(self.allocator, transfer);- transfer.park(.robots);- return;- }-const client = transfer.client;// The context, the response buffer and the pending-map key live on@@ -107,6 +89,15 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transerrdefer arena.release();const owned_url = try arena.dupeZ(u8, robots_url);+ const result = try self.single_flight.enter(owned_url, transfer, .robots);+ if (result == .queued) {+ // Someone else already owns an in-flight fetch for this key+ // (under its own, earlier-allocated owned_url). This arena was+ // never handed to a fetch, so release it ourselves.+ arena.release();+ return;+ }+const robots_ctx = try arena.create(RobotsContext);robots_ctx.* = .{.gate = self,@@ -116,16 +107,6 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans.robots_url = owned_url,};- var waiting: std.ArrayList(*Transfer) = .empty;- try waiting.append(self.allocator, transfer);- errdefer waiting.deinit(self.allocator);-- try self.pending.putNoClobber(self.allocator, owned_url, waiting);- errdefer _ = self.pending.remove(owned_url);-- transfer.park(.robots);- errdefer transfer.unpark();-log.debug(.browser, "fetching robots.txt", .{ .robots_url = owned_url });// Only the parent's frame/loader ids (CDP correlation) and notification@@ -161,11 +142,11 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans// each judged against its own path. No store entry (fetch failed, or a 200// whose body never got parsed) fails open.fn flushPending(self: *RobotsGate, robots_url: []const u8) void {- var queued = self.pending.fetchRemove(robots_url) orelse return;- defer queued.value.deinit(self.allocator);+ var queued = self.single_flight.take(robots_url) orelse return;+ defer queued.deinit(self.allocator);const robot_entry = self.network.robot_store.get(robots_url);- for (queued.value.items) |transfer| {+ for (queued.items) |transfer| {transfer.unpark();const allowed = if (robot_entry) |entry| switch (entry) {@@ -189,15 +170,6 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {}}-// shutdown_callback fires when the fetch is kill()'d. The fetch is-// ownerless, so that only happens at client-wide teardown (Client.abort),-// where every waiter is being kill()'d by the same loop; their deinit-// finds no gate entry left (or unlinks itself first) and no-ops.-fn flushPendingShutdown(self: *RobotsGate, robots_url: []const u8) void {- var pending = self.pending.fetchRemove(robots_url) orelse return;- pending.value.deinit(self.allocator);-}-const RobotsContext = struct {gate: *RobotsGate,arena: *lp.Arena,@@ -278,7 +250,7 @@ const RobotsContext = struct {log.debug(.http, "robots fetch shutdown", .{});const gate = self.gate;const arena = self.arena;- gate.flushPendingShutdown(self.robots_url);+ gate.single_flight.discard(self.robots_url);arena.release();}