Lightpanda: the headless browser designed for AI and automation
fix uaf when aborting a robots-parked transfer
Karl Seguin377de019b85b
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex b8dc2070f..48d60a135 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -380,6 +380,9 @@ pub fn abort(self: *Client) void {std.debug.assert(self.in_use.first == null);std.debug.assert(self.ready_queue.first == null);std.debug.assert(self.dirty.first == null);+ // - self.robots.pending : each robots fetch's shutdown_callback+ // drops its entry; parked waiters unlink in their own deinit.+ std.debug.assert(self.robots.pending.count() == 0);}}@@ -1881,10 +1884,7 @@ pub const Transfer = struct {self._conn = null;}- // Unlink from client.queue if we were waiting for a handle.- // Without this, deinit'ing a queued transfer (e.g. via owner-list- // abort during navigation) leaves a dangling _node in the queue- // that the next tick would pop and hand to libcurl → UAF.+ // Unlink from client.pending_queue if we were waiting for a handle.if (self.state == .queued) {self.client.pending_queue.remove(&self._node);}@@ -1900,6 +1900,12 @@ pub const Transfer = struct {self.client.dispatch_count -= 1;}+ // And for the robots gate: RobotsGate.pending holds a raw *Transfer+ // while we're parked.+ if (self.state == .parked and self.state.parked == .robots) {+ self.client.robots.remove(self);+ }+// A pending revalidation entry owns cache resources (possibly an// open file). It's resolved at completion; if we die first, release.switch (self._cache_intent) {@@ -2832,6 +2838,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {client.cache = null;client.serve_mode = false;client.obey_robots = false;+ client.robots = .{ .allocator = testing.allocator, .network = undefined };}test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {@@ -2911,6 +2918,62 @@ test "HttpClient: fulfillIntercepted survives a done_callback that tears down thtry testing.expectEqual(null, owner.transfers.first);}+test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {+ // Regression: RobotsGate.pending kept a raw *Transfer with nothing+ // removing it when a parked transfer was aborted out-of-band+ // (xhr.abort(), owner teardown). The robots.txt resolution would then+ // unpark freed memory.+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ var client: Client = undefined;+ initTestClient(&client, &pool);+ defer client.transfers.deinit(testing.allocator);+ defer client.robots.deinit();++ const robots_url = "http://example.com/robots.txt";++ var waiting: std.ArrayList(*Transfer) = .empty;+ for (0..2) |i| {+ const arena = try pool.acquire(.small, "test");+ const transfer = try arena.create(Transfer);+ transfer.* = .{+ .arena = arena,+ .owner = null,+ .req = .{+ .frame_id = 0,+ .loader_id = 0,+ .method = .GET,+ .url = "http://example.com/",+ .cookie_jar = null,+ .cookie_origin = "",+ .resource_type = .document,+ .notification = undefined,+ .shutdown_callback = noopShutdown,+ },+ .client = &client,+ .id = @intCast(i + 1),+ .start_time = 0,+ };+ try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);+ try waiting.append(testing.allocator, transfer);+ transfer.park(.robots);+ }+ try client.robots.pending.putNoClobber(testing.allocator, robots_url, waiting);++ const t1 = client.robots.pending.get(robots_url).?.items[0];+ const t2 = client.robots.pending.get(robots_url).?.items[1];++ t1.abort(error.Abort);+ try testing.expectEqual(1, client.robots.pending.get(robots_url).?.items.len);+ try testing.expect(client.robots.pending.get(robots_url).?.items[0] == t2);+ try testing.expectEqual(1, client.transfers.count());++ t2.abort(error.Abort);+ try testing.expectEqual(0, client.robots.pending.get(robots_url).?.items.len);+ try testing.expectEqual(0, client.transfers.count());+}+test "HttpClient: fulfillIntercepted follows a 3xx redirect" {// Regression for #2828: a CDP Fetch.fulfillRequest with a 3xx status + a// Location header must be followed like a real network redirect (re-issueddiff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex 0fec17b3a..27bbf7068 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -25,6 +25,7 @@ const std = @import("std");const lp = @import("lightpanda");const URL = @import("../browser/URL.zig");+const ArenaPool = @import("../ArenaPool.zig");const Robots = @import("Robots.zig");const Network = @import("Network.zig");@@ -71,40 +72,66 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {return .pending;}-fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {- const entry = try self.pending.getOrPut(self.allocator, robots_url);+// A parked transfer is dying out-of-band (abort, owner teardown) — unlink+// it so the robots.txt resolution doesn't touch freed memory. The map entry+// stays: the in-flight fetch owns it (the key lives on the fetch's context+// arena) and still resolves the remaining waiters.+pub fn remove(self: *RobotsGate, transfer: *Transfer) void {+ var it = self.pending.valueIterator();+ while (it.next()) |waiting| {+ for (waiting.items, 0..) |t, i| {+ if (t == transfer) {+ _ = waiting.swapRemove(i);+ return;+ }+ }+ }+}- if (entry.found_existing) {+fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {+ if (self.pending.getPtr(robots_url)) |waiting| {// A fetch for this robots.txt is already in flight, queue behind it.- try entry.value_ptr.append(self.allocator, transfer);+ try waiting.append(self.allocator, transfer);transfer.park(.robots);return;}- errdefer _ = self.pending.remove(robots_url);- entry.value_ptr.* = .empty;+ const client = transfer.client;- try entry.value_ptr.append(self.allocator, transfer);- transfer.park(.robots);- errdefer {- entry.value_ptr.deinit(self.allocator);- transfer.unpark();- }+ // The context, the response buffer and the pending-map key live on+ // their own pooled arena, NOT on transfer.arena — any waiter (this one+ // included) can be aborted while the fetch is still in flight, and the+ // fetch's callbacks must survive that. The arena is released by+ // whichever terminal callback fires (done / error / shutdown).+ const arena = try client.arena_pool.acquire(.small, "RobotsGate.RobotsContext");+ errdefer client.arena_pool.release(arena);- const robots_ctx = try transfer.arena.create(RobotsContext);+ const owned_url = try arena.dupeZ(u8, robots_url);+ const robots_ctx = try arena.create(RobotsContext);robots_ctx.* = .{.gate = self,.buffer = .empty,- .arena = transfer.arena,- .robots_url = robots_url,+ .arena = arena,+ .arena_pool = client.arena_pool,+ .robots_url = owned_url,};- log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });+ var waiting: std.ArrayList(*Transfer) = .empty;+ try waiting.append(self.allocator, transfer);+ errdefer waiting.deinit(self.allocator);++ try self.pending.putNoClobber(self.allocator, owned_url, waiting);+ errdefer _ = self.pending.remove(owned_url);++ transfer.park(.robots);+ errdefer transfer.unpark();++ log.debug(.browser, "fetching robots.txt", .{ .robots_url = owned_url });// Only the parent's frame/loader ids (CDP correlation) and notification// carry over — no cookies, credentials, headers, or timeout.- try transfer.client.request(.{- .url = robots_url,+ const fetch_transfer = try client.newRequest(.{+ .url = owned_url,.method = .GET,.internal = true,.resource_type = .fetch,@@ -112,7 +139,7 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans.loader_id = transfer.req.loader_id,.notification = transfer.req.notification,.cookie_jar = null,- .cookie_origin = robots_url,+ .cookie_origin = owned_url,.ctx = robots_ctx,.header_callback = RobotsContext.headerCallback,.data_callback = RobotsContext.dataCallback,@@ -120,14 +147,31 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans.error_callback = RobotsContext.errorCallback,.shutdown_callback = RobotsContext.shutdownCallback,}, transfer.owner);++ // From here the fetch owns the pending entry and the context arena. If+ // submit fails it fires error_callback — possibly synchronously, right+ // here — which resolves the waiters (fail-open, may already have resumed+ // `transfer`) and releases the arena. So there is nothing to unwind+ // locally and the errdefers above must not run: swallow the error.+ fetch_transfer.submit() catch {};}-fn flushPending(self: *RobotsGate, robots_url: [:0]const u8, allowed: bool) void {+// The robots.txt fetch resolved: hand every waiter back to the pipeline,+// each judged against its own path. No store entry (fetch failed, or a 200+// whose body never got parsed) fails open.+fn flushPending(self: *RobotsGate, robots_url: []const u8) void {var queued = self.pending.fetchRemove(robots_url) orelse return;defer queued.value.deinit(self.allocator);+ const robot_entry = self.network.robot_store.get(robots_url);for (queued.value.items) |transfer| {transfer.unpark();++ const allowed = if (robot_entry) |entry| switch (entry) {+ .absent => true,+ .present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),+ } else true;+if (!allowed) {log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });transfer.failAsync(error.RobotsBlocked);@@ -143,9 +187,12 @@ fn flushPending(self: *RobotsGate, robots_url: [:0]const u8, allowed: bool) void}}-// shutdown_callback is only called on owner shutdown. And if this robot's fetch-// is being shutdown, than any transfers waiting for it will be shutdown too.-fn flushPendingShutdown(self: *RobotsGate, robots_url: [:0]const u8) void {+// shutdown_callback fires on owner shutdown. Drop the entry without+// resuming anyone. Waiters from the same owner are being kill()'d by the+// same teardown loop; their deinit finds no gate entry left and no-ops.+// A waiter parked by a different owner stays parked until its own owner+// tears it down — it is never resumed (known limitation).+fn flushPendingShutdown(self: *RobotsGate, robots_url: []const u8) void {var pending = self.pending.fetchRemove(robots_url) orelse return;pending.value.deinit(self.allocator);}@@ -153,6 +200,7 @@ fn flushPendingShutdown(self: *RobotsGate, robots_url: [:0]const u8) void {const RobotsContext = struct {gate: *RobotsGate,arena: Allocator,+ arena_pool: *ArenaPool,robots_url: [:0]const u8,buffer: std.ArrayList(u8),status: u16 = 0,@@ -176,11 +224,8 @@ const RobotsContext = struct {fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));- const gate = self.gate;const robots_url = self.robots_url;-- var allowed = true;- const network = gate.network;+ const network = self.gate.network;switch (self.status) {200 => {@@ -195,8 +240,6 @@ const RobotsContext = struct {};if (robots) |r| {try network.robot_store.put(robots_url, r);- const path = URL.getPathname(gate.pending.get(robots_url).?.items[0].req.url);- allowed = r.isAllowed(path);}}},@@ -213,20 +256,34 @@ const RobotsContext = struct {},}- gate.flushPending(robots_url, allowed);+ // If anything above threw, error_callback fires next and resolves+ // instead — resolve() must run exactly once.+ self.resolve();}fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));log.warn(.http, "robots fetch failed", .{ .err = err });- self.gate.flushPending(self.robots_url, true);+ self.resolve();}fn shutdownCallback(ctx_ptr: *anyopaque) void {const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));log.debug(.http, "robots fetch shutdown", .{});- self.gate.flushPendingShutdown(self.robots_url);+ const gate = self.gate;+ const pool = self.arena_pool;+ const arena = self.arena;+ gate.flushPendingShutdown(self.robots_url);+ pool.release(arena);+ }++ fn resolve(self: *RobotsContext) void {+ const gate = self.gate;+ const pool = self.arena_pool;+ const arena = self.arena;+ gate.flushPending(self.robots_url);+ pool.release(arena);}};