Lightpanda: the headless browser designed for AI and automation
mem: reduce memory usage of cloned HTTP responses
Karl Seguinf81f6e4eb7d0
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex 436c89632..097e3f336 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -698,19 +698,14 @@ pub const Script = struct {lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });- const content_length = transfer.getContentLength();+ const body_len = transfer.bodyLen();if (self.source_arena == null) {// A redirect re-runs this callback; keep the arena we already have.- self.source_arena = if (content_length) |cl|- try self.manager.acquireArena(cl, "SM.source")- else- try self.manager.acquireArena(.large, "SM.source");+ self.source_arena = try self.manager.acquireArena(body_len, "SM.source");}var buffer: std.ArrayList(u8) = .empty;- if (content_length) |cl| {- try buffer.ensureTotalCapacityPrecise(self.sourceAllocator(), cl);- }+ try buffer.ensureTotalCapacityPrecise(self.sourceAllocator(), body_len);self.source = .{ .remote = buffer };return .proceed;}diff --git a/src/browser/webapi/SharedWorkerGlobalScope.zig b/src/browser/webapi/SharedWorkerGlobalScope.zigindex cbeae1a8e..f9c02c765 100644--- a/src/browser/webapi/SharedWorkerGlobalScope.zig+++ b/src/browser/webapi/SharedWorkerGlobalScope.zig@@ -209,9 +209,7 @@ fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {return .abort;}- if (transfer.getContentLength()) |cl| {- try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), cl);- }+ try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), transfer.bodyLen());return .proceed;}diff --git a/src/browser/webapi/Worker.zig b/src/browser/webapi/Worker.zigindex d1a7bae13..7ffd8263a 100644--- a/src/browser/webapi/Worker.zig+++ b/src/browser/webapi/Worker.zig@@ -163,9 +163,7 @@ fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {return .abort;}- if (transfer.getContentLength()) |cl| {- try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), cl);- }+ try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), transfer.bodyLen());return .proceed;}diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex a0db01e94..c339727f5 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -160,9 +160,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {const arena = self._response._arena;if (!is_opaque) {- if (transfer.getContentLength()) |cl| {- try self._buf.ensureTotalCapacityPrecise(arena.allocator(), cl);- }+ try self._buf.ensureTotalCapacityPrecise(arena.allocator(), transfer.bodyLen());}const res = self._response;diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex 90dea8f4a..7aac7ab51 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -654,8 +654,8 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {self._response_status = transfer.responseStatus().?;if (transfer.getContentLength()) |cl| {self._response_len = cl;- try self._response_data.ensureTotalCapacityPrecise(self._arena.allocator(), cl);}+ try self._response_data.ensureTotalCapacityPrecise(self._arena.allocator(), transfer.bodyLen());self._response_url = try self._arena.dupeZ(u8, transfer.req.url);const exec = self._exec;diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex fb61c5bc8..7fc0b6c25 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1374,8 +1374,9 @@ const SyncContext = struct {const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));lp.assert(transfer.responseStatus() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = transfer.responseStatus() });self.status = transfer.responseStatus().?;- if (transfer.getContentLength()) |cl| {- try self.body.ensureTotalCapacityPrecise(try self.bodyAllocator(cl), cl);+ const body_len = transfer.bodyLen();+ if (body_len > 0) {+ try self.body.ensureTotalCapacityPrecise(try self.bodyAllocator(body_len), body_len);}return .proceed;}@@ -2324,6 +2325,10 @@ pub const Transfer = struct {// Content length reported on the CDP loadingFinished event._content_length: usize = 0,+ // Length of the body data_callback will receive. Can be different than+ // Content-Length if the content was compressed+ _body_len: usize = 0,+_conn_id: i64 = 0,_conn_reused: bool = false,_timing: ResourceTiming = .{},@@ -3068,6 +3073,7 @@ pub const Transfer = struct {lp.metrics.http_response_size_bytes.observe(body.len);}+ self._body_len = body.len;try self._events.ensureUnusedCapacity(self.arena.allocator(), 4);self._events.appendAssumeCapacity(.start);self._events.appendAssumeCapacity(.header);@@ -3640,6 +3646,13 @@ pub const Transfer = struct {res.callback_error = error.ResponseTooLarge;return http.writefunc_error;}+ // TODO: Because of compression, Content-Length is the wire+ // length, not necessarily the final length. The chunks+ // are read into the transfer.*ARENA* so growth doesn't free+ // previous allocations. We could look at Content-Encoding+ // and `cl * 3` or something, but that's just a guess.+ // I prefer to leave this simple; easier for someone to come+ // up with a good solution.res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};}}@@ -3764,6 +3777,14 @@ pub const Transfer = struct {return std.fmt.parseInt(usize, cl, 10) catch null;}+ // Unless streaming, we've read the entire body before calling+ // header_callback. Code that needs to own the body (most callers) should+ // use the body length NOT the Content-Length (which would be the compressed+ // on-the-wire size, not the actual final length). 0 for streaming.+ pub fn bodyLen(self: *const Transfer) usize {+ return self._body_len;+ }+fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {// Materialized headers (dispatch time, any source).for (self.res.headers) |hdr| {@@ -4953,6 +4974,70 @@ test "HttpClient: kill during a non-terminal callback defers shutdown_callback"try testing.expectEqual(null, owner.transfers.first);}+test "HttpClient: bodyLen is the buffered body, not Content-Length" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ var client: Client = undefined;+ initTestClient(&client, &pool);+ // Runs before pool.deinit (LIFO): retired arenas must go back first.+ defer client.processGraveyard();+ defer client.transfers.deinit(testing.allocator);++ var owner = testOwner(null, null);++ const Ctx = struct {+ body_len: usize = 0,+ content_length: ?usize = null,++ fn headerCallback(transfer: *Transfer) !Transfer.HeaderResult {+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));+ self.body_len = transfer.bodyLen();+ self.content_length = transfer.getContentLength();+ return .proceed;+ }+ };+ var ctx = Ctx{};++ const arena = try pool.acquire(.small, "test");+ const transfer = try arena.create(Transfer);+ transfer.* = .{+ .arena = arena,+ .owner = null,+ .req = .{+ .method = .GET,+ .url = "http://example.com/",+ .origin = null,+ .credentials_mode = .omit,+ .request_mode = .no_cors,+ .resource_type = .xhr,+ .shutdown_callback = noopShutdown,+ .ctx = &ctx,+ .header_callback = Ctx.headerCallback,+ },+ .client = &client,+ .id = 1,+ .start_time = 0,+ };++ try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);+ owner.addTransfer(transfer);+ transfer.owner = &owner;++ transfer.park(.intercept_request);+ client.intercepted += 1;+ try client.fulfillIntercepted(transfer, 200, &.{+ .{ .name = "Content-Length", .value = "323838382838" },+ }, "hello");+ _ = client.dispatchCompleted(.all);++ try testing.expectEqual(323838382838, ctx.content_length);+ try testing.expectEqual(5, ctx.body_len);++ try testing.expectEqual(0, client.transfers.count());+ try testing.expectEqual(null, owner.transfers.first);+}+test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {// Regression: RobotsGate's single-flight map kept a raw *Transfer with// nothing removing it when a parked transfer was aborted out-of-banddiff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex e4d9427ce..cf9e855d1 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -182,9 +182,7 @@ const RobotsContext = struct {self.status = hdr.status;}lp.metrics.robots_status.incr(http.statusCategory(self.status));- if (transfer.getContentLength()) |cl| {- try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), cl);- }+ try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), transfer.bodyLen());return .proceed;}diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 228fd7aff..14be21be9 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -1031,11 +1031,11 @@ pub const BrowserContext = struct {if (!gop.found_existing) {gop.value_ptr.* = .{.data = blk: {- const cl = msg.transfer.getContentLength() orelse break :blk .empty;- if (cl > self.network_limits.resource) {+ const body_len = msg.transfer.bodyLen();+ if (body_len > self.network_limits.resource) {break :blk null;}- break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl);+ break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, body_len);},// Encode the data in base64 by default, but don't encode// for well known content-type.