Skip to content

Lightpanda: the headless browser designed for AI and automation

mem: reduce memory usage of cloned HTTP responses

Karl Seguinf81f6e4eb7d0
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index 436c89632..097e3f336 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -698,19 +698,14 @@ pub const Script = struct {
lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });
- const content_length = transfer.getContentLength();
+ const body_len = transfer.bodyLen();
if (self.source_arena == null) {
// A redirect re-runs this callback; keep the arena we already have.
- self.source_arena = if (content_length) |cl|
- try self.manager.acquireArena(cl, "SM.source")
- else
- try self.manager.acquireArena(.large, "SM.source");
+ self.source_arena = try self.manager.acquireArena(body_len, "SM.source");
}
var buffer: std.ArrayList(u8) = .empty;
- if (content_length) |cl| {
- try buffer.ensureTotalCapacityPrecise(self.sourceAllocator(), cl);
- }
+ try buffer.ensureTotalCapacityPrecise(self.sourceAllocator(), body_len);
self.source = .{ .remote = buffer };
return .proceed;
}
diff --git a/src/browser/webapi/SharedWorkerGlobalScope.zig b/src/browser/webapi/SharedWorkerGlobalScope.zig
index cbeae1a8e..f9c02c765 100644
--- a/src/browser/webapi/SharedWorkerGlobalScope.zig
+++ b/src/browser/webapi/SharedWorkerGlobalScope.zig
@@ -209,9 +209,7 @@ fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {
return .abort;
}
- if (transfer.getContentLength()) |cl| {
- try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), cl);
- }
+ try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), transfer.bodyLen());
return .proceed;
}
diff --git a/src/browser/webapi/Worker.zig b/src/browser/webapi/Worker.zig
index d1a7bae13..7ffd8263a 100644
--- a/src/browser/webapi/Worker.zig
+++ b/src/browser/webapi/Worker.zig
@@ -163,9 +163,7 @@ fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {
return .abort;
}
- if (transfer.getContentLength()) |cl| {
- try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), cl);
- }
+ try self._script_buffer.ensureTotalCapacityPrecise(self._script_arena.?.allocator(), transfer.bodyLen());
return .proceed;
}
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index a0db01e94..c339727f5 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -160,9 +160,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
const arena = self._response._arena;
if (!is_opaque) {
- if (transfer.getContentLength()) |cl| {
- try self._buf.ensureTotalCapacityPrecise(arena.allocator(), cl);
- }
+ try self._buf.ensureTotalCapacityPrecise(arena.allocator(), transfer.bodyLen());
}
const res = self._response;
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index 90dea8f4a..7aac7ab51 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -654,8 +654,8 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
self._response_status = transfer.responseStatus().?;
if (transfer.getContentLength()) |cl| {
self._response_len = cl;
- try self._response_data.ensureTotalCapacityPrecise(self._arena.allocator(), cl);
}
+ try self._response_data.ensureTotalCapacityPrecise(self._arena.allocator(), transfer.bodyLen());
self._response_url = try self._arena.dupeZ(u8, transfer.req.url);
const exec = self._exec;
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index fb61c5bc8..7fc0b6c25 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1374,8 +1374,9 @@ const SyncContext = struct {
const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));
lp.assert(transfer.responseStatus() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = transfer.responseStatus() });
self.status = transfer.responseStatus().?;
- if (transfer.getContentLength()) |cl| {
- try self.body.ensureTotalCapacityPrecise(try self.bodyAllocator(cl), cl);
+ const body_len = transfer.bodyLen();
+ if (body_len > 0) {
+ try self.body.ensureTotalCapacityPrecise(try self.bodyAllocator(body_len), body_len);
}
return .proceed;
}
@@ -2324,6 +2325,10 @@ pub const Transfer = struct {
// Content length reported on the CDP loadingFinished event.
_content_length: usize = 0,
+ // Length of the body data_callback will receive. Can be different than
+ // Content-Length if the content was compressed
+ _body_len: usize = 0,
+
_conn_id: i64 = 0,
_conn_reused: bool = false,
_timing: ResourceTiming = .{},
@@ -3068,6 +3073,7 @@ pub const Transfer = struct {
lp.metrics.http_response_size_bytes.observe(body.len);
}
+ self._body_len = body.len;
try self._events.ensureUnusedCapacity(self.arena.allocator(), 4);
self._events.appendAssumeCapacity(.start);
self._events.appendAssumeCapacity(.header);
@@ -3640,6 +3646,13 @@ pub const Transfer = struct {
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
}
+ // TODO: Because of compression, Content-Length is the wire
+ // length, not necessarily the final length. The chunks
+ // are read into the transfer.*ARENA* so growth doesn't free
+ // previous allocations. We could look at Content-Encoding
+ // and `cl * 3` or something, but that's just a guess.
+ // I prefer to leave this simple; easier for someone to come
+ // up with a good solution.
res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};
}
}
@@ -3764,6 +3777,14 @@ pub const Transfer = struct {
return std.fmt.parseInt(usize, cl, 10) catch null;
}
+ // Unless streaming, we've read the entire body before calling
+ // header_callback. Code that needs to own the body (most callers) should
+ // use the body length NOT the Content-Length (which would be the compressed
+ // on-the-wire size, not the actual final length). 0 for streaming.
+ pub fn bodyLen(self: *const Transfer) usize {
+ return self._body_len;
+ }
+
fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {
// Materialized headers (dispatch time, any source).
for (self.res.headers) |hdr| {
@@ -4953,6 +4974,70 @@ test "HttpClient: kill during a non-terminal callback defers shutdown_callback"
try testing.expectEqual(null, owner.transfers.first);
}
+test "HttpClient: bodyLen is the buffered body, not Content-Length" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ var client: Client = undefined;
+ initTestClient(&client, &pool);
+ // Runs before pool.deinit (LIFO): retired arenas must go back first.
+ defer client.processGraveyard();
+ defer client.transfers.deinit(testing.allocator);
+
+ var owner = testOwner(null, null);
+
+ const Ctx = struct {
+ body_len: usize = 0,
+ content_length: ?usize = null,
+
+ fn headerCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));
+ self.body_len = transfer.bodyLen();
+ self.content_length = transfer.getContentLength();
+ return .proceed;
+ }
+ };
+ var ctx = Ctx{};
+
+ const arena = try pool.acquire(.small, "test");
+ const transfer = try arena.create(Transfer);
+ transfer.* = .{
+ .arena = arena,
+ .owner = null,
+ .req = .{
+ .method = .GET,
+ .url = "http://example.com/",
+ .origin = null,
+ .credentials_mode = .omit,
+ .request_mode = .no_cors,
+ .resource_type = .xhr,
+ .shutdown_callback = noopShutdown,
+ .ctx = &ctx,
+ .header_callback = Ctx.headerCallback,
+ },
+ .client = &client,
+ .id = 1,
+ .start_time = 0,
+ };
+
+ try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);
+ owner.addTransfer(transfer);
+ transfer.owner = &owner;
+
+ transfer.park(.intercept_request);
+ client.intercepted += 1;
+ try client.fulfillIntercepted(transfer, 200, &.{
+ .{ .name = "Content-Length", .value = "323838382838" },
+ }, "hello");
+ _ = client.dispatchCompleted(.all);
+
+ try testing.expectEqual(323838382838, ctx.content_length);
+ try testing.expectEqual(5, ctx.body_len);
+
+ try testing.expectEqual(0, client.transfers.count());
+ try testing.expectEqual(null, owner.transfers.first);
+}
+
test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" {
// Regression: RobotsGate's single-flight map kept a raw *Transfer with
// nothing removing it when a parked transfer was aborted out-of-band
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index e4d9427ce..cf9e855d1 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -182,9 +182,7 @@ const RobotsContext = struct {
self.status = hdr.status;
}
lp.metrics.robots_status.incr(http.statusCategory(self.status));
- if (transfer.getContentLength()) |cl| {
- try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), cl);
- }
+ try self.buffer.ensureTotalCapacityPrecise(self.arena.allocator(), transfer.bodyLen());
return .proceed;
}
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index 228fd7aff..14be21be9 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -1031,11 +1031,11 @@ pub const BrowserContext = struct {
if (!gop.found_existing) {
gop.value_ptr.* = .{
.data = blk: {
- const cl = msg.transfer.getContentLength() orelse break :blk .empty;
- if (cl > self.network_limits.resource) {
+ const body_len = msg.transfer.bodyLen();
+ if (body_len > self.network_limits.resource) {
break :blk null;
}
- break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl);
+ break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, body_len);
},
// Encode the data in base64 by default, but don't encode
// for well known content-type.