Skip to content

Lightpanda: the headless browser designed for AI and automation

http: streaming transfers skip the Content-Length cap and buffer

Adrià Arrufatd61864fb7a55
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 72e1fab7a..f71c74809 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -3923,6 +3923,43 @@ test "download: a multi-megabyte body arrives intact" {
try std.testing.expect(std.mem.endsWith(u8, body, ".large-hide { display: none; }"));
}
+test "download: --http-max-response-size caps memory, not a streamed body's Content-Length" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ const client = &testing.test_session.browser.http_client;
+ const max = client.max_response_size;
+ defer client.max_response_size = max;
+ client.max_response_size = 1024 * 1024;
+
+ var page = try testing.pageTest("mcp_actions.html", .{});
+ defer page.close();
+
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ const path = try testing.tmpPath(&tmp, "large.css");
+
+ const r = try testDownload(&registry, "http://127.0.0.1:9582/styles/large.css", path);
+ try std.testing.expect(std.mem.startsWith(u8, r.text, "saved 2621455 bytes"));
+}
+
+test "download: a 404 saves nothing" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ var page = try testing.pageTest("mcp_actions.html", .{});
+ defer page.close();
+
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ const path = try testing.tmpPath(&tmp, "missing.html");
+
+ const r = try testDownload(&registry, "http://127.0.0.1:9582/download_tool/missing", path);
+ try std.testing.expect(r.is_error);
+ try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);
+ try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));
+}
+
test "download: a 204 keeps the file already at path" {
var registry: NodeRegistry = .init(std.testing.allocator);
defer registry.deinit();
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index c4ab1e811..c440e9025 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -2329,9 +2329,11 @@ pub fn cancelAuthChallenge(self: *Client, transfer: *Transfer) !void {
return transfer.failAsync(err);
}
- if (transfer.getContentLength()) |cl| {
- if (cl > self.max_response_size) {
- return transfer.failAsync(error.ResponseTooLarge);
+ if (!transfer.req.streaming) {
+ if (transfer.getContentLength()) |cl| {
+ if (cl > self.max_response_size) {
+ return transfer.failAsync(error.ResponseTooLarge);
+ }
}
}
@@ -3761,8 +3763,9 @@ pub const Transfer = struct {
// A partial fetch is exempt: the cap exists to bound how much body we
// buffer, and this transfer buffers at most PARTIAL_DRAIN_MAX of it. Failing a 4 MB image
// we were never going to read would turn the size limit into a
- // spurious `error` event on a perfectly good response.
- if (opts.check_content_length and self.req.partial == null) {
+ // spurious `error` event on a perfectly good response. A stream's
+ // batch is capped in dataCallback instead.
+ if (opts.check_content_length and self.req.partial == null and !self.req.streaming) {
if (self.getContentLength()) |cl| {
if (cl > self.client.max_response_size) {
return error.ResponseTooLarge;
@@ -4345,7 +4348,8 @@ pub const Transfer = struct {
return @intCast(chunk_len);
}
- if (transfer.req.partial == null) {
+ // A stream only ever buffers its undelivered batch.
+ if (transfer.req.partial == null and !transfer.req.streaming) {
// Headers aren't materialized yet; read them off the conn.
const content_length: ?usize = blk: {
const hdr = conn.getResponseHeader("content-length", 0) orelse break :blk null;