Lightpanda: the headless browser designed for AI and automation
http: streaming transfers skip the Content-Length cap and buffer
Adrià Arrufatd61864fb7a55
diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 72e1fab7a..f71c74809 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -3923,6 +3923,43 @@ test "download: a multi-megabyte body arrives intact" {try std.testing.expect(std.mem.endsWith(u8, body, ".large-hide { display: none; }"));}+test "download: --http-max-response-size caps memory, not a streamed body's Content-Length" {+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();++ const client = &testing.test_session.browser.http_client;+ const max = client.max_response_size;+ defer client.max_response_size = max;+ client.max_response_size = 1024 * 1024;++ var page = try testing.pageTest("mcp_actions.html", .{});+ defer page.close();++ var tmp = std.testing.tmpDir(.{});+ defer tmp.cleanup();+ const path = try testing.tmpPath(&tmp, "large.css");++ const r = try testDownload(®istry, "http://127.0.0.1:9582/styles/large.css", path);+ try std.testing.expect(std.mem.startsWith(u8, r.text, "saved 2621455 bytes"));+}++test "download: a 404 saves nothing" {+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();++ var page = try testing.pageTest("mcp_actions.html", .{});+ defer page.close();++ var tmp = std.testing.tmpDir(.{});+ defer tmp.cleanup();+ const path = try testing.tmpPath(&tmp, "missing.html");++ const r = try testDownload(®istry, "http://127.0.0.1:9582/download_tool/missing", path);+ try std.testing.expect(r.is_error);+ try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);+ try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));+}+test "download: a 204 keeps the file already at path" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex c4ab1e811..c440e9025 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -2329,9 +2329,11 @@ pub fn cancelAuthChallenge(self: *Client, transfer: *Transfer) !void {return transfer.failAsync(err);}- if (transfer.getContentLength()) |cl| {- if (cl > self.max_response_size) {- return transfer.failAsync(error.ResponseTooLarge);+ if (!transfer.req.streaming) {+ if (transfer.getContentLength()) |cl| {+ if (cl > self.max_response_size) {+ return transfer.failAsync(error.ResponseTooLarge);+ }}}@@ -3761,8 +3763,9 @@ pub const Transfer = struct {// A partial fetch is exempt: the cap exists to bound how much body we// buffer, and this transfer buffers at most PARTIAL_DRAIN_MAX of it. Failing a 4 MB image// we were never going to read would turn the size limit into a- // spurious `error` event on a perfectly good response.- if (opts.check_content_length and self.req.partial == null) {+ // spurious `error` event on a perfectly good response. A stream's+ // batch is capped in dataCallback instead.+ if (opts.check_content_length and self.req.partial == null and !self.req.streaming) {if (self.getContentLength()) |cl| {if (cl > self.client.max_response_size) {return error.ResponseTooLarge;@@ -4345,7 +4348,8 @@ pub const Transfer = struct {return @intCast(chunk_len);}- if (transfer.req.partial == null) {+ // A stream only ever buffers its undelivered batch.+ if (transfer.req.partial == null and !transfer.req.streaming) {// Headers aren't materialized yet; read them off the conn.const content_length: ?usize = blk: {const hdr = conn.getResponseHeader("content-length", 0) orelse break :blk null;