Lightpanda: the headless browser designed for AI and automation
tools: stream downloads to disk and share Download's request
Adrià Arrufat0a8bb1b6cf80
diff --git a/src/browser/Download.zig b/src/browser/Download.zigindex c57254f4e..55a454627 100644--- a/src/browser/Download.zig+++ b/src/browser/Download.zig@@ -166,9 +166,25 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi// `transfer.deinit()` fires no callbacks, so this is still ours on error.errdefer frame._factory.destroy(download);- // Setup transfer.- const transfer = try frame.newRequest(.{- .ctx = download,+ var req = request(frame, url);+ req.ctx = download;+ req.header_callback = headerCallback;+ req.data_callback = dataCallback;+ req.done_callback = doneCallback;+ req.error_callback = errorCallback;+ req.shutdown_callback = shutdownCallback;+ const transfer = try newTransfer(frame, req);++ transfer.submit() catch |err| {+ log.debug(.frame, "href download", .{ .err = err, .url = url });+ };+}++/// A download is a navigation to `url` from `frame`'s page: no CORS,+/// redirects followed, credentials included, the body streamed. The caller+/// sets the callbacks.+pub fn request(frame: *Frame, url: [:0]const u8) HttpClient.Request {+ return .{.url = url,.method = .GET,.origin = frame.origin,@@ -177,20 +193,18 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi.credentials_mode = .include,.resource_type = .download,.streaming = true,- .header_callback = headerCallback,- .data_callback = dataCallback,- .done_callback = doneCallback,- .error_callback = errorCallback,- .shutdown_callback = shutdownCallback,- });+ .shutdown_callback = HttpClient.noopShutdown,+ };+}++/// The transfer for a `request`, with the headers a download sends.+pub fn newTransfer(frame: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {+ const transfer = try frame.newRequest(req);errdefer transfer.deinit();try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});// Set common headers.try frame.headersForRequest(transfer, .{});-- transfer.submit() catch |err| {- log.debug(.frame, "href download", .{ .err = err, .url = url });- };+ return transfer;}/// Feeds the operation with bytes by either writing to a file or streaming over CDP.@@ -329,7 +343,7 @@ fn fromRaw(ptr: *anyopaque) struct { *Download, bool } {// Chrome downloads only 200-203; any other status is announced, then// canceled at once, and nothing is written. 206 is only valid for a range// request, which we never send.-inline fn isSuccessStatus(status: u16) bool {+pub inline fn isSuccessStatus(status: u16) bool {return status >= 200 and status <= 203;}diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex f8db52994..b36d360f1 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -22,10 +22,10 @@ const zenai = @import("zenai");const NodeRegistry = @import("../NodeRegistry.zig");const TelemetryTool = @import("../telemetry/telemetry.zig").Event.Tool;-const HttpClient = @import("../network/HttpClient.zig");const DOMNode = @import("webapi/Node.zig");const Selector = @import("webapi/selector/Selector.zig");+const Download = @import("Download.zig");const SelectorPath = @import("SelectorPath.zig");const Element = @import("webapi/Element.zig");@@ -807,7 +807,7 @@ pub const Tool = enum {),},.download => .{- .description = "Fetch a URL and save the response body to a local file, sending the session's cookies so authenticated sites work. The way to get a binary or large document (PDF, spreadsheet, archive) out of the browser. Nothing is saved for a non-2xx response. Requires a loaded page — it fails before the first `goto`.",+ .description = "Fetch a URL and save the response body to a local file, sending the session's cookies so authenticated sites work. The way to get a binary or large document (PDF, spreadsheet, archive) out of the browser. Nothing is saved unless the status is 200-203 (like Chrome), and a failed download leaves an existing file at `path` untouched. Requires a loaded page — it fails before the first `goto`.",.summary = "Fetch a URL with the session's cookies into a file",.input_schema = minify(\\{@@ -2698,42 +2698,32 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.// authenticates with, and attributes the request to the page for CDP.const frame = try requireFrame(session);- const transfer = frame.newRequest(.{- .url = args.url,- .method = .GET,- .origin = frame.origin,- .initiator_origin = frame.origin,- // A download stands in for a top-level navigation to `url`: no CORS,- // redirects followed, engine defaults (proxy, TLS, blocklist) apply.- .request_mode = .navigate,- .credentials_mode = .include,- .resource_type = .download,- .timeout_ms = download_timeout_ms,- // Already saved to `path`; a cached copy would be a third one.- .skip_cache = true,- .shutdown_callback = HttpClient.noopShutdown, // submitSync installs its own- }) catch |err| switch (err) {+ // Streamed into a temp file that replaces `path` only on success, so a+ // failed download leaves whatever was there.+ var af = std.Io.Dir.cwd().createFileAtomic(lp.io, args.path, .{ .replace = true }) catch |err| {+ return writeError(arena, args.path, err);+ };+ defer af.deinit(lp.io);+ var buf: [64 * 1024]u8 = undefined;+ var file_writer = af.file.writer(lp.io, &buf);++ var req = Download.request(frame, args.url);+ req.timeout_ms = download_timeout_ms;+ const transfer = Download.newTransfer(frame, req) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,else => {log.debug(.browser, "download request", .{ .err = err, .url = args.url });return ToolError.InternalError;},};- {- // Only a pre-submit failure is ours: past the submit, the client- // deinits the transfer at delivery.- errdefer transfer.deinit();- try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});- // A Referer from the current page, like a download launched from it.- try frame.headersForRequest(transfer, .{});- }// Pumps the session inline: other transfers' callbacks for this frame are// held back until the fetch settles, so nothing runs JS under us.- var response = transfer.submitSync(.{}) catch |err| switch (err) {+ var response = transfer.submitSync(.{ .body_writer = &file_writer.interface }) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,error.ClientDisconnected, error.SyncWaitInterrupted => return error.Cancelled,error.UrlBlocked => return .{ .text = "url is refused by --block-urls or --adblock-lists", .is_error = true },+ error.WriteFailed => return writeError(arena, args.path, file_writer.err orelse err),else => {log.debug(.browser, "download failed", .{ .err = err, .url = args.url });return .{@@ -2744,7 +2734,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.};defer response.deinit();- if (response.status < 200 or response.status >= 300) {+ if (!Download.isSuccessStatus(response.status)) {return .{.text = arena.print("HTTP {d} {s}; nothing saved", .{response.status,@@ -2754,22 +2744,25 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.};}- std.Io.Dir.cwd().writeFile(lp.io, .{ .sub_path = args.path, .data = response.body.items }) catch |err| {- return .{- .text = arena.print("could not write {s}: {s}", .{ args.path, @errorName(err) }) catch return ToolError.OutOfMemory,- .is_error = true,- };- };+ file_writer.end() catch return writeError(arena, args.path, file_writer.err orelse error.WriteFailed);+ af.replace(lp.io) catch |err| return writeError(arena, args.path, err);return .{.text = arena.print("saved {d} bytes ({s}) to {s}", .{- response.body.items.len,+ file_writer.logicalPos(),response.content_type orelse "unknown content type",absolutePath(arena, args.path),}) catch return ToolError.OutOfMemory,};}+fn writeError(arena: std.mem.Allocator, path: []const u8, err: anyerror) ToolError!ToolResult {+ return .{+ .text = arena.print("could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,+ .is_error = true,+ };+}+fn requireFrame(session: *lp.Session) ToolError!*lp.Frame {return session.currentFrame() orelse ToolError.FrameNotLoaded;}@@ -3915,7 +3908,26 @@ test "download: a redirect's Set-Cookie lands in the jar and the next hop carrietry std.testing.expect(std.mem.find(u8, body, "Lightpanda/") != null);}-test "download: a non-2xx response saves nothing" {+test "download: a multi-megabyte body streams to the file intact" {+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();++ var page = try testing.pageTest("mcp_actions.html", .{});+ defer page.close();++ var tmp = std.testing.tmpDir(.{});+ defer tmp.cleanup();+ const path = try testing.tmpPath(&tmp, "large.css");++ const r = try testDownload(®istry, "http://127.0.0.1:9582/styles/large.css", path);+ try std.testing.expect(std.mem.startsWith(u8, r.text, "saved 2621455 bytes (text/css)"));++ const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(4 * 1024 * 1024));+ try std.testing.expectEqual(2621455, body.len);+ try std.testing.expect(std.mem.endsWith(u8, body, ".large-hide { display: none; }"));+}++test "download: a 404 saves nothing" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();@@ -3931,3 +3943,28 @@ test "download: a non-2xx response saves nothing" {try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));}++test "download: a 204 keeps the file already at path" {+ var registry: NodeRegistry = .init(std.testing.allocator);+ defer registry.deinit();++ var page = try testing.pageTest("mcp_actions.html", .{});+ defer page.close();++ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ try tmp.dir.writeFile(lp.io, .{ .sub_path = "report.pdf", .data = "good" });+ const path = try testing.tmpPath(&tmp, "report.pdf");++ const r = try testDownload(®istry, "http://127.0.0.1:9582/download_tool/no_content", path);+ try std.testing.expectEqualStrings("HTTP 204 No Content; nothing saved", r.text);++ const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024));+ try std.testing.expectEqualStrings("good", body);++ // The temp file is gone too.+ var it = tmp.dir.iterate();+ var count: usize = 0;+ while (try it.next(lp.io)) |_| count += 1;+ try std.testing.expectEqual(1, count);+}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 896e63392..8de7f041b 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1450,6 +1450,9 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {pub const SyncOptions = struct {// Whether to copy the response headers into SyncResponse.headers.copy_headers: bool = false,+ // Write the body here instead of buffering it into SyncResponse.body.+ // Pair it with Request.streaming so the transfer doesn't buffer it either.+ body_writer: ?*std.Io.Writer = null,};const SyncContext = struct {@@ -1480,13 +1483,16 @@ const SyncContext = struct {self.status = transfer.responseStatus().?;self.status_text = transfer.res.status_text;+ // A streamed body's length is unknown here, and it's written out+ // rather than buffered.+ const buffers_body = self.options.body_writer == null;const body_len = transfer.bodyLen();- if (body_len == 0 and self.options.copy_headers == false) {+ if (body_len == 0 and buffers_body and self.options.copy_headers == false) {return .proceed;}- const allocator = try self.bodyAllocator(body_len);- if (body_len > 0) {+ const allocator = try self.bodyAllocator(if (buffers_body) body_len else 0);+ if (body_len > 0 and buffers_body) {try self.body.ensureTotalCapacityPrecise(allocator, body_len);}@@ -1528,6 +1534,9 @@ const SyncContext = struct {fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));+ if (self.options.body_writer) |w| {+ return w.writeAll(data);+ }try self.body.appendSlice(try self.bodyAllocator(null), data);}diff --git a/src/script/skill.zig b/src/script/skill.zigindex 434a303e9..0796ce99a 100644--- a/src/script/skill.zig+++ b/src/script/skill.zig@@ -208,7 +208,7 @@ fn note(tool: browser_tools.Tool) []const u8 {.waitForScript => "Re-evaluates page JS until truthy.",.waitForState => "",.screenshot => "`path` is required: writes a PNG of the text layout.",- .download => "Sends the session's cookies, so authenticated downloads work. Nothing is written for a non-2xx response.",+ .download => "Sends the session's cookies, so authenticated downloads work. Nothing is written unless the status is 200-203, and a failed download leaves `path` as it was.",.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",.findElement => "`name` is a case-insensitive substring, or a JS regex literal like `/sign (in|up)/` (also case-insensitive).",diff --git a/src/test_routes.zig b/src/test_routes.zigindex fddcdf019..88d7d1ce6 100644--- a/src/test_routes.zig+++ b/src/test_routes.zig@@ -1036,6 +1036,10 @@ pub fn handle(req: *std.http.Server.Request) !void {// Fixture for the browser `download` tool tests. Its cookie is scoped to// this path prefix, so it can't reach the handlers other tests on the// shared session rely on.+ if (std.mem.eql(u8, path, "/download_tool/no_content")) {+ return req.respond("", .{ .status = .no_content });+ }+if (std.mem.eql(u8, path, "/download_tool/redirect")) {return req.respond("", .{.status = .found,