Skip to content

Lightpanda: the headless browser designed for AI and automation

tools: stream downloads to disk and share Download's request

Adrià Arrufat0a8bb1b6cf80
diff --git a/src/browser/Download.zig b/src/browser/Download.zig
index c57254f4e..55a454627 100644
--- a/src/browser/Download.zig
+++ b/src/browser/Download.zig
@@ -166,9 +166,25 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi
// `transfer.deinit()` fires no callbacks, so this is still ours on error.
errdefer frame._factory.destroy(download);
- // Setup transfer.
- const transfer = try frame.newRequest(.{
- .ctx = download,
+ var req = request(frame, url);
+ req.ctx = download;
+ req.header_callback = headerCallback;
+ req.data_callback = dataCallback;
+ req.done_callback = doneCallback;
+ req.error_callback = errorCallback;
+ req.shutdown_callback = shutdownCallback;
+ const transfer = try newTransfer(frame, req);
+
+ transfer.submit() catch |err| {
+ log.debug(.frame, "href download", .{ .err = err, .url = url });
+ };
+}
+
+/// A download is a navigation to `url` from `frame`'s page: no CORS,
+/// redirects followed, credentials included, the body streamed. The caller
+/// sets the callbacks.
+pub fn request(frame: *Frame, url: [:0]const u8) HttpClient.Request {
+ return .{
.url = url,
.method = .GET,
.origin = frame.origin,
@@ -177,20 +193,18 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi
.credentials_mode = .include,
.resource_type = .download,
.streaming = true,
- .header_callback = headerCallback,
- .data_callback = dataCallback,
- .done_callback = doneCallback,
- .error_callback = errorCallback,
- .shutdown_callback = shutdownCallback,
- });
+ .shutdown_callback = HttpClient.noopShutdown,
+ };
+}
+
+/// The transfer for a `request`, with the headers a download sends.
+pub fn newTransfer(frame: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {
+ const transfer = try frame.newRequest(req);
errdefer transfer.deinit();
try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});
// Set common headers.
try frame.headersForRequest(transfer, .{});
-
- transfer.submit() catch |err| {
- log.debug(.frame, "href download", .{ .err = err, .url = url });
- };
+ return transfer;
}
/// Feeds the operation with bytes by either writing to a file or streaming over CDP.
@@ -329,7 +343,7 @@ fn fromRaw(ptr: *anyopaque) struct { *Download, bool } {
// Chrome downloads only 200-203; any other status is announced, then
// canceled at once, and nothing is written. 206 is only valid for a range
// request, which we never send.
-inline fn isSuccessStatus(status: u16) bool {
+pub inline fn isSuccessStatus(status: u16) bool {
return status >= 200 and status <= 203;
}
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index f8db52994..b36d360f1 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -22,10 +22,10 @@ const zenai = @import("zenai");
const NodeRegistry = @import("../NodeRegistry.zig");
const TelemetryTool = @import("../telemetry/telemetry.zig").Event.Tool;
-const HttpClient = @import("../network/HttpClient.zig");
const DOMNode = @import("webapi/Node.zig");
const Selector = @import("webapi/selector/Selector.zig");
+const Download = @import("Download.zig");
const SelectorPath = @import("SelectorPath.zig");
const Element = @import("webapi/Element.zig");
@@ -807,7 +807,7 @@ pub const Tool = enum {
),
},
.download => .{
- .description = "Fetch a URL and save the response body to a local file, sending the session's cookies so authenticated sites work. The way to get a binary or large document (PDF, spreadsheet, archive) out of the browser. Nothing is saved for a non-2xx response. Requires a loaded page — it fails before the first `goto`.",
+ .description = "Fetch a URL and save the response body to a local file, sending the session's cookies so authenticated sites work. The way to get a binary or large document (PDF, spreadsheet, archive) out of the browser. Nothing is saved unless the status is 200-203 (like Chrome), and a failed download leaves an existing file at `path` untouched. Requires a loaded page — it fails before the first `goto`.",
.summary = "Fetch a URL with the session's cookies into a file",
.input_schema = minify(
\\{
@@ -2698,42 +2698,32 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.
// authenticates with, and attributes the request to the page for CDP.
const frame = try requireFrame(session);
- const transfer = frame.newRequest(.{
- .url = args.url,
- .method = .GET,
- .origin = frame.origin,
- .initiator_origin = frame.origin,
- // A download stands in for a top-level navigation to `url`: no CORS,
- // redirects followed, engine defaults (proxy, TLS, blocklist) apply.
- .request_mode = .navigate,
- .credentials_mode = .include,
- .resource_type = .download,
- .timeout_ms = download_timeout_ms,
- // Already saved to `path`; a cached copy would be a third one.
- .skip_cache = true,
- .shutdown_callback = HttpClient.noopShutdown, // submitSync installs its own
- }) catch |err| switch (err) {
+ // Streamed into a temp file that replaces `path` only on success, so a
+ // failed download leaves whatever was there.
+ var af = std.Io.Dir.cwd().createFileAtomic(lp.io, args.path, .{ .replace = true }) catch |err| {
+ return writeError(arena, args.path, err);
+ };
+ defer af.deinit(lp.io);
+ var buf: [64 * 1024]u8 = undefined;
+ var file_writer = af.file.writer(lp.io, &buf);
+
+ var req = Download.request(frame, args.url);
+ req.timeout_ms = download_timeout_ms;
+ const transfer = Download.newTransfer(frame, req) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => {
log.debug(.browser, "download request", .{ .err = err, .url = args.url });
return ToolError.InternalError;
},
};
- {
- // Only a pre-submit failure is ours: past the submit, the client
- // deinits the transfer at delivery.
- errdefer transfer.deinit();
- try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});
- // A Referer from the current page, like a download launched from it.
- try frame.headersForRequest(transfer, .{});
- }
// Pumps the session inline: other transfers' callbacks for this frame are
// held back until the fetch settles, so nothing runs JS under us.
- var response = transfer.submitSync(.{}) catch |err| switch (err) {
+ var response = transfer.submitSync(.{ .body_writer = &file_writer.interface }) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.ClientDisconnected, error.SyncWaitInterrupted => return error.Cancelled,
error.UrlBlocked => return .{ .text = "url is refused by --block-urls or --adblock-lists", .is_error = true },
+ error.WriteFailed => return writeError(arena, args.path, file_writer.err orelse err),
else => {
log.debug(.browser, "download failed", .{ .err = err, .url = args.url });
return .{
@@ -2744,7 +2734,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.
};
defer response.deinit();
- if (response.status < 200 or response.status >= 300) {
+ if (!Download.isSuccessStatus(response.status)) {
return .{
.text = arena.print("HTTP {d} {s}; nothing saved", .{
response.status,
@@ -2754,22 +2744,25 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.
};
}
- std.Io.Dir.cwd().writeFile(lp.io, .{ .sub_path = args.path, .data = response.body.items }) catch |err| {
- return .{
- .text = arena.print("could not write {s}: {s}", .{ args.path, @errorName(err) }) catch return ToolError.OutOfMemory,
- .is_error = true,
- };
- };
+ file_writer.end() catch return writeError(arena, args.path, file_writer.err orelse error.WriteFailed);
+ af.replace(lp.io) catch |err| return writeError(arena, args.path, err);
return .{
.text = arena.print("saved {d} bytes ({s}) to {s}", .{
- response.body.items.len,
+ file_writer.logicalPos(),
response.content_type orelse "unknown content type",
absolutePath(arena, args.path),
}) catch return ToolError.OutOfMemory,
};
}
+fn writeError(arena: std.mem.Allocator, path: []const u8, err: anyerror) ToolError!ToolResult {
+ return .{
+ .text = arena.print("could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,
+ .is_error = true,
+ };
+}
+
fn requireFrame(session: *lp.Session) ToolError!*lp.Frame {
return session.currentFrame() orelse ToolError.FrameNotLoaded;
}
@@ -3915,7 +3908,26 @@ test "download: a redirect's Set-Cookie lands in the jar and the next hop carrie
try std.testing.expect(std.mem.find(u8, body, "Lightpanda/") != null);
}
-test "download: a non-2xx response saves nothing" {
+test "download: a multi-megabyte body streams to the file intact" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ var page = try testing.pageTest("mcp_actions.html", .{});
+ defer page.close();
+
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ const path = try testing.tmpPath(&tmp, "large.css");
+
+ const r = try testDownload(&registry, "http://127.0.0.1:9582/styles/large.css", path);
+ try std.testing.expect(std.mem.startsWith(u8, r.text, "saved 2621455 bytes (text/css)"));
+
+ const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(4 * 1024 * 1024));
+ try std.testing.expectEqual(2621455, body.len);
+ try std.testing.expect(std.mem.endsWith(u8, body, ".large-hide { display: none; }"));
+}
+
+test "download: a 404 saves nothing" {
var registry: NodeRegistry = .init(std.testing.allocator);
defer registry.deinit();
@@ -3931,3 +3943,28 @@ test "download: a non-2xx response saves nothing" {
try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);
try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));
}
+
+test "download: a 204 keeps the file already at path" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ var page = try testing.pageTest("mcp_actions.html", .{});
+ defer page.close();
+
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ try tmp.dir.writeFile(lp.io, .{ .sub_path = "report.pdf", .data = "good" });
+ const path = try testing.tmpPath(&tmp, "report.pdf");
+
+ const r = try testDownload(&registry, "http://127.0.0.1:9582/download_tool/no_content", path);
+ try std.testing.expectEqualStrings("HTTP 204 No Content; nothing saved", r.text);
+
+ const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024));
+ try std.testing.expectEqualStrings("good", body);
+
+ // The temp file is gone too.
+ var it = tmp.dir.iterate();
+ var count: usize = 0;
+ while (try it.next(lp.io)) |_| count += 1;
+ try std.testing.expectEqual(1, count);
+}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 896e63392..8de7f041b 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1450,6 +1450,9 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
pub const SyncOptions = struct {
// Whether to copy the response headers into SyncResponse.headers.
copy_headers: bool = false,
+ // Write the body here instead of buffering it into SyncResponse.body.
+ // Pair it with Request.streaming so the transfer doesn't buffer it either.
+ body_writer: ?*std.Io.Writer = null,
};
const SyncContext = struct {
@@ -1480,13 +1483,16 @@ const SyncContext = struct {
self.status = transfer.responseStatus().?;
self.status_text = transfer.res.status_text;
+ // A streamed body's length is unknown here, and it's written out
+ // rather than buffered.
+ const buffers_body = self.options.body_writer == null;
const body_len = transfer.bodyLen();
- if (body_len == 0 and self.options.copy_headers == false) {
+ if (body_len == 0 and buffers_body and self.options.copy_headers == false) {
return .proceed;
}
- const allocator = try self.bodyAllocator(body_len);
- if (body_len > 0) {
+ const allocator = try self.bodyAllocator(if (buffers_body) body_len else 0);
+ if (body_len > 0 and buffers_body) {
try self.body.ensureTotalCapacityPrecise(allocator, body_len);
}
@@ -1528,6 +1534,9 @@ const SyncContext = struct {
fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {
const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));
+ if (self.options.body_writer) |w| {
+ return w.writeAll(data);
+ }
try self.body.appendSlice(try self.bodyAllocator(null), data);
}
diff --git a/src/script/skill.zig b/src/script/skill.zig
index 434a303e9..0796ce99a 100644
--- a/src/script/skill.zig
+++ b/src/script/skill.zig
@@ -208,7 +208,7 @@ fn note(tool: browser_tools.Tool) []const u8 {
.waitForScript => "Re-evaluates page JS until truthy.",
.waitForState => "",
.screenshot => "`path` is required: writes a PNG of the text layout.",
- .download => "Sends the session's cookies, so authenticated downloads work. Nothing is written for a non-2xx response.",
+ .download => "Sends the session's cookies, so authenticated downloads work. Nothing is written unless the status is 200-203, and a failed download leaves `path` as it was.",
.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",
.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",
.findElement => "`name` is a case-insensitive substring, or a JS regex literal like `/sign (in|up)/` (also case-insensitive).",
diff --git a/src/test_routes.zig b/src/test_routes.zig
index fddcdf019..88d7d1ce6 100644
--- a/src/test_routes.zig
+++ b/src/test_routes.zig
@@ -1036,6 +1036,10 @@ pub fn handle(req: *std.http.Server.Request) !void {
// Fixture for the browser `download` tool tests. Its cookie is scoped to
// this path prefix, so it can't reach the handlers other tests on the
// shared session rely on.
+ if (std.mem.eql(u8, path, "/download_tool/no_content")) {
+ return req.respond("", .{ .status = .no_content });
+ }
+
if (std.mem.eql(u8, path, "/download_tool/redirect")) {
return req.respond("", .{
.status = .found,