Lightpanda: the headless browser designed for AI and automation
tools: simplify the download streaming
Adrià Arrufat6216074bda08
diff --git a/src/browser/Download.zig b/src/browser/Download.zigindex 55a454627..93616eac2 100644--- a/src/browser/Download.zig+++ b/src/browser/Download.zig@@ -166,13 +166,12 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi// `transfer.deinit()` fires no callbacks, so this is still ours on error.errdefer frame._factory.destroy(download);- var req = request(frame, url);+ var req = request(frame, url, shutdownCallback);req.ctx = download;req.header_callback = headerCallback;req.data_callback = dataCallback;req.done_callback = doneCallback;req.error_callback = errorCallback;- req.shutdown_callback = shutdownCallback;const transfer = try newTransfer(frame, req);transfer.submit() catch |err| {@@ -180,10 +179,9 @@ pub fn fromHref(frame: *Frame, href: []const u8, filename_hint: []const u8) !voi};}-/// A download is a navigation to `url` from `frame`'s page: no CORS,-/// redirects followed, credentials included, the body streamed. The caller-/// sets the callbacks.-pub fn request(frame: *Frame, url: [:0]const u8) HttpClient.Request {+/// A download is a navigation from `frame`'s page: no CORS, redirects+/// followed, credentials included.+pub fn request(frame: *Frame, url: [:0]const u8, shutdown_callback: HttpClient.Request.ShutdownCallback) HttpClient.Request {return .{.url = url,.method = .GET,@@ -193,16 +191,14 @@ pub fn request(frame: *Frame, url: [:0]const u8) HttpClient.Request {.credentials_mode = .include,.resource_type = .download,.streaming = true,- .shutdown_callback = HttpClient.noopShutdown,+ .shutdown_callback = shutdown_callback,};}-/// The transfer for a `request`, with the headers a download sends.pub fn newTransfer(frame: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {const transfer = try frame.newRequest(req);errdefer transfer.deinit();try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});- // Set common headers.try frame.headersForRequest(transfer, .{});return transfer;}diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex b36d360f1..72e1fab7a 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -22,6 +22,7 @@ const zenai = @import("zenai");const NodeRegistry = @import("../NodeRegistry.zig");const TelemetryTool = @import("../telemetry/telemetry.zig").Event.Tool;+const HttpClient = @import("../network/HttpClient.zig");const DOMNode = @import("webapi/Node.zig");const Selector = @import("webapi/selector/Selector.zig");@@ -1786,10 +1787,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *Nodreturn ToolError.InternalError;if (args.path) |path| {- const content_height = writePng(&prepared, path) catch |err| return .{- .text = arena.print("could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,- .is_error = true,- };+ const content_height = writePng(&prepared, path) catch |err| return writeError(arena, path, err);// The renderer reports the content height; a fixed strip is its own height.const height = if (prepared.opts.height == 0) content_height else prepared.opts.height;return .{ .text = arena.print("Saved {d}x{d} PNG to {s}", .{ prepared.opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };@@ -2698,8 +2696,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.// authenticates with, and attributes the request to the page for CDP.const frame = try requireFrame(session);- // Streamed into a temp file that replaces `path` only on success, so a- // failed download leaves whatever was there.+ // Replaces `path` only on success, so a failed download leaves it as it was.var af = std.Io.Dir.cwd().createFileAtomic(lp.io, args.path, .{ .replace = true }) catch |err| {return writeError(arena, args.path, err);};@@ -2707,7 +2704,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.var buf: [64 * 1024]u8 = undefined;var file_writer = af.file.writer(lp.io, &buf);- var req = Download.request(frame, args.url);+ var req = Download.request(frame, args.url, HttpClient.noopShutdown); // submitSync installs its ownreq.timeout_ms = download_timeout_ms;const transfer = Download.newTransfer(frame, req) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,@@ -2723,7 +2720,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.error.OutOfMemory => return error.OutOfMemory,error.ClientDisconnected, error.SyncWaitInterrupted => return error.Cancelled,error.UrlBlocked => return .{ .text = "url is refused by --block-urls or --adblock-lists", .is_error = true },- error.WriteFailed => return writeError(arena, args.path, file_writer.err orelse err),+ error.WriteFailed => return writeError(arena, args.path, file_writer.err.?),else => {log.debug(.browser, "download failed", .{ .err = err, .url = args.url });return .{@@ -2744,7 +2741,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.};}- file_writer.end() catch return writeError(arena, args.path, file_writer.err orelse error.WriteFailed);+ file_writer.flush() catch |err| return writeError(arena, args.path, err);af.replace(lp.io) catch |err| return writeError(arena, args.path, err);return .{@@ -3908,7 +3905,7 @@ test "download: a redirect's Set-Cookie lands in the jar and the next hop carrietry std.testing.expect(std.mem.find(u8, body, "Lightpanda/") != null);}-test "download: a multi-megabyte body streams to the file intact" {+test "download: a multi-megabyte body arrives intact" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();@@ -3923,27 +3920,9 @@ test "download: a multi-megabyte body streams to the file intact" {try std.testing.expect(std.mem.startsWith(u8, r.text, "saved 2621455 bytes (text/css)"));const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(4 * 1024 * 1024));- try std.testing.expectEqual(2621455, body.len);try std.testing.expect(std.mem.endsWith(u8, body, ".large-hide { display: none; }"));}-test "download: a 404 saves nothing" {- var registry: NodeRegistry = .init(std.testing.allocator);- defer registry.deinit();-- var page = try testing.pageTest("mcp_actions.html", .{});- defer page.close();-- var tmp = std.testing.tmpDir(.{});- defer tmp.cleanup();- const path = try testing.tmpPath(&tmp, "missing.html");-- const r = try testDownload(®istry, "http://127.0.0.1:9582/download_tool/missing", path);- try std.testing.expect(r.is_error);- try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);- try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));-}-test "download: a 204 keeps the file already at path" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();@@ -3962,7 +3941,6 @@ test "download: a 204 keeps the file already at path" {const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024));try std.testing.expectEqualStrings("good", body);- // The temp file is gone too.var it = tmp.dir.iterate();var count: usize = 0;while (try it.next(lp.io)) |_| count += 1;diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 8de7f041b..c4ab1e811 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1451,7 +1451,6 @@ pub const SyncOptions = struct {// Whether to copy the response headers into SyncResponse.headers.copy_headers: bool = false,// Write the body here instead of buffering it into SyncResponse.body.- // Pair it with Request.streaming so the transfer doesn't buffer it either.body_writer: ?*std.Io.Writer = null,};@@ -1483,16 +1482,14 @@ const SyncContext = struct {self.status = transfer.responseStatus().?;self.status_text = transfer.res.status_text;- // A streamed body's length is unknown here, and it's written out- // rather than buffered.- const buffers_body = self.options.body_writer == null;+ // A streamed body's length is 0 here, but its content type is still wanted.const body_len = transfer.bodyLen();- if (body_len == 0 and buffers_body and self.options.copy_headers == false) {+ if (body_len == 0 and self.options.body_writer == null and self.options.copy_headers == false) {return .proceed;}- const allocator = try self.bodyAllocator(if (buffers_body) body_len else 0);- if (body_len > 0 and buffers_body) {+ const allocator = try self.bodyAllocator(body_len);+ if (body_len > 0) {try self.body.ensureTotalCapacityPrecise(allocator, body_len);}@@ -2115,7 +2112,7 @@ pub const Request = struct {const DataCallback = *const fn (transfer: *Transfer, data: []const u8) anyerror!void;const DoneCallback = *const fn (ctx: *anyopaque) anyerror!void;const ErrorCallback = *const fn (ctx: *anyopaque, err: anyerror) void;- const ShutdownCallback = *const fn (ctx: *anyopaque) void;+ pub const ShutdownCallback = *const fn (ctx: *anyopaque) void;pub const ResourceType = enum {document,@@ -2952,6 +2949,7 @@ pub const Transfer = struct {const req = &self.req;req.sync = true;+ if (opts.body_writer != null) req.streaming = true;req.ctx = &sync_ctx;req.header_callback = SyncContext.headerCallback;req.data_callback = SyncContext.dataCallback;