Lightpanda: the headless browser designed for AI and automation
cdp: support downloading files over CDP
Halil Durak3e7c80653737
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 8515463bd..4cca95929 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1882,6 +1882,113 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {return true;}+/// Downloads from a link asynchronously. In order to download to finish,+/// contrary to Chrome, `Frame` must live throughout the download process.+/// The function may or may not use `file_name_hint`, hence it's a _hint_.+pub fn downloadLink(self: *Frame, href: []const u8, file_name_hint: []const u8) !void {+ // Return an error?+ if (self._session.download_behavior == .deny) {+ return;+ }++ const url = try URL.resolve(self.page_arena, self.navigationBase(), href, .{ .encoding = self.charset });+ // Wraps everything `Transfer` of this function needs.+ const Ctx = struct {+ frame: *Frame,+ /// Owned by `page_arena`, not used after `headerCallback`.+ hint: []const u8,+ /// Set by `headerCallback`; null until then, and if it aborts.+ download: ?Download = null,++ fn headerCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {+ const ctx: *@This() = @ptrCast(@alignCast(transfer.req.ctx));+ const frame = ctx.frame;+ // The final URL, after any redirects.+ const final_url = transfer.req.url;++ const disposition_name: ?[]const u8 = blk: {+ for (transfer.responseHeaders()) |header| {+ if (std.mem.eql(u8, header.name, "content-disposition")) {+ break :blk dispositionFilename(header);+ }+ }+ break :blk null;+ };++ const maybe_suggestion = disposition_name orelse+ sanitizeFilename(ctx.hint) orelse (try urlBasename(frame.page_arena, final_url));+ const maybe_download_size: ?u64 = if (transfer.getContentLength()) |cl| cl else null;++ ctx.download = Download.init(frame, final_url, maybe_suggestion, maybe_download_size) catch |err| {+ log.err(.frame, "download init", .{ .err = err });+ return .abort;+ };+ return .proceed;+ }++ fn dataCallback(transfer: *HttpClient.Transfer, bytes: []const u8) !void {+ const ctx: *@This() = @ptrCast(@alignCast(transfer.req.ctx));+ return ctx.download.?.feed(bytes);+ }++ // This must not fail! A `done_callback` error is routed to+ // `error_callback`, which would destroy `ctx` a second time.+ fn doneCallback(raw_ctx: *anyopaque) !void {+ const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));+ defer ctx.frame._factory.destroy(ctx);+ ctx.download.?.finish(ctx.frame);+ }++ fn errorCallback(raw_ctx: *anyopaque, err: anyerror) void {+ const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));+ defer ctx.frame._factory.destroy(ctx);+ log.warn(.frame, "link download", .{ .err = err });+ // This is null when it failed at or before the headers.+ // No downloadWillBegin was sent, so there's nothing to cancel.+ if (ctx.download) |*download| download.cancel(ctx.frame);+ }++ // Frame teardown or a superseding navigation. Unlike Chrome, downloads+ // don't outlive their `Frame`, so report it canceled. Must not run JS.+ fn shutdownCallback(raw_ctx: *anyopaque) void {+ const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));+ defer ctx.frame._factory.destroy(ctx);+ if (ctx.download) |*download| download.cancel(ctx.frame);+ }+ };++ const ctx = try self._factory.create(Ctx{+ .frame = self,+ .hint = try self.page_arena.dupe(u8, file_name_hint),+ });+ // `transfer.deinit()` fires no callbacks, so this is still ours on error.+ errdefer self._factory.destroy(ctx);++ const transfer = try self.newRequest(.{+ .ctx = ctx,+ .url = url,+ .method = .GET,+ .origin = self.origin,+ .initiator_origin = self.origin,+ .request_mode = .navigate,+ .credentials_mode = .include,+ .resource_type = .download,+ .header_callback = Ctx.headerCallback,+ .data_callback = Ctx.dataCallback,+ .done_callback = Ctx.doneCallback,+ .error_callback = Ctx.errorCallback,+ .shutdown_callback = Ctx.shutdownCallback,+ });+ errdefer transfer.deinit();+ try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});+ // Set common headers.+ try self.headersForRequest(transfer, .{});++ transfer.submit() catch |err| {+ log.debug(.frame, "link download", .{ .err = err, .url = url });+ };+}+// Extracts the filename from a Content-Disposition header, handling the quoted,// unquoted, and RFC 5987 (filename*=charset''value) forms. Path components are// stripped so the result is always a bare basename.@@ -3892,6 +3999,111 @@ const Download = struct {received: u64,// from Content-Length, when the response advertised one.total: ?u64,++ /// Initializes a `Download` for a given `url`. `maybe_suggestion` is the+ /// caller's sanitized pick; `null` falls back to the `guid`.+ fn init(frame: *Frame, url: []const u8, maybe_suggestion: ?[]const u8, maybe_total: ?u64) !Download {+ const uuidv4 = @import("../id.zig").uuidv4;+ const latin1ToUtf8 = @import("../string.zig").latin1ToUtf8;++ const session = frame._session;+ // setDownloadBehavior can change while an `<a download>` fetch is+ // in-flight.+ if (session.download_behavior == .deny) {+ log.warn(.frame, "downloads are denied", .{ .url = url });+ return error.DownloadDenied;+ }+ const download_path = session.download_path orelse {+ log.warn(.frame, "download without downloadPath", .{ .url = url });+ return error.WithoutDownloadPath;+ };++ // `guid` is the CDP "Global Unique Identifier" which helps us find+ // where to dispatch events on.+ var guid_buf: [36]u8 = undefined;+ uuidv4(&guid_buf);+ const guid = try frame.page_arena.dupe(u8, &guid_buf);++ // The suggestion may point into the response headers, which don't+ // outlive the transfer. Content-Disposition filenames can be in a+ // legacy encoding (e.g. Shift_JIS) but must yield valid UTF-8.+ const name = maybe_suggestion orelse guid;+ const filename = if (std.unicode.utf8ValidateSlice(name))+ try frame.page_arena.dupe(u8, name)+ else+ try latin1ToUtf8(frame.page_arena, name);++ const on_disk_name = switch (session.download_behavior) {+ // Stores the file under its guid.+ .allow_and_name => guid,+ else => filename,+ };++ const cwd = std.Io.Dir.cwd();+ try cwd.createDirPath(lp.io, download_path);+ var dir = try cwd.openDir(lp.io, download_path, .{});+ defer dir.close(lp.io);++ const file = try dir.createFile(lp.io, on_disk_name, .{ .truncate = true });++ if (session.download_events_enabled) {+ session.notification.dispatch(.download_will_begin, &.{+ .frame_id = frame._frame_id,+ .guid = guid,+ .url = url,+ .suggested_filename = filename,+ });+ session.notification.dispatch(.download_progress, &.{+ .guid = guid,+ .total_bytes = maybe_total orelse 0,+ .received_bytes = 0,+ .state = .in_progress,+ });+ }++ return .{+ .guid = guid,+ .file = file,+ .filename = filename,+ .received = 0,+ .total = maybe_total,+ };+ }++ /// Feeds the download via bytes.+ fn feed(self: *Download, bytes: []const u8) !void {+ self.file.writeStreamingAll(lp.io, bytes) catch |err| {+ log.err(.frame, "download write", .{ .err = err, .guid = self.guid });+ return err;+ };+ self.received += bytes.len;+ }++ // The body was fully received.+ fn finish(self: *Download, frame: *Frame) void {+ self.file.close(lp.io);+ self.report(frame, .completed);+ }++ // The transfer failed partway. The partial file stays on disk for now.+ fn cancel(self: *Download, frame: *Frame) void {+ self.file.close(lp.io);+ self.report(frame, .canceled);+ }++ // Playwright only settles a download on a `completed` or `canceled`+ // downloadProgress; without one, download.path() and friends never return.+ fn report(self: *const Download, frame: *Frame, state: lp.Notification.DownloadProgress.State) void {+ const session = frame._session;+ if (session.download_events_enabled) {+ session.notification.dispatch(.download_progress, &.{+ .guid = self.guid,+ .total_bytes = self.total orelse self.received,+ .received_bytes = self.received,+ .state = state,+ });+ }+ }};const LoadState = enum {diff --git a/src/browser/frame/user_input.zig b/src/browser/frame/user_input.zigindex 378bb52d3..44de53fed 100644--- a/src/browser/frame/user_input.zig+++ b/src/browser/frame/user_input.zig@@ -1390,9 +1390,11 @@ fn followLink(frame: *Frame, target: *Node, element: *Element, href: []const u8,// a script clicks a link in another frame.const owner = target.ownerFrame(frame) orelse return;- if (URL.isJavascriptUrl(href) == false and try element.hasAttribute(comptime .wrap("download"), frame)) {- log.debug(.browser, "a.download", .{ .type = owner._type, .url = owner.url });- return;+ // <a href="/images/panda.png" download>+ const maybe_download = try element.getAttribute(comptime .wrap("download"), frame);+ if (URL.isJavascriptUrl(href) == false and maybe_download != null) {+ const download = maybe_download orelse unreachable;+ return owner.downloadLink(href, download.str());}const target_frame = blk: {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 23fb2025d..9c215a7ca 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -2107,6 +2107,9 @@ pub const Request = struct {worker,// navigator.sendBeaconbeacon,+ // An <a download> fetch. The owner is the frame that started it, and+ // that frame keeps its document.+ download,// Allowed Values: Document, Stylesheet, Image, Media, Font, Script,// TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest,@@ -2124,6 +2127,7 @@ pub const Request = struct {.text_track => "TextTrack",.worker => "Script",.beacon => "Ping",+ .download => "Other",};}};@@ -3421,6 +3425,8 @@ pub const Transfer = struct {};fn timingTarget(self: *const Transfer) ?TimingTarget {+ // A download isn't a resource of the document that started it.+ if (self.req.resource_type == .download) return null;const owner = self.owner orelse return null;const target: *const Owner = if (self.req.resource_type == .document)owner.parent orelse return null@@ -3504,6 +3510,7 @@ pub const Transfer = struct {.image => "img",.text_track => "track",.beacon => "beacon",+ .download => unreachable, // timingTarget skips downloads},.protocol = t.protocol,.start = t.start_time,diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zigindex a481e7ea2..59b9b4bc0 100644--- a/src/network/adblock/Engine.zig+++ b/src/network/adblock/Engine.zig@@ -148,7 +148,7 @@ pub const Request = struct {.stylesheet => .{ .stylesheet = true },.xhr, .fetch => .{ .xmlhttprequest = true },.image => .{ .image = true },- .eventsource, .text_track => .{ .other = true },+ .eventsource, .text_track, .download => .{ .other = true },.worker => .{ .script = true },.beacon => .{ .ping = true },};diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zigindex 283c6b63a..e76cd36cd 100644--- a/src/server/cdp/domains/fetch.zig+++ b/src/server/cdp/domains/fetch.zig@@ -689,6 +689,7 @@ pub const InterceptState = struct {.image => .Image,.text_track => .TextTrack,.beacon => .Ping,+ .download => .Other,};}diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex 2b5d2d885..c16eff46a 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -790,7 +790,7 @@ const ResponseWriter = struct {fn initialPriority(resource_type: HttpClient.Request.ResourceType) []const u8 {return switch (resource_type) {- .document, .stylesheet => "VeryHigh",+ .document, .stylesheet, .download => "VeryHigh",.script, .worker, .xhr, .fetch, .eventsource => "High",.image, .text_track => "Low",.beacon => "VeryLow",