Lightpanda: the headless browser designed for AI and automation
cdp: introuce streaming files over CDP
Halil Durak640d483af3bf
diff --git a/src/Notification.zig b/src/Notification.zigindex 2df53d6c0..063303868 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -102,6 +102,7 @@ const EventListeners = struct {cookie_changed: List = .{},download_will_begin: List = .{},download_progress: List = .{},+ download_data: List = .{},};const Events = union(enum) {@@ -136,6 +137,7 @@ const Events = union(enum) {cookie_changed: *const CookieChanged,download_will_begin: *const DownloadWillBegin,download_progress: *const DownloadProgress,+ download_data: *const DownloadData,};pub const EventType = std.meta.FieldEnum(Events);@@ -345,6 +347,11 @@ pub const DownloadProgress = struct {};};+pub const DownloadData = struct {+ guid: []const u8,+ data: []const u8,+};+pub const DialogResponse = struct {accept: bool = false,// Set when the CDP client sent a `promptText` with `accept: true`. Memorydiff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 4cca95929..72c128c15 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1800,6 +1800,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {const session = self._session;switch (session.download_behavior) {.allow, .allow_and_name => {},+ .@"lp-stream-over-cdp" => return false, // TODO.deny => return false,}@@ -1862,6 +1863,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {.filename = suggested_filename,.received = 0,.total = total,+ .session = self._session,} };if (session.download_events_enabled) {@@ -1973,6 +1975,7 @@ pub fn downloadLink(self: *Frame, href: []const u8, file_name_hint: []const u8).request_mode = .navigate,.credentials_mode = .include,.resource_type = .download,+ .streaming = true,.header_callback = Ctx.headerCallback,.data_callback = Ctx.dataCallback,.done_callback = Ctx.doneCallback,@@ -2157,7 +2160,7 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {},.raw, .image => |*buf| try buf.appendSlice(self.page_arena, data),.download => |*download| {- download.file.writeStreamingAll(lp.io, data) catch |err| {+ download.file.?.writeStreamingAll(lp.io, data) catch |err| {// TODO(#2701 follow-up): surface the write failure properly. We// can't set `_parse_state = .err` here because the next chunk// would then hit the `.err => unreachable` branch below, and we@@ -2281,7 +2284,7 @@ fn frameDoneCallback(ctx: *anyopaque) !void {self.documentIsComplete();},.download => |*download| {- download.file.close(lp.io);+ download.file.?.close(lp.io);// Capture before invalidating the union below.const guid = download.guid;@@ -3981,7 +3984,7 @@ const ParseState = union(enum) {// Only reached when a frame is torn down mid-download (the normal// completion path in frameDoneCallback already closes the file and// transitions to .complete).- .download => |*download| download.file.close(lp.io),+ .download => |*download| download.file.?.close(lp.io),else => {},}}@@ -3993,12 +3996,14 @@ const ParseState = union(enum) {const Download = struct {// uuidv4, arena-owned. Matches the guid reported in the CDP events.guid: []const u8,- file: std.Io.File,+ /// `null` if session's `download_behavior` set for streaming over CDP.+ file: ?std.Io.File,// suggested filename surfaced to the client, arena-owned.filename: []const u8,received: u64,// from Content-Length, when the response advertised one.total: ?u64,+ session: *Session,/// Initializes a `Download` for a given `url`. `maybe_suggestion` is the/// caller's sanitized pick; `null` falls back to the `guid`.@@ -4013,10 +4018,6 @@ const Download = struct {log.warn(.frame, "downloads are denied", .{ .url = url });return error.DownloadDenied;}- const download_path = session.download_path orelse {- log.warn(.frame, "download without downloadPath", .{ .url = url });- return error.WithoutDownloadPath;- };// `guid` is the CDP "Global Unique Identifier" which helps us find// where to dispatch events on.@@ -4033,19 +4034,25 @@ const Download = struct {elsetry latin1ToUtf8(frame.page_arena, name);- const on_disk_name = switch (session.download_behavior) {- // Stores the file under its guid.- .allow_and_name => guid,- else => filename,+ const maybe_file: ?std.Io.File = switch (session.download_behavior) {+ .allow, .allow_and_name => blk: {+ const download_path = session.download_path orelse {+ log.warn(.frame, "download without downloadPath", .{ .url = url });+ return error.WithoutDownloadPath;+ };+ // allowAndName stores the file under its guid.+ const on_disk_name = if (session.download_behavior == .allow_and_name) guid else filename;++ const cwd = std.Io.Dir.cwd();+ try cwd.createDirPath(lp.io, download_path);+ var dir = try cwd.openDir(lp.io, download_path, .{});+ defer dir.close(lp.io);+ break :blk try dir.createFile(lp.io, on_disk_name, .{ .truncate = true });+ },+ .deny => unreachable,+ .@"lp-stream-over-cdp" => null,};- const cwd = std.Io.Dir.cwd();- try cwd.createDirPath(lp.io, download_path);- var dir = try cwd.openDir(lp.io, download_path, .{});- defer dir.close(lp.io);-- const file = try dir.createFile(lp.io, on_disk_name, .{ .truncate = true });-if (session.download_events_enabled) {session.notification.dispatch(.download_will_begin, &.{.frame_id = frame._frame_id,@@ -4063,32 +4070,45 @@ const Download = struct {return .{.guid = guid,- .file = file,+ .file = maybe_file,.filename = filename,.received = 0,.total = maybe_total,+ .session = session,};}/// Feeds the download via bytes.fn feed(self: *Download, bytes: []const u8) !void {- self.file.writeStreamingAll(lp.io, bytes) catch |err| {- log.err(.frame, "download write", .{ .err = err, .guid = self.guid });- return err;- };+ const session = self.session;++ if (self.file) |file| {+ file.writeStreamingAll(lp.io, bytes) catch |err| {+ log.err(.frame, "download write", .{ .err = err, .guid = self.guid });+ return err;+ };+ } else {+ if (session.download_events_enabled) {+ session.notification.dispatch(.download_data, &.{ .guid = self.guid, .data = bytes });+ }+ }self.received += bytes.len;}// The body was fully received.fn finish(self: *Download, frame: *Frame) void {- self.file.close(lp.io);- self.report(frame, .completed);+ defer self.report(frame, .completed);+ if (self.file) |file| {+ file.close(lp.io);+ }}// The transfer failed partway. The partial file stays on disk for now.fn cancel(self: *Download, frame: *Frame) void {- self.file.close(lp.io);- self.report(frame, .canceled);+ defer self.report(frame, .canceled);+ if (self.file) |file| {+ file.close(lp.io);+ }}// Playwright only settles a download on a `completed` or `canceled`diff --git a/src/browser/Session.zig b/src/browser/Session.zigindex a43fb2bb2..af47ac242 100644--- a/src/browser/Session.zig+++ b/src/browser/Session.zig@@ -141,6 +141,9 @@ pub const DownloadBehavior = enum {// The CDP `default` behavior is mapped to `deny`: we don't write downloads// to disk unless the driver explicitly opts in with `allow`/`allowAndName`.deny,+ /// LP extension to `DownloadBehavior`; instead of writing files to disk,+ /// they are streamed over CDP.+ @"lp-stream-over-cdp",};const CancelHook = struct {diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex a82aa4d4a..819f4823b 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -536,6 +536,8 @@ pub const BrowserContext = struct {intercept_state: InterceptState,fetch_session_id: ?[]const u8 = null,+ download_session_id: ?[]const u8 = null,+// Request bodies retained for Network.getRequestPostData, which can be// called after the transfer is gone. Capped at max_post_data_size.captured_requests: std.AutoHashMapUnmanaged(CapturedKey, []const u8),@@ -995,11 +997,13 @@ pub const BrowserContext = struct {pub fn downloadEventsEnable(self: *BrowserContext) !void {try self.notification.register(.download_will_begin, self, onDownloadWillBegin);try self.notification.register(.download_progress, self, onDownloadProgress);+ try self.notification.register(.download_data, self, onDownloadData);}pub fn downloadEventsDisable(self: *BrowserContext) void {self.notification.unregister(.download_will_begin, self);self.notification.unregister(.download_progress, self);+ self.notification.unregister(.download_data, self);}fn onDownloadWillBegin(ctx: *anyopaque, msg: *const Notification.DownloadWillBegin) !void {@@ -1012,6 +1016,11 @@ pub const BrowserContext = struct {return @import("domains/browser.zig").downloadProgress(self, msg);}+ fn onDownloadData(ctx: *anyopaque, msg: *const Notification.DownloadData) !void {+ const self: *BrowserContext = @ptrCast(@alignCast(ctx));+ return @import("domains/lp.zig").downloadData(self, msg);+ }+pub fn webmcpEnable(self: *BrowserContext) !void {try self.notification.register(.model_context_tool_added, self, onModelContextToolAdded);try self.notification.register(.model_context_tool_removed, self, onModelContextToolRemoved);diff --git a/src/server/cdp/domains/browser.zig b/src/server/cdp/domains/browser.zigindex 9ab7ffb2b..4ba9fb927 100644--- a/src/server/cdp/domains/browser.zig+++ b/src/server/cdp/domains/browser.zig@@ -93,6 +93,8 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {}}+ const events_enabled = params.eventsEnabled orelse false;+// `default` defers the choice to the browser; we map it to `deny` (no// download is written unless a driver explicitly opts in).const behavior: Session.DownloadBehavior = if (std.mem.eql(u8, params.behavior, "allow"))@@ -101,6 +103,9 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {.allow_and_nameelse if (std.mem.eql(u8, params.behavior, "deny") or std.mem.eql(u8, params.behavior, "default")).deny+ else if (std.mem.eql(u8, params.behavior, "lp-stream-over-cdp") and events_enabled == true)+ // Our extension to download behavior.+ .@"lp-stream-over-cdp"elsereturn error.InvalidParams;@@ -119,7 +124,10 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {// downloadPath comes from the (transient) command arena; persist it on the// session arena since Frame reads it on later navigations.session.download_path = if (params.downloadPath) |p| try session.arena.dupe(u8, p) else null;- session.download_events_enabled = params.eventsEnabled orelse false;+ session.download_events_enabled = events_enabled;+ // resolveSessionId returns the CDP-owned copy; the command's string+ // doesn't outlive the message.+ bc.download_session_id = if (cmd.input.session_id) |sid| cmd.cdp.resolveSessionId(sid) else null;if (session.download_events_enabled) {try bc.downloadEventsEnable();@@ -140,7 +148,7 @@ pub fn downloadWillBegin(bc: *CDP.BrowserContext, event: *const Notification.Dow.guid = event.guid,.url = event.url,.suggestedFilename = event.suggested_filename,- }, .{});+ }, .{ .session_id = bc.download_session_id });}// https://chromedevtools.github.io/devtools-protocol/tot/Browser/#event-downloadProgress@@ -155,7 +163,7 @@ pub fn downloadProgress(bc: *CDP.BrowserContext, msg: *const Notification.Downlo.completed => "completed",.canceled => "canceled",},- }, .{});+ }, .{ .session_id = bc.download_session_id });}fn getWindowForTarget(cmd: *CDP.Command) !void {diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zigindex fc33529fc..fc46637a2 100644--- a/src/server/cdp/domains/lp.zig+++ b/src/server/cdp/domains/lp.zig@@ -20,6 +20,7 @@ const std = @import("std");const lp = @import("lightpanda");const CDP = @import("../CDP.zig");+const Base64Body = @import("network.zig").Base64Body;const Robots = @import("../../../network/Robots.zig");const DOMNode = @import("../../../browser/webapi/Node.zig");@@ -73,6 +74,13 @@ pub fn processMessage(cmd: *CDP.Command) !void {}}+pub fn downloadData(bc: *CDP.BrowserContext, msg: *const lp.Notification.DownloadData) !void {+ return bc.cdp.sendEvent("LP.downloadData", .{+ .guid = msg.guid,+ .data = Base64Body{ .data = msg.data },+ }, .{ .session_id = bc.download_session_id });+}+fn version(cmd: *CDP.Command) !void {return cmd.sendResult(.{.version = lp.build_config.version,