Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: introuce streaming files over CDP

Halil Durak640d483af3bf
diff --git a/src/Notification.zig b/src/Notification.zig
index 2df53d6c0..063303868 100644
--- a/src/Notification.zig
+++ b/src/Notification.zig
@@ -102,6 +102,7 @@ const EventListeners = struct {
cookie_changed: List = .{},
download_will_begin: List = .{},
download_progress: List = .{},
+ download_data: List = .{},
};
const Events = union(enum) {
@@ -136,6 +137,7 @@ const Events = union(enum) {
cookie_changed: *const CookieChanged,
download_will_begin: *const DownloadWillBegin,
download_progress: *const DownloadProgress,
+ download_data: *const DownloadData,
};
pub const EventType = std.meta.FieldEnum(Events);
@@ -345,6 +347,11 @@ pub const DownloadProgress = struct {
};
};
+pub const DownloadData = struct {
+ guid: []const u8,
+ data: []const u8,
+};
+
pub const DialogResponse = struct {
accept: bool = false,
// Set when the CDP client sent a `promptText` with `accept: true`. Memory
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 4cca95929..72c128c15 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1800,6 +1800,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {
const session = self._session;
switch (session.download_behavior) {
.allow, .allow_and_name => {},
+ .@"lp-stream-over-cdp" => return false, // TODO
.deny => return false,
}
@@ -1862,6 +1863,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {
.filename = suggested_filename,
.received = 0,
.total = total,
+ .session = self._session,
} };
if (session.download_events_enabled) {
@@ -1973,6 +1975,7 @@ pub fn downloadLink(self: *Frame, href: []const u8, file_name_hint: []const u8)
.request_mode = .navigate,
.credentials_mode = .include,
.resource_type = .download,
+ .streaming = true,
.header_callback = Ctx.headerCallback,
.data_callback = Ctx.dataCallback,
.done_callback = Ctx.doneCallback,
@@ -2157,7 +2160,7 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
},
.raw, .image => |*buf| try buf.appendSlice(self.page_arena, data),
.download => |*download| {
- download.file.writeStreamingAll(lp.io, data) catch |err| {
+ download.file.?.writeStreamingAll(lp.io, data) catch |err| {
// TODO(#2701 follow-up): surface the write failure properly. We
// can't set `_parse_state = .err` here because the next chunk
// would then hit the `.err => unreachable` branch below, and we
@@ -2281,7 +2284,7 @@ fn frameDoneCallback(ctx: *anyopaque) !void {
self.documentIsComplete();
},
.download => |*download| {
- download.file.close(lp.io);
+ download.file.?.close(lp.io);
// Capture before invalidating the union below.
const guid = download.guid;
@@ -3981,7 +3984,7 @@ const ParseState = union(enum) {
// Only reached when a frame is torn down mid-download (the normal
// completion path in frameDoneCallback already closes the file and
// transitions to .complete).
- .download => |*download| download.file.close(lp.io),
+ .download => |*download| download.file.?.close(lp.io),
else => {},
}
}
@@ -3993,12 +3996,14 @@ const ParseState = union(enum) {
const Download = struct {
// uuidv4, arena-owned. Matches the guid reported in the CDP events.
guid: []const u8,
- file: std.Io.File,
+ /// `null` if session's `download_behavior` set for streaming over CDP.
+ file: ?std.Io.File,
// suggested filename surfaced to the client, arena-owned.
filename: []const u8,
received: u64,
// from Content-Length, when the response advertised one.
total: ?u64,
+ session: *Session,
/// Initializes a `Download` for a given `url`. `maybe_suggestion` is the
/// caller's sanitized pick; `null` falls back to the `guid`.
@@ -4013,10 +4018,6 @@ const Download = struct {
log.warn(.frame, "downloads are denied", .{ .url = url });
return error.DownloadDenied;
}
- const download_path = session.download_path orelse {
- log.warn(.frame, "download without downloadPath", .{ .url = url });
- return error.WithoutDownloadPath;
- };
// `guid` is the CDP "Global Unique Identifier" which helps us find
// where to dispatch events on.
@@ -4033,19 +4034,25 @@ const Download = struct {
else
try latin1ToUtf8(frame.page_arena, name);
- const on_disk_name = switch (session.download_behavior) {
- // Stores the file under its guid.
- .allow_and_name => guid,
- else => filename,
+ const maybe_file: ?std.Io.File = switch (session.download_behavior) {
+ .allow, .allow_and_name => blk: {
+ const download_path = session.download_path orelse {
+ log.warn(.frame, "download without downloadPath", .{ .url = url });
+ return error.WithoutDownloadPath;
+ };
+ // allowAndName stores the file under its guid.
+ const on_disk_name = if (session.download_behavior == .allow_and_name) guid else filename;
+
+ const cwd = std.Io.Dir.cwd();
+ try cwd.createDirPath(lp.io, download_path);
+ var dir = try cwd.openDir(lp.io, download_path, .{});
+ defer dir.close(lp.io);
+ break :blk try dir.createFile(lp.io, on_disk_name, .{ .truncate = true });
+ },
+ .deny => unreachable,
+ .@"lp-stream-over-cdp" => null,
};
- const cwd = std.Io.Dir.cwd();
- try cwd.createDirPath(lp.io, download_path);
- var dir = try cwd.openDir(lp.io, download_path, .{});
- defer dir.close(lp.io);
-
- const file = try dir.createFile(lp.io, on_disk_name, .{ .truncate = true });
-
if (session.download_events_enabled) {
session.notification.dispatch(.download_will_begin, &.{
.frame_id = frame._frame_id,
@@ -4063,32 +4070,45 @@ const Download = struct {
return .{
.guid = guid,
- .file = file,
+ .file = maybe_file,
.filename = filename,
.received = 0,
.total = maybe_total,
+ .session = session,
};
}
/// Feeds the download via bytes.
fn feed(self: *Download, bytes: []const u8) !void {
- self.file.writeStreamingAll(lp.io, bytes) catch |err| {
- log.err(.frame, "download write", .{ .err = err, .guid = self.guid });
- return err;
- };
+ const session = self.session;
+
+ if (self.file) |file| {
+ file.writeStreamingAll(lp.io, bytes) catch |err| {
+ log.err(.frame, "download write", .{ .err = err, .guid = self.guid });
+ return err;
+ };
+ } else {
+ if (session.download_events_enabled) {
+ session.notification.dispatch(.download_data, &.{ .guid = self.guid, .data = bytes });
+ }
+ }
self.received += bytes.len;
}
// The body was fully received.
fn finish(self: *Download, frame: *Frame) void {
- self.file.close(lp.io);
- self.report(frame, .completed);
+ defer self.report(frame, .completed);
+ if (self.file) |file| {
+ file.close(lp.io);
+ }
}
// The transfer failed partway. The partial file stays on disk for now.
fn cancel(self: *Download, frame: *Frame) void {
- self.file.close(lp.io);
- self.report(frame, .canceled);
+ defer self.report(frame, .canceled);
+ if (self.file) |file| {
+ file.close(lp.io);
+ }
}
// Playwright only settles a download on a `completed` or `canceled`
diff --git a/src/browser/Session.zig b/src/browser/Session.zig
index a43fb2bb2..af47ac242 100644
--- a/src/browser/Session.zig
+++ b/src/browser/Session.zig
@@ -141,6 +141,9 @@ pub const DownloadBehavior = enum {
// The CDP `default` behavior is mapped to `deny`: we don't write downloads
// to disk unless the driver explicitly opts in with `allow`/`allowAndName`.
deny,
+ /// LP extension to `DownloadBehavior`; instead of writing files to disk,
+ /// they are streamed over CDP.
+ @"lp-stream-over-cdp",
};
const CancelHook = struct {
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index a82aa4d4a..819f4823b 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -536,6 +536,8 @@ pub const BrowserContext = struct {
intercept_state: InterceptState,
fetch_session_id: ?[]const u8 = null,
+ download_session_id: ?[]const u8 = null,
+
// Request bodies retained for Network.getRequestPostData, which can be
// called after the transfer is gone. Capped at max_post_data_size.
captured_requests: std.AutoHashMapUnmanaged(CapturedKey, []const u8),
@@ -995,11 +997,13 @@ pub const BrowserContext = struct {
pub fn downloadEventsEnable(self: *BrowserContext) !void {
try self.notification.register(.download_will_begin, self, onDownloadWillBegin);
try self.notification.register(.download_progress, self, onDownloadProgress);
+ try self.notification.register(.download_data, self, onDownloadData);
}
pub fn downloadEventsDisable(self: *BrowserContext) void {
self.notification.unregister(.download_will_begin, self);
self.notification.unregister(.download_progress, self);
+ self.notification.unregister(.download_data, self);
}
fn onDownloadWillBegin(ctx: *anyopaque, msg: *const Notification.DownloadWillBegin) !void {
@@ -1012,6 +1016,11 @@ pub const BrowserContext = struct {
return @import("domains/browser.zig").downloadProgress(self, msg);
}
+ fn onDownloadData(ctx: *anyopaque, msg: *const Notification.DownloadData) !void {
+ const self: *BrowserContext = @ptrCast(@alignCast(ctx));
+ return @import("domains/lp.zig").downloadData(self, msg);
+ }
+
pub fn webmcpEnable(self: *BrowserContext) !void {
try self.notification.register(.model_context_tool_added, self, onModelContextToolAdded);
try self.notification.register(.model_context_tool_removed, self, onModelContextToolRemoved);
diff --git a/src/server/cdp/domains/browser.zig b/src/server/cdp/domains/browser.zig
index 9ab7ffb2b..4ba9fb927 100644
--- a/src/server/cdp/domains/browser.zig
+++ b/src/server/cdp/domains/browser.zig
@@ -93,6 +93,8 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {
}
}
+ const events_enabled = params.eventsEnabled orelse false;
+
// `default` defers the choice to the browser; we map it to `deny` (no
// download is written unless a driver explicitly opts in).
const behavior: Session.DownloadBehavior = if (std.mem.eql(u8, params.behavior, "allow"))
@@ -101,6 +103,9 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {
.allow_and_name
else if (std.mem.eql(u8, params.behavior, "deny") or std.mem.eql(u8, params.behavior, "default"))
.deny
+ else if (std.mem.eql(u8, params.behavior, "lp-stream-over-cdp") and events_enabled == true)
+ // Our extension to download behavior.
+ .@"lp-stream-over-cdp"
else
return error.InvalidParams;
@@ -119,7 +124,10 @@ fn setDownloadBehavior(cmd: *CDP.Command) !void {
// downloadPath comes from the (transient) command arena; persist it on the
// session arena since Frame reads it on later navigations.
session.download_path = if (params.downloadPath) |p| try session.arena.dupe(u8, p) else null;
- session.download_events_enabled = params.eventsEnabled orelse false;
+ session.download_events_enabled = events_enabled;
+ // resolveSessionId returns the CDP-owned copy; the command's string
+ // doesn't outlive the message.
+ bc.download_session_id = if (cmd.input.session_id) |sid| cmd.cdp.resolveSessionId(sid) else null;
if (session.download_events_enabled) {
try bc.downloadEventsEnable();
@@ -140,7 +148,7 @@ pub fn downloadWillBegin(bc: *CDP.BrowserContext, event: *const Notification.Dow
.guid = event.guid,
.url = event.url,
.suggestedFilename = event.suggested_filename,
- }, .{});
+ }, .{ .session_id = bc.download_session_id });
}
// https://chromedevtools.github.io/devtools-protocol/tot/Browser/#event-downloadProgress
@@ -155,7 +163,7 @@ pub fn downloadProgress(bc: *CDP.BrowserContext, msg: *const Notification.Downlo
.completed => "completed",
.canceled => "canceled",
},
- }, .{});
+ }, .{ .session_id = bc.download_session_id });
}
fn getWindowForTarget(cmd: *CDP.Command) !void {
diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zig
index fc33529fc..fc46637a2 100644
--- a/src/server/cdp/domains/lp.zig
+++ b/src/server/cdp/domains/lp.zig
@@ -20,6 +20,7 @@ const std = @import("std");
const lp = @import("lightpanda");
const CDP = @import("../CDP.zig");
+const Base64Body = @import("network.zig").Base64Body;
const Robots = @import("../../../network/Robots.zig");
const DOMNode = @import("../../../browser/webapi/Node.zig");
@@ -73,6 +74,13 @@ pub fn processMessage(cmd: *CDP.Command) !void {
}
}
+pub fn downloadData(bc: *CDP.BrowserContext, msg: *const lp.Notification.DownloadData) !void {
+ return bc.cdp.sendEvent("LP.downloadData", .{
+ .guid = msg.guid,
+ .data = Base64Body{ .data = msg.data },
+ }, .{ .session_id = bc.download_session_id });
+}
+
fn version(cmd: *CDP.Command) !void {
return cmd.sendResult(.{
.version = lp.build_config.version,