Lightpanda: the headless browser designed for AI and automation
tools: simplify the download tool
Adrià Arrufatb2036f49323e
diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex f0e2d53bf..f8db52994 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -383,13 +383,12 @@ pub const Tool = enum {/// can't ship undocumented.pub fn group(self: Tool) Group {return switch (self) {- .goto, .search => .navigation,+ .goto, .search, .download => .navigation,.markdown, .html, .screenshot, .tree, .links, .nodeDetails, .findElement, .interactiveElements, .structuredData, .detectForms => .reading,.extract, .evaluate => .extraction,.click, .fill, .scroll, .hover, .press, .selectOption, .setChecked => .interaction,.waitForSelector, .waitForScript, .waitForState => .waiting,.getUrl, .getCookies, .getEnv, .consoleLogs => .state,- .download => .navigation,};}@@ -2683,23 +2682,18 @@ fn execGetCookies(arena: std.mem.Allocator, session: *lp.Session, arguments: ?streturn aw.written();}-/// Generous, unlike the read tools' 10s: the body is the payload, and a slow-/// multi-megabyte PDF is a successful download, not a timeout.-const default_download_timeout_ms: u32 = 120_000;--const DownloadParams = struct {- url: [:0]const u8,- path: []const u8,-};+/// Generous: the body is the payload, and a slow multi-megabyte PDF is a+/// successful download, not a timeout.+const download_timeout_ms: u32 = 120_000;fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.json.Value) ToolError!ToolResult {- const args = try parseArgs(DownloadParams, arena, arguments);+ const Params = struct {+ url: [:0]const u8,+ path: []const u8,+ };+ const args = try parseArgs(Params, arena, arguments);if (!isPathSafe(args.path)) return .{ .text = unsafe_path_message, .is_error = true };- if (session.browser.http_client.blocksUrl(args.url)) {- return .{ .text = "url is refused by --block-urls", .is_error = true };- }-// Nothing renders, but the frame owns the cookie jar the download// authenticates with, and attributes the request to the page for CDP.const frame = try requireFrame(session);@@ -2714,7 +2708,9 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std..request_mode = .navigate,.credentials_mode = .include,.resource_type = .download,- .timeout_ms = default_download_timeout_ms,+ .timeout_ms = download_timeout_ms,+ // Already saved to `path`; a cached copy would be a third one.+ .skip_cache = true,.shutdown_callback = HttpClient.noopShutdown, // submitSync installs its own}) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,@@ -2737,6 +2733,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.var response = transfer.submitSync(.{}) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,error.ClientDisconnected, error.SyncWaitInterrupted => return error.Cancelled,+ error.UrlBlocked => return .{ .text = "url is refused by --block-urls or --adblock-lists", .is_error = true },else => {log.debug(.browser, "download failed", .{ .err = err, .url = args.url });return .{@@ -2757,7 +2754,7 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.};}- writeFile(args.path, response.body.items) catch |err| {+ std.Io.Dir.cwd().writeFile(lp.io, .{ .sub_path = args.path, .data = response.body.items }) catch |err| {return .{.text = arena.print("could not write {s}: {s}", .{ args.path, @errorName(err) }) catch return ToolError.OutOfMemory,.is_error = true,@@ -2773,12 +2770,6 @@ fn execDownload(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.};}-fn writeFile(path: []const u8, bytes: []const u8) !void {- const file = try std.Io.Dir.cwd().createFile(lp.io, path, .{ .truncate = true });- defer file.close(lp.io);- try file.writeStreamingAll(lp.io, bytes);-}-fn requireFrame(session: *lp.Session) ToolError!*lp.Frame {return session.currentFrame() orelse ToolError.FrameNotLoaded;}@@ -3875,95 +3866,59 @@ test "markdown: a same-url page whose navigation failed is navigated again" {try std.testing.expect((try call(aa, session, ®istry, "markdown", args, .{ .source = .user })).is_error);}+fn testDownload(registry: *NodeRegistry, url: []const u8, path: []const u8) !ToolResult {+ const aa = testing.arena_allocator;+ const args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print(+ "{{\"url\":\"{s}\",\"path\":\"{s}\"}}",+ .{ url, path },+ ), .{});+ return call(aa, testing.test_session, registry, "download", args, .{ .source = .user });+}+test "download: an unsafe path is rejected without fetching" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();- const aa = testing.arena_allocator;- const args = try std.json.parseFromSliceLeaky(std.json.Value, aa,- \\{"url":"http://127.0.0.1:9582/x","path":"../escape.bin"}- , .{});- const r = try call(aa, testing.test_session, ®istry, "download", args, .{ .source = .user });+ const r = try testDownload(®istry, "http://127.0.0.1:9582/x", "../escape.bin");try std.testing.expect(r.is_error);try std.testing.expectEqualStrings(unsafe_path_message, r.text);}-test "download: session cookies ride along and Set-Cookie lands in the jar" {+test "download: a redirect's Set-Cookie lands in the jar and the next hop carries it" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();const session = testing.test_session;defer session.cookie_jar.clearRetainingCapacity();// The page makes the download same-site (same 127.0.0.1 host), so the- // Lax cookie the server hands out flows on the next call.+ // Lax cookie the redirect hands out flows on the followed hop.var page = try testing.pageTest("mcp_actions.html", .{});defer page.close();var tmp = std.testing.tmpDir(.{});defer tmp.cleanup();- const path = try testing.tmpPath(&tmp, "headers.txt");+ const path = try testing.tmpPath(&tmp, "redirected.txt");++ const r = try testDownload(®istry, "http://127.0.0.1:9582/download_tool/redirect", path);+ try std.testing.expect(std.mem.startsWith(u8, r.text, "saved "));+ try std.testing.expect(std.mem.find(u8, r.text, "text/plain") != null);- const aa = testing.arena_allocator;- var args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print(- "{{\"url\":\"http://127.0.0.1:9582/download_tool/set_cookie\",\"path\":\"{s}\"}}",- .{path},- ), .{});- const first = try call(aa, session, ®istry, "download", args, .{ .source = .user });- try std.testing.expect(first.is_error == false);- try std.testing.expect(std.mem.startsWith(u8, first.text, "saved 2 bytes"));var in_jar = false;var it = session.cookie_jar.iterator();while (it.next()) |cookie| {- if (std.mem.eql(u8, cookie.name, "lp_download")) in_jar = true;+ if (std.mem.eql(u8, cookie.name, "lp_redirect")) in_jar = true;}try std.testing.expect(in_jar);- const echo_path = try testing.tmpPath(&tmp, "echo.txt");- args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print(- "{{\"url\":\"http://127.0.0.1:9582/download_tool/echo_headers\",\"path\":\"{s}\"}}",- .{echo_path},- ), .{});- const second = try call(aa, session, ®istry, "download", args, .{ .source = .user });- try std.testing.expect(second.is_error == false);- try std.testing.expect(std.mem.find(u8, second.text, "text/plain") != null);- try std.testing.expect(std.mem.startsWith(u8, second.text, "saved "));-- const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, echo_path, aa, .limited(8192));- try std.testing.expect(std.mem.find(u8, body, "lp_download=1") != null);- try std.testing.expect(std.mem.find(u8, body, "Lightpanda/") != null);-}--test "download: a redirect's Set-Cookie lands in the jar and the next hop carries it" {- var registry: NodeRegistry = .init(std.testing.allocator);- defer registry.deinit();-- const session = testing.test_session;- defer session.cookie_jar.clearRetainingCapacity();- var page = try testing.pageTest("mcp_actions.html", .{});- defer page.close();-- var tmp = std.testing.tmpDir(.{});- defer tmp.cleanup();- const path = try testing.tmpPath(&tmp, "redirected.txt");-- const aa = testing.arena_allocator;- const args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print(- "{{\"url\":\"http://127.0.0.1:9582/download_tool/redirect\",\"path\":\"{s}\"}}",- .{path},- ), .{});- const r = try call(aa, session, ®istry, "download", args, .{ .source = .user });- try std.testing.expect(r.is_error == false);- try std.testing.expect(std.mem.startsWith(u8, r.text, "saved "));-- const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, aa, .limited(8192));+ const body = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(8192));try std.testing.expect(std.mem.find(u8, body, "lp_redirect=1") != null);+ try std.testing.expect(std.mem.find(u8, body, "Lightpanda/") != null);}test "download: a non-2xx response saves nothing" {var registry: NodeRegistry = .init(std.testing.allocator);defer registry.deinit();- const session = testing.test_session;var page = try testing.pageTest("mcp_actions.html", .{});defer page.close();@@ -3971,13 +3926,8 @@ test "download: a non-2xx response saves nothing" {defer tmp.cleanup();const path = try testing.tmpPath(&tmp, "missing.html");- const aa = testing.arena_allocator;- const args = try std.json.parseFromSliceLeaky(std.json.Value, aa, try aa.print(- "{{\"url\":\"http://127.0.0.1:9582/download_tool/missing\",\"path\":\"{s}\"}}",- .{path},- ), .{});- const r = try call(aa, session, ®istry, "download", args, .{ .source = .user });+ const r = try testDownload(®istry, "http://127.0.0.1:9582/download_tool/missing", path);try std.testing.expect(r.is_error);try std.testing.expectEqualStrings("HTTP 404 Not Found; nothing saved", r.text);- try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, aa, .limited(1024)));+ try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024)));}diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex fc357b886..580e6241a 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -39,7 +39,7 @@ fn annotations(tool: BrowserTool) protocol.ToolAnnotations {// Drains the buffer: a second call returns something else..consoleLogs => .{ .readOnlyHint = true, .destructiveHint = false, .openWorldHint = false },.goto, .search, .markdown, .html, .links, .tree, .interactiveElements, .structuredData, .detectForms => .{ .destructiveHint = false, .idempotentHint = true },- // Overwrites `path` when given.+ // Overwrite `path`..screenshot, .download => .{ .idempotentHint = true },.waitForSelector, .waitForScript, .waitForState => .{ .destructiveHint = false, .idempotentHint = true, .openWorldHint = false },.evaluate, .click, .press => .{},diff --git a/src/test_routes.zig b/src/test_routes.zigindex 0003327bb..fddcdf019 100644--- a/src/test_routes.zig+++ b/src/test_routes.zig@@ -919,7 +919,8 @@ pub fn handle(req: *std.http.Server.Request) !void {});}- if (std.mem.eql(u8, path, "/echo_headers")) {+ // The /download_tool copy sits under that fixture's cookie scope.+ if (std.mem.eql(u8, path, "/echo_headers") or std.mem.eql(u8, path, "/download_tool/echo_headers")) {if (req.head.method == .OPTIONS) {return req.respond("", .{.extra_headers = &.{@@ -1032,17 +1033,9 @@ pub fn handle(req: *std.http.Server.Request) !void {});}- // Fixture for the browser `download` tool tests. Everything is scoped to- // this path prefix, so its cookies can't reach the handlers other tests- // on the shared session rely on.- if (std.mem.eql(u8, path, "/download_tool/set_cookie")) {- return req.respond("ok", .{- .extra_headers = &.{- .{ .name = "Set-Cookie", .value = "lp_download=1; Path=/download_tool" },- },- });- }-+ // Fixture for the browser `download` tool tests. Its cookie is scoped to+ // this path prefix, so it can't reach the handlers other tests on the+ // shared session rely on.if (std.mem.eql(u8, path, "/download_tool/redirect")) {return req.respond("", .{.status = .found,@@ -1053,22 +1046,6 @@ pub fn handle(req: *std.http.Server.Request) !void {});}- if (std.mem.eql(u8, path, "/download_tool/echo_headers")) {- // Same contract as /echo_headers, under the fixture's cookie scope.- var buf: [8192]u8 = undefined;- var pos: usize = 0;- var it = req.iterateHeaders();- while (it.next()) |header| {- const line = try std.mem.print(buf[pos..], "{s}: {s}\n", .{ header.name, header.value });- pos += line.len;- }- return req.respond(buf[0..pos], .{- .extra_headers = &.{- .{ .name = "Content-Type", .value = "text/plain; charset=utf-8" },- },- });- }-if (std.mem.startsWith(u8, path, "/src/browser/tests/")) {if (std.mem.find(u8, path, "delay_ms=")) |pos| {const digits_start = pos + "delay_ms=".len;