Lightpanda: the headless browser designed for AI and automation
feat(cdp): add Network.setBlockedURLs and --block-urls request blocking
Matt Van Horn311ab3aaae11
diff --git a/src/Config.zig b/src/Config.zigindex 9609642b0..14b3ca652 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -199,6 +199,7 @@ const CommonOptions = .{.{ .name = "user_agent", .type = ?[]const u8 },.{ .name = "block_private_networks", .type = bool },.{ .name = "block_cidrs", .type = ?[]const u8 },+ .{ .name = "block_urls", .type = ?[]const u8 },.{ .name = "cookie", .type = ?[]const u8 },.{ .name = "cookie_jar", .type = ?[]const u8 },.{ .name = "storage_engine", .type = ?Storage.EngineType },@@ -689,6 +690,14 @@ pub fn blockCidrs(self: *const Config) ?[]const u8 {};}+pub fn blockedUrlPatterns(self: *const Config) ?std.mem.SplitIterator(u8, .scalar) {+ const patterns = switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.block_urls,+ else => unreachable,+ } orelse return null;+ return std.mem.splitScalar(u8, patterns, ',');+}+pub fn maxConnections(self: *const Config) u16 {return switch (self.mode) {.serve => |opts| opts.cdp_max_connections,@@ -971,6 +980,18 @@ pub fn parseArgs(allocator: Allocator, proc_args: std.process.Args) !Config {return config;}+test "Config: blockedUrlPatterns splits comma-separated patterns" {+ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{+ .block_urls = "*doubleclick*,*://*/*.png",+ } });+ defer config.deinit(std.testing.allocator);++ var patterns = config.blockedUrlPatterns().?;+ try std.testing.expectEqualStrings("*doubleclick*", patterns.next().?);+ try std.testing.expectEqualStrings("*://*/*.png", patterns.next().?);+ try std.testing.expectEqual(null, patterns.next());+}+pub fn validateUserAgent(ua: []const u8) !void {for (ua) |c| {if (!std.ascii.isPrint(c)) {diff --git a/src/Notification.zig b/src/Notification.zigindex 2d501cb54..55dcc4c1b 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -250,8 +250,13 @@ pub const RequestDone = struct {};pub const RequestFail = struct {+ pub const BlockedReason = enum {+ inspector,+ };+transfer: *Transfer,err: anyerror,+ blocked_reason: ?BlockedReason = null,};pub const RequestServedFromCache = struct {diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 89db15679..c6a2a7091 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -28,6 +28,7 @@ const Mime = @import("../../browser/Mime.zig");const Notification = @import("../../Notification.zig");const timestamp = @import("../../datetime.zig").timestamp;+const HttpClient = @import("../../network/HttpClient.zig");const Cache = @import("../../network/cache/Cache.zig");const Headers = @import("../../network/HttpClient.zig").Headers;const Transfer = @import("../../network/HttpClient.zig").Transfer;@@ -42,6 +43,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {enable,disable,setCacheDisabled,+ setBlockedURLs,setExtraHTTPHeaders,setUserAgentOverride,deleteCookies,@@ -59,6 +61,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {.enable => return enable(cmd),.disable => return disable(cmd),.setCacheDisabled => return setCacheDisabled(cmd),+ .setBlockedURLs => return setBlockedURLs(cmd),.setUserAgentOverride => return @import("emulation.zig").setUserAgentOverride(cmd),.setExtraHTTPHeaders => return setExtraHTTPHeaders(cmd),.deleteCookies => return deleteCookies(cmd),@@ -98,6 +101,16 @@ fn setCacheDisabled(cmd: *CDP.Command) !void {return cmd.sendResult(null, .{});}+fn setBlockedURLs(cmd: *CDP.Command) !void {+ const params = (try cmd.params(struct {+ urls: []const []const u8,+ })) orelse return error.InvalidParams;++ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;+ try bc.cdp.browser.http_client.setBlockedUrls(params.urls);+ return cmd.sendResult(null, .{});+}+fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {const params = (try cmd.params(struct {headers: std.json.ArrayHashMap([]const u8),@@ -316,6 +329,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request.type = "Ping",.errorText = msg.err,.canceled = false,+ .blockedReason = msg.blocked_reason,}, .{ .session_id = session_id });}@@ -981,3 +995,87 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" {try ctx.expectSentResult(null, .{ .id = 2 });try testing.expect(client.cache == &cache);}++test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {+ const filter: testing.LogFilter = .init(&.{.http});+ defer filter.deinit();++ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{+ .id = "BID-BLOCK",+ .session_id = "SID-BLOCK",+ });+ const page = try bc.session.createPage();+ const client = &bc.cdp.browser.http_client;+ defer client.setBlockedUrls(&.{}) catch unreachable;++ try ctx.processMessage(.{+ .id = 1,+ .method = "Network.enable",+ });+ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.setBlockedURLs",+ .params = .{ .urls = &[_][]const u8{"*://blocked.test/*"} },+ });+ try ctx.expectSentResult(null, .{ .id = 2 });++ const ErrorContext = struct {+ err: ?anyerror = null,++ fn callback(raw: *anyopaque, err: anyerror) void {+ const self: *@This() = @ptrCast(@alignCast(raw));+ self.err = err;+ }+ };+ var error_context: ErrorContext = .{};++ try client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 1,+ .method = .GET,+ .url = "https://blocked.test/script.js",+ .cookie_jar = null,+ .cookie_origin = "https://blocked.test/",+ .resource_type = .script,+ .notification = bc.session.notification,+ .ctx = &error_context,+ .error_callback = ErrorContext.callback,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ try ctx.expectSentEvent("Network.loadingFailed", .{+ .errorText = error.UrlBlocked,+ .blockedReason = "inspector",+ }, .{ .session_id = "SID-BLOCK" });+ try testing.expectEqual(error.UrlBlocked, error_context.err.?);++ try client.setBlockedUrls(&.{"*redirect-target*"});+ error_context.err = null;++ var redirect_request_id: [14]u8 = undefined;+ _ = std.fmt.bufPrint(&redirect_request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;++ try client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 1,+ .method = .GET,+ .url = "http://127.0.0.1:9582/redirect-no-fragment",+ .cookie_jar = null,+ .cookie_origin = "http://127.0.0.1:9582/",+ .resource_type = .script,+ .notification = bc.session.notification,+ .ctx = &error_context,+ .error_callback = ErrorContext.callback,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ try ctx.expectSentEvent("Network.loadingFailed", .{+ .requestId = &redirect_request_id,+ .errorText = error.UrlBlocked,+ .blockedReason = "inspector",+ }, .{ .session_id = "SID-BLOCK" });+ try testing.expectEqual(error.UrlBlocked, error_context.err.?);+}diff --git a/src/help.zon b/src/help.zonindex 1800a6026..e88ede3b0 100644--- a/src/help.zon+++ b/src/help.zon@@ -301,6 +301,10 @@\\ Prefix with '-' to allow (exempt from blocking).\\ e.g. --block-cidrs 10.0.0.0/8,-10.0.0.42/32\\ Can be combined with --block-private-networks.+ \\ --block-urls <LIST>+ \\ URL patterns to block, comma-separated. Patterns are matched+ \\ case-insensitively against the full URL and '*' is a wildcard.+ \\ e.g. --block-urls "*doubleclick*,*://*/*.png"\\ --block-private-networks\\ Block HTTP requests to private/internal IP addresses after DNS\\ resolution.diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex c6cab86cf..a6cbe2a02 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -35,6 +35,7 @@ const http = @import("http.zig");const Network = @import("Network.zig");const Cache = @import("cache/Cache.zig");const RobotsGate = @import("RobotsGate.zig");+const UrlBlocklist = @import("UrlBlocklist.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -186,6 +187,7 @@ serve_mode: bool,obey_robots: bool,robots: RobotsGate,+url_blocklist: ?UrlBlocklist,pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {var handles = try http.Handles.init(network.config);@@ -193,6 +195,21 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)const http_proxy = network.config.httpProxy();+ var url_blocklist: ?UrlBlocklist = null;+ if (network.config.blockedUrlPatterns()) |initial_patterns| {+ var patterns: std.ArrayList([]const u8) = .empty;+ defer patterns.deinit(allocator);++ var it = initial_patterns;+ while (it.next()) |pattern| {+ if (pattern.len > 0) try patterns.append(allocator, pattern);+ }+ if (patterns.items.len > 0) {+ url_blocklist = try UrlBlocklist.init(allocator, patterns.items);+ }+ }+ errdefer if (url_blocklist) |*blocklist| blocklist.deinit();+self.* = Client{.handles = handles,.network = network,@@ -209,6 +226,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).serve_mode = network.config.mode == .serve,.obey_robots = network.config.obeyRobots(),.robots = .{ .allocator = allocator, .network = network },+ .url_blocklist = url_blocklist,.arena_pool = &network.app.arena_pool,};}@@ -237,6 +255,7 @@ pub fn deinit(self: *Client) void {self.allocator.free(owned);}+ self.clearUrlBlocklist();self.robots.deinit();self.blocking_requests.deinit(self.allocator);self.transfers.deinit(self.allocator);@@ -334,6 +353,32 @@ pub fn changeProxy(self: *Client, proxy: ?[:0]const u8) !void {self.use_proxy = self.http_proxy != null;}+/// Replace the current URL blocklist. The caller retains ownership of+/// `patterns`; compiled copies remain valid after the CDP command arena is+/// released.+pub fn setBlockedUrls(self: *Client, patterns: []const []const u8) !void {+ const replacement: ?UrlBlocklist = if (patterns.len == 0)+ null+ else+ try UrlBlocklist.init(self.allocator, patterns);++ self.clearUrlBlocklist();+ self.url_blocklist = replacement;+}++fn clearUrlBlocklist(self: *Client) void {+ if (self.url_blocklist) |*blocklist| {+ blocklist.deinit();+ self.url_blocklist = null;+ }+}++fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {+ if (internal) return false;+ const blocklist = self.url_blocklist orelse return false;+ return blocklist.isBlocked(url);+}+pub fn newHeaders(self: *const Client) !http.Headers {const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;return http.Headers.init(ua_header);@@ -808,6 +853,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {continue :sw SubmitFrom.after_intercept;},.after_intercept => {+ if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {+ log.warn(.http, "blocked url", .{ .url = transfer.req.url });+ return transfer.failAsync(error.UrlBlocked);+ }if (try self.cacheLookup(transfer)) {// response came from the cache, we're donereturn;@@ -1397,6 +1446,14 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Ttry transfer.handleRedirect(location.value);if (!transfer.req.internal) lp.metrics.http_redirects.incr();+ if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {+ log.warn(.http, "blocked url", .{ .url = transfer.req.url });+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.failAsync(error.UrlBlocked);+ return true;+ }+const conn = transfer._conn.?;try self.handles.remove(conn);@@ -2190,6 +2247,7 @@ pub const Transfer = struct {self.req.notification.dispatch(.http_request_fail, &.{.transfer = self,.err = err,+ .blocked_reason = if (err == error.UrlBlocked) .inspector else null,});}@@ -3135,6 +3193,41 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {client.serve_mode = false;client.obey_robots = false;client.robots = .{ .allocator = testing.allocator, .network = undefined };+ client.url_blocklist = null;+}++test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ var client: Client = undefined;+ initTestClient(&client, &pool);+ defer client.clearUrlBlocklist();++ var first = [_]u8{ '*', 'f', 'i', 'r', 's', 't', '*' };+ try client.setBlockedUrls(&.{&first});+ @memset(&first, 'x');++ try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/first.js"));+ try client.setBlockedUrls(&.{"*second*"});+ try testing.expect(!client.url_blocklist.?.isBlocked("https://example.test/first.js"));+ try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/second.js"));++ try client.setBlockedUrls(&.{});+ try testing.expectEqual(null, client.url_blocklist);+}++test "HttpClient: URL blocking exempts internal transfers" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ var client: Client = undefined;+ initTestClient(&client, &pool);+ defer client.clearUrlBlocklist();++ try client.setBlockedUrls(&.{"*example.test*"});+ try testing.expect(client.isUrlBlocked("https://example.test/script.js", false));+ try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));}test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {diff --git a/src/network/UrlBlocklist.zig b/src/network/UrlBlocklist.zignew file mode 100644index 000000000..4de1dfbef--- /dev/null+++ b/src/network/UrlBlocklist.zig@@ -0,0 +1,141 @@+// Copyright (C) 2026 Lightpanda (Selecy SAS)+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");++const UrlBlocklist = @This();++allocator: std.mem.Allocator,+patterns: []const []const u8,++/// Compile and own a set of URL wildcard patterns. Patterns are normalized to+/// lowercase once so request matching remains allocation-free.+pub fn init(allocator: std.mem.Allocator, patterns: []const []const u8) !UrlBlocklist {+ const owned = try allocator.alloc([]const u8, patterns.len);+ var initialized: usize = 0;+ errdefer {+ for (owned[0..initialized]) |pattern| allocator.free(pattern);+ allocator.free(owned);+ }++ for (patterns, owned) |pattern, *compiled| {+ const normalized = try allocator.alloc(u8, pattern.len);+ for (pattern, normalized) |char, *lower| {+ lower.* = std.ascii.toLower(char);+ }+ compiled.* = normalized;+ initialized += 1;+ }++ return .{+ .allocator = allocator,+ .patterns = owned,+ };+}++pub fn deinit(self: *UrlBlocklist) void {+ for (self.patterns) |pattern| self.allocator.free(pattern);+ self.allocator.free(self.patterns);+}++pub fn isBlocked(self: *const UrlBlocklist, url: []const u8) bool {+ for (self.patterns) |pattern| {+ if (wildcardMatch(pattern, url)) return true;+ }+ return false;+}++fn wildcardMatch(pattern: []const u8, value: []const u8) bool {+ var pattern_index: usize = 0;+ var value_index: usize = 0;+ var star_index: ?usize = null;+ var star_value_index: usize = 0;++ while (value_index < value.len) {+ if (pattern_index < pattern.len and+ pattern[pattern_index] != '*' and+ pattern[pattern_index] == std.ascii.toLower(value[value_index]))+ {+ pattern_index += 1;+ value_index += 1;+ continue;+ }++ if (pattern_index < pattern.len and pattern[pattern_index] == '*') {+ star_index = pattern_index;+ pattern_index += 1;+ star_value_index = value_index;+ continue;+ }++ if (star_index) |star| {+ pattern_index = star + 1;+ star_value_index += 1;+ value_index = star_value_index;+ continue;+ }++ return false;+ }++ while (pattern_index < pattern.len and pattern[pattern_index] == '*') {+ pattern_index += 1;+ }+ return pattern_index == pattern.len;+}++test "UrlBlocklist: matches full URLs with case-insensitive wildcards" {+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{+ "*://*/*.png",+ "*doubleclick*",+ "https://example.com/exact",+ });+ defer blocklist.deinit();++ try std.testing.expect(blocklist.isBlocked("https://cdn.example.com/images/HERO.PNG"));+ try std.testing.expect(blocklist.isBlocked("https://ads.DOUBLECLICK.net/activity"));+ try std.testing.expect(blocklist.isBlocked("HTTPS://EXAMPLE.COM/EXACT"));+ try std.testing.expect(!blocklist.isBlocked("https://example.com/image.jpg"));+ try std.testing.expect(!blocklist.isBlocked("https://example.com/exact/path"));+}++test "UrlBlocklist: wildcard matching backtracks and handles empty patterns" {+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{+ "",+ "**tracker***pixel*",+ });+ defer blocklist.deinit();++ try std.testing.expect(blocklist.isBlocked(""));+ try std.testing.expect(blocklist.isBlocked("https://tracker.test/a/pixel.gif"));+ try std.testing.expect(!blocklist.isBlocked("https://anything.test/"));++ var match_all = try UrlBlocklist.init(std.testing.allocator, &.{"*"});+ defer match_all.deinit();+ try std.testing.expect(match_all.isBlocked("https://anything.test/"));++ var empty = try UrlBlocklist.init(std.testing.allocator, &.{});+ defer empty.deinit();+ try std.testing.expect(!empty.isBlocked("https://example.com/"));+}++test "UrlBlocklist: owns compiled patterns" {+ var pattern = [_]u8{ '*', 'a', 'd', 's', '*' };+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{&pattern});+ defer blocklist.deinit();++ @memset(&pattern, 'x');+ try std.testing.expect(blocklist.isBlocked("https://example.com/ads/script.js"));+}