Skip to content

Lightpanda: the headless browser designed for AI and automation

feat(cdp): add Network.setBlockedURLs and --block-urls request blocking

Matt Van Horn311ab3aaae11
diff --git a/src/Config.zig b/src/Config.zig
index 9609642b0..14b3ca652 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -199,6 +199,7 @@ const CommonOptions = .{
.{ .name = "user_agent", .type = ?[]const u8 },
.{ .name = "block_private_networks", .type = bool },
.{ .name = "block_cidrs", .type = ?[]const u8 },
+ .{ .name = "block_urls", .type = ?[]const u8 },
.{ .name = "cookie", .type = ?[]const u8 },
.{ .name = "cookie_jar", .type = ?[]const u8 },
.{ .name = "storage_engine", .type = ?Storage.EngineType },
@@ -689,6 +690,14 @@ pub fn blockCidrs(self: *const Config) ?[]const u8 {
};
}
+pub fn blockedUrlPatterns(self: *const Config) ?std.mem.SplitIterator(u8, .scalar) {
+ const patterns = switch (self.mode) {
+ inline .serve, .fetch, .mcp, .agent => |opts| opts.block_urls,
+ else => unreachable,
+ } orelse return null;
+ return std.mem.splitScalar(u8, patterns, ',');
+}
+
pub fn maxConnections(self: *const Config) u16 {
return switch (self.mode) {
.serve => |opts| opts.cdp_max_connections,
@@ -971,6 +980,18 @@ pub fn parseArgs(allocator: Allocator, proc_args: std.process.Args) !Config {
return config;
}
+test "Config: blockedUrlPatterns splits comma-separated patterns" {
+ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{
+ .block_urls = "*doubleclick*,*://*/*.png",
+ } });
+ defer config.deinit(std.testing.allocator);
+
+ var patterns = config.blockedUrlPatterns().?;
+ try std.testing.expectEqualStrings("*doubleclick*", patterns.next().?);
+ try std.testing.expectEqualStrings("*://*/*.png", patterns.next().?);
+ try std.testing.expectEqual(null, patterns.next());
+}
+
pub fn validateUserAgent(ua: []const u8) !void {
for (ua) |c| {
if (!std.ascii.isPrint(c)) {
diff --git a/src/Notification.zig b/src/Notification.zig
index 2d501cb54..55dcc4c1b 100644
--- a/src/Notification.zig
+++ b/src/Notification.zig
@@ -250,8 +250,13 @@ pub const RequestDone = struct {
};
pub const RequestFail = struct {
+ pub const BlockedReason = enum {
+ inspector,
+ };
+
transfer: *Transfer,
err: anyerror,
+ blocked_reason: ?BlockedReason = null,
};
pub const RequestServedFromCache = struct {
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 89db15679..c6a2a7091 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -28,6 +28,7 @@ const Mime = @import("../../browser/Mime.zig");
const Notification = @import("../../Notification.zig");
const timestamp = @import("../../datetime.zig").timestamp;
+const HttpClient = @import("../../network/HttpClient.zig");
const Cache = @import("../../network/cache/Cache.zig");
const Headers = @import("../../network/HttpClient.zig").Headers;
const Transfer = @import("../../network/HttpClient.zig").Transfer;
@@ -42,6 +43,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
enable,
disable,
setCacheDisabled,
+ setBlockedURLs,
setExtraHTTPHeaders,
setUserAgentOverride,
deleteCookies,
@@ -59,6 +61,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
.enable => return enable(cmd),
.disable => return disable(cmd),
.setCacheDisabled => return setCacheDisabled(cmd),
+ .setBlockedURLs => return setBlockedURLs(cmd),
.setUserAgentOverride => return @import("emulation.zig").setUserAgentOverride(cmd),
.setExtraHTTPHeaders => return setExtraHTTPHeaders(cmd),
.deleteCookies => return deleteCookies(cmd),
@@ -98,6 +101,16 @@ fn setCacheDisabled(cmd: *CDP.Command) !void {
return cmd.sendResult(null, .{});
}
+fn setBlockedURLs(cmd: *CDP.Command) !void {
+ const params = (try cmd.params(struct {
+ urls: []const []const u8,
+ })) orelse return error.InvalidParams;
+
+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+ try bc.cdp.browser.http_client.setBlockedUrls(params.urls);
+ return cmd.sendResult(null, .{});
+}
+
fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {
const params = (try cmd.params(struct {
headers: std.json.ArrayHashMap([]const u8),
@@ -316,6 +329,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request
.type = "Ping",
.errorText = msg.err,
.canceled = false,
+ .blockedReason = msg.blocked_reason,
}, .{ .session_id = session_id });
}
@@ -981,3 +995,87 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" {
try ctx.expectSentResult(null, .{ .id = 2 });
try testing.expect(client.cache == &cache);
}
+
+test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {
+ const filter: testing.LogFilter = .init(&.{.http});
+ defer filter.deinit();
+
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{
+ .id = "BID-BLOCK",
+ .session_id = "SID-BLOCK",
+ });
+ const page = try bc.session.createPage();
+ const client = &bc.cdp.browser.http_client;
+ defer client.setBlockedUrls(&.{}) catch unreachable;
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Network.enable",
+ });
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.setBlockedURLs",
+ .params = .{ .urls = &[_][]const u8{"*://blocked.test/*"} },
+ });
+ try ctx.expectSentResult(null, .{ .id = 2 });
+
+ const ErrorContext = struct {
+ err: ?anyerror = null,
+
+ fn callback(raw: *anyopaque, err: anyerror) void {
+ const self: *@This() = @ptrCast(@alignCast(raw));
+ self.err = err;
+ }
+ };
+ var error_context: ErrorContext = .{};
+
+ try client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 1,
+ .method = .GET,
+ .url = "https://blocked.test/script.js",
+ .cookie_jar = null,
+ .cookie_origin = "https://blocked.test/",
+ .resource_type = .script,
+ .notification = bc.session.notification,
+ .ctx = &error_context,
+ .error_callback = ErrorContext.callback,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ try ctx.expectSentEvent("Network.loadingFailed", .{
+ .errorText = error.UrlBlocked,
+ .blockedReason = "inspector",
+ }, .{ .session_id = "SID-BLOCK" });
+ try testing.expectEqual(error.UrlBlocked, error_context.err.?);
+
+ try client.setBlockedUrls(&.{"*redirect-target*"});
+ error_context.err = null;
+
+ var redirect_request_id: [14]u8 = undefined;
+ _ = std.fmt.bufPrint(&redirect_request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+ try client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 1,
+ .method = .GET,
+ .url = "http://127.0.0.1:9582/redirect-no-fragment",
+ .cookie_jar = null,
+ .cookie_origin = "http://127.0.0.1:9582/",
+ .resource_type = .script,
+ .notification = bc.session.notification,
+ .ctx = &error_context,
+ .error_callback = ErrorContext.callback,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ try ctx.expectSentEvent("Network.loadingFailed", .{
+ .requestId = &redirect_request_id,
+ .errorText = error.UrlBlocked,
+ .blockedReason = "inspector",
+ }, .{ .session_id = "SID-BLOCK" });
+ try testing.expectEqual(error.UrlBlocked, error_context.err.?);
+}
diff --git a/src/help.zon b/src/help.zon
index 1800a6026..e88ede3b0 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -301,6 +301,10 @@
\\ Prefix with '-' to allow (exempt from blocking).
\\ e.g. --block-cidrs 10.0.0.0/8,-10.0.0.42/32
\\ Can be combined with --block-private-networks.
+ \\ --block-urls <LIST>
+ \\ URL patterns to block, comma-separated. Patterns are matched
+ \\ case-insensitively against the full URL and '*' is a wildcard.
+ \\ e.g. --block-urls "*doubleclick*,*://*/*.png"
\\ --block-private-networks
\\ Block HTTP requests to private/internal IP addresses after DNS
\\ resolution.
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index c6cab86cf..a6cbe2a02 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -35,6 +35,7 @@ const http = @import("http.zig");
const Network = @import("Network.zig");
const Cache = @import("cache/Cache.zig");
const RobotsGate = @import("RobotsGate.zig");
+const UrlBlocklist = @import("UrlBlocklist.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -186,6 +187,7 @@ serve_mode: bool,
obey_robots: bool,
robots: RobotsGate,
+url_blocklist: ?UrlBlocklist,
pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {
var handles = try http.Handles.init(network.config);
@@ -193,6 +195,21 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
const http_proxy = network.config.httpProxy();
+ var url_blocklist: ?UrlBlocklist = null;
+ if (network.config.blockedUrlPatterns()) |initial_patterns| {
+ var patterns: std.ArrayList([]const u8) = .empty;
+ defer patterns.deinit(allocator);
+
+ var it = initial_patterns;
+ while (it.next()) |pattern| {
+ if (pattern.len > 0) try patterns.append(allocator, pattern);
+ }
+ if (patterns.items.len > 0) {
+ url_blocklist = try UrlBlocklist.init(allocator, patterns.items);
+ }
+ }
+ errdefer if (url_blocklist) |*blocklist| blocklist.deinit();
+
self.* = Client{
.handles = handles,
.network = network,
@@ -209,6 +226,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.serve_mode = network.config.mode == .serve,
.obey_robots = network.config.obeyRobots(),
.robots = .{ .allocator = allocator, .network = network },
+ .url_blocklist = url_blocklist,
.arena_pool = &network.app.arena_pool,
};
}
@@ -237,6 +255,7 @@ pub fn deinit(self: *Client) void {
self.allocator.free(owned);
}
+ self.clearUrlBlocklist();
self.robots.deinit();
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
@@ -334,6 +353,32 @@ pub fn changeProxy(self: *Client, proxy: ?[:0]const u8) !void {
self.use_proxy = self.http_proxy != null;
}
+/// Replace the current URL blocklist. The caller retains ownership of
+/// `patterns`; compiled copies remain valid after the CDP command arena is
+/// released.
+pub fn setBlockedUrls(self: *Client, patterns: []const []const u8) !void {
+ const replacement: ?UrlBlocklist = if (patterns.len == 0)
+ null
+ else
+ try UrlBlocklist.init(self.allocator, patterns);
+
+ self.clearUrlBlocklist();
+ self.url_blocklist = replacement;
+}
+
+fn clearUrlBlocklist(self: *Client) void {
+ if (self.url_blocklist) |*blocklist| {
+ blocklist.deinit();
+ self.url_blocklist = null;
+ }
+}
+
+fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {
+ if (internal) return false;
+ const blocklist = self.url_blocklist orelse return false;
+ return blocklist.isBlocked(url);
+}
+
pub fn newHeaders(self: *const Client) !http.Headers {
const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;
return http.Headers.init(ua_header);
@@ -808,6 +853,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
continue :sw SubmitFrom.after_intercept;
},
.after_intercept => {
+ if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
+ log.warn(.http, "blocked url", .{ .url = transfer.req.url });
+ return transfer.failAsync(error.UrlBlocked);
+ }
if (try self.cacheLookup(transfer)) {
// response came from the cache, we're done
return;
@@ -1397,6 +1446,14 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
try transfer.handleRedirect(location.value);
if (!transfer.req.internal) lp.metrics.http_redirects.incr();
+ if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
+ log.warn(.http, "blocked url", .{ .url = transfer.req.url });
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.failAsync(error.UrlBlocked);
+ return true;
+ }
+
const conn = transfer._conn.?;
try self.handles.remove(conn);
@@ -2190,6 +2247,7 @@ pub const Transfer = struct {
self.req.notification.dispatch(.http_request_fail, &.{
.transfer = self,
.err = err,
+ .blocked_reason = if (err == error.UrlBlocked) .inspector else null,
});
}
@@ -3135,6 +3193,41 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
client.serve_mode = false;
client.obey_robots = false;
client.robots = .{ .allocator = testing.allocator, .network = undefined };
+ client.url_blocklist = null;
+}
+
+test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ var client: Client = undefined;
+ initTestClient(&client, &pool);
+ defer client.clearUrlBlocklist();
+
+ var first = [_]u8{ '*', 'f', 'i', 'r', 's', 't', '*' };
+ try client.setBlockedUrls(&.{&first});
+ @memset(&first, 'x');
+
+ try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/first.js"));
+ try client.setBlockedUrls(&.{"*second*"});
+ try testing.expect(!client.url_blocklist.?.isBlocked("https://example.test/first.js"));
+ try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/second.js"));
+
+ try client.setBlockedUrls(&.{});
+ try testing.expectEqual(null, client.url_blocklist);
+}
+
+test "HttpClient: URL blocking exempts internal transfers" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ var client: Client = undefined;
+ initTestClient(&client, &pool);
+ defer client.clearUrlBlocklist();
+
+ try client.setBlockedUrls(&.{"*example.test*"});
+ try testing.expect(client.isUrlBlocked("https://example.test/script.js", false));
+ try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));
}
test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {
diff --git a/src/network/UrlBlocklist.zig b/src/network/UrlBlocklist.zig
new file mode 100644
index 000000000..4de1dfbef
--- /dev/null
+++ b/src/network/UrlBlocklist.zig
@@ -0,0 +1,141 @@
+// Copyright (C) 2026 Lightpanda (Selecy SAS)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+
+const UrlBlocklist = @This();
+
+allocator: std.mem.Allocator,
+patterns: []const []const u8,
+
+/// Compile and own a set of URL wildcard patterns. Patterns are normalized to
+/// lowercase once so request matching remains allocation-free.
+pub fn init(allocator: std.mem.Allocator, patterns: []const []const u8) !UrlBlocklist {
+ const owned = try allocator.alloc([]const u8, patterns.len);
+ var initialized: usize = 0;
+ errdefer {
+ for (owned[0..initialized]) |pattern| allocator.free(pattern);
+ allocator.free(owned);
+ }
+
+ for (patterns, owned) |pattern, *compiled| {
+ const normalized = try allocator.alloc(u8, pattern.len);
+ for (pattern, normalized) |char, *lower| {
+ lower.* = std.ascii.toLower(char);
+ }
+ compiled.* = normalized;
+ initialized += 1;
+ }
+
+ return .{
+ .allocator = allocator,
+ .patterns = owned,
+ };
+}
+
+pub fn deinit(self: *UrlBlocklist) void {
+ for (self.patterns) |pattern| self.allocator.free(pattern);
+ self.allocator.free(self.patterns);
+}
+
+pub fn isBlocked(self: *const UrlBlocklist, url: []const u8) bool {
+ for (self.patterns) |pattern| {
+ if (wildcardMatch(pattern, url)) return true;
+ }
+ return false;
+}
+
+fn wildcardMatch(pattern: []const u8, value: []const u8) bool {
+ var pattern_index: usize = 0;
+ var value_index: usize = 0;
+ var star_index: ?usize = null;
+ var star_value_index: usize = 0;
+
+ while (value_index < value.len) {
+ if (pattern_index < pattern.len and
+ pattern[pattern_index] != '*' and
+ pattern[pattern_index] == std.ascii.toLower(value[value_index]))
+ {
+ pattern_index += 1;
+ value_index += 1;
+ continue;
+ }
+
+ if (pattern_index < pattern.len and pattern[pattern_index] == '*') {
+ star_index = pattern_index;
+ pattern_index += 1;
+ star_value_index = value_index;
+ continue;
+ }
+
+ if (star_index) |star| {
+ pattern_index = star + 1;
+ star_value_index += 1;
+ value_index = star_value_index;
+ continue;
+ }
+
+ return false;
+ }
+
+ while (pattern_index < pattern.len and pattern[pattern_index] == '*') {
+ pattern_index += 1;
+ }
+ return pattern_index == pattern.len;
+}
+
+test "UrlBlocklist: matches full URLs with case-insensitive wildcards" {
+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{
+ "*://*/*.png",
+ "*doubleclick*",
+ "https://example.com/exact",
+ });
+ defer blocklist.deinit();
+
+ try std.testing.expect(blocklist.isBlocked("https://cdn.example.com/images/HERO.PNG"));
+ try std.testing.expect(blocklist.isBlocked("https://ads.DOUBLECLICK.net/activity"));
+ try std.testing.expect(blocklist.isBlocked("HTTPS://EXAMPLE.COM/EXACT"));
+ try std.testing.expect(!blocklist.isBlocked("https://example.com/image.jpg"));
+ try std.testing.expect(!blocklist.isBlocked("https://example.com/exact/path"));
+}
+
+test "UrlBlocklist: wildcard matching backtracks and handles empty patterns" {
+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{
+ "",
+ "**tracker***pixel*",
+ });
+ defer blocklist.deinit();
+
+ try std.testing.expect(blocklist.isBlocked(""));
+ try std.testing.expect(blocklist.isBlocked("https://tracker.test/a/pixel.gif"));
+ try std.testing.expect(!blocklist.isBlocked("https://anything.test/"));
+
+ var match_all = try UrlBlocklist.init(std.testing.allocator, &.{"*"});
+ defer match_all.deinit();
+ try std.testing.expect(match_all.isBlocked("https://anything.test/"));
+
+ var empty = try UrlBlocklist.init(std.testing.allocator, &.{});
+ defer empty.deinit();
+ try std.testing.expect(!empty.isBlocked("https://example.com/"));
+}
+
+test "UrlBlocklist: owns compiled patterns" {
+ var pattern = [_]u8{ '*', 'a', 'd', 's', '*' };
+ var blocklist = try UrlBlocklist.init(std.testing.allocator, &.{&pattern});
+ defer blocklist.deinit();
+
+ @memset(&pattern, 'x');
+ try std.testing.expect(blocklist.isBlocked("https://example.com/ads/script.js"));
+}