Skip to content

Lightpanda: the headless browser designed for AI and automation

CLI option for adblock list passing + integrate adblocker to http client

Halil Durakb1b9a6457319
diff --git a/src/Config.zig b/src/Config.zig
index 699402da4..44849ef05 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -199,6 +199,7 @@ const CommonOptions = .{
.{ .name = "block_private_networks", .type = bool },
.{ .name = "block_cidrs", .type = ?[]const u8 },
.{ .name = "block_urls", .type = ?[]const u8 },
+ .{ .name = "adblock_lists", .type = ?[]const u8 },
.{ .name = "cookie", .type = ?[]const u8 },
.{ .name = "cookie_jar", .type = ?[]const u8 },
.{ .name = "disable_subframes", .type = bool },
@@ -731,6 +732,14 @@ pub fn blockedUrlPatterns(self: *const Config) ?std.mem.SplitIterator(u8, .scala
return std.mem.splitScalar(u8, patterns, ',');
}
+pub fn adblockLists(self: *const Config) ?std.mem.SplitIterator(u8, .scalar) {
+ const paths = switch (self.mode) {
+ inline .serve, .fetch, .mcp, .agent => |opts| opts.adblock_lists,
+ else => unreachable,
+ } orelse return null;
+ return std.mem.splitScalar(u8, paths, ',');
+}
+
pub fn maxConnections(self: *const Config) u16 {
return switch (self.mode) {
.serve => |opts| opts.cdp_max_connections,
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 8324ef7db..dfc2b23c8 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -36,6 +36,7 @@ const Cache = @import("cache/Cache.zig");
const RobotsGate = @import("RobotsGate.zig");
const UrlBlocklist = @import("UrlBlocklist.zig");
pub const BlockPattern = UrlBlocklist.Pattern;
+const AdBlocker = @import("../adblock/AdBlocker.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -185,6 +186,8 @@ obey_robots: bool,
robots: RobotsGate,
url_blocklist: ?UrlBlocklist,
+/// Hostname dictionaries built from `--adblock-lists` filter files.
+adblocker: ?AdBlocker,
pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {
var handles = try http.Handles.init(network.config);
@@ -207,6 +210,20 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
}
errdefer if (url_blocklist) |*blocklist| blocklist.deinit();
+ var adblocker: ?AdBlocker = null;
+ errdefer if (adblocker) |*blocker| blocker.deinit();
+ if (network.config.adblockLists()) |initial_paths| {
+ var it = initial_paths;
+ while (it.next()) |path| {
+ if (path.len == 0) continue;
+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);
+ loadAdblockList(allocator, &adblocker.?, path) catch |err| {
+ log.err(.http, "adblock list load failed", .{ .path = path, .err = err });
+ return err;
+ };
+ }
+ }
+
self.* = Client{
.handles = handles,
.network = network,
@@ -224,6 +241,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.obey_robots = network.config.obeyRobots(),
.robots = .{ .allocator = allocator, .network = network },
.url_blocklist = url_blocklist,
+ .adblocker = adblocker,
.arena_pool = &network.app.arena_pool,
};
}
@@ -253,6 +271,7 @@ pub fn deinit(self: *Client) void {
}
self.clearUrlBlocklist();
+ if (self.adblocker) |*blocker| blocker.deinit();
self.robots.deinit();
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
@@ -375,6 +394,43 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {
return blocklist.isBlocked(url);
}
+fn loadAdblockList(allocator: Allocator, blocker: *AdBlocker, path: []const u8) !void {
+ const text = try std.Io.Dir.cwd().readFileAllocOptions(
+ lp.io,
+ path,
+ allocator,
+ .limited(64 * 1024 * 1024),
+ .of(u8),
+ null,
+ );
+ defer allocator.free(text);
+
+ var reader: std.Io.Reader = .fixed(text);
+ const stats = try blocker.parse(&reader);
+ log.info(.http, "adblock list loaded", .{
+ .path = path,
+ .filters = stats.network,
+ .invalid = stats.invalid,
+ });
+}
+
+fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) bool {
+ if (internal) return false;
+ const blocker: *const AdBlocker = if (self.adblocker) |*b| b else return false;
+ const host = URL.getHostname(url);
+ if (host.len == 0 or host.len > 253) return false;
+ // The trie expects normalized (lowercase) hostnames; URLs aren't
+ // guaranteed to arrive that way.
+ var buf: [253]u8 = undefined;
+ const hostname = std.ascii.lowerString(&buf, host);
+ return blocker.matchHostname(hostname) == .blocked;
+}
+
+pub fn newHeaders(self: *const Client) !http.Headers {
+ const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;
+ return http.Headers.init(ua_header);
+}
+
pub fn getUserAgent(self: *const Client) [:0]const u8 {
return self.user_agent_override orelse self.network.config.http_headers.user_agent;
}
@@ -846,6 +902,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
return transfer.failAsync(error.UrlBlocked);
}
+ if (self.isHostAdblocked(transfer.req.url, transfer.req.internal)) {
+ log.info(.http, "adblock blocked", .{ .url = transfer.req.url });
+ return transfer.failAsync(error.AdblockBlocked);
+ }
if (try self.cacheLookup(transfer)) {
// response came from the cache, we're done
return;
@@ -3388,6 +3448,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
client.obey_robots = false;
client.robots = .{ .allocator = testing.allocator, .network = undefined };
client.url_blocklist = null;
+ client.adblocker = null;
}
test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {
diff --git a/src/network/http.zig b/src/network/http.zig
index b2631dd04..8af92c319 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -828,6 +828,7 @@ pub const ErrorReason = enum {
too_large,
aborted,
robots_blocked,
+ adblock_blocked,
other,
};
@@ -855,6 +856,7 @@ pub fn errorReason(err: anyerror) ErrorReason {
error.SyncWaitInterrupted,
=> .aborted,
error.RobotsBlocked => .robots_blocked,
+ error.AdblockBlocked => .adblock_blocked,
else => .other,
};
}