Lightpanda: the headless browser designed for AI and automation
CLI option for adblock list passing + integrate adblocker to http client
Halil Durakb1b9a6457319
diff --git a/src/Config.zig b/src/Config.zigindex 699402da4..44849ef05 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -199,6 +199,7 @@ const CommonOptions = .{.{ .name = "block_private_networks", .type = bool },.{ .name = "block_cidrs", .type = ?[]const u8 },.{ .name = "block_urls", .type = ?[]const u8 },+ .{ .name = "adblock_lists", .type = ?[]const u8 },.{ .name = "cookie", .type = ?[]const u8 },.{ .name = "cookie_jar", .type = ?[]const u8 },.{ .name = "disable_subframes", .type = bool },@@ -731,6 +732,14 @@ pub fn blockedUrlPatterns(self: *const Config) ?std.mem.SplitIterator(u8, .scalareturn std.mem.splitScalar(u8, patterns, ',');}+pub fn adblockLists(self: *const Config) ?std.mem.SplitIterator(u8, .scalar) {+ const paths = switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.adblock_lists,+ else => unreachable,+ } orelse return null;+ return std.mem.splitScalar(u8, paths, ',');+}+pub fn maxConnections(self: *const Config) u16 {return switch (self.mode) {.serve => |opts| opts.cdp_max_connections,diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 8324ef7db..dfc2b23c8 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -36,6 +36,7 @@ const Cache = @import("cache/Cache.zig");const RobotsGate = @import("RobotsGate.zig");const UrlBlocklist = @import("UrlBlocklist.zig");pub const BlockPattern = UrlBlocklist.Pattern;+const AdBlocker = @import("../adblock/AdBlocker.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -185,6 +186,8 @@ obey_robots: bool,robots: RobotsGate,url_blocklist: ?UrlBlocklist,+/// Hostname dictionaries built from `--adblock-lists` filter files.+adblocker: ?AdBlocker,pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {var handles = try http.Handles.init(network.config);@@ -207,6 +210,20 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)}errdefer if (url_blocklist) |*blocklist| blocklist.deinit();+ var adblocker: ?AdBlocker = null;+ errdefer if (adblocker) |*blocker| blocker.deinit();+ if (network.config.adblockLists()) |initial_paths| {+ var it = initial_paths;+ while (it.next()) |path| {+ if (path.len == 0) continue;+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);+ loadAdblockList(allocator, &adblocker.?, path) catch |err| {+ log.err(.http, "adblock list load failed", .{ .path = path, .err = err });+ return err;+ };+ }+ }+self.* = Client{.handles = handles,.network = network,@@ -224,6 +241,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).obey_robots = network.config.obeyRobots(),.robots = .{ .allocator = allocator, .network = network },.url_blocklist = url_blocklist,+ .adblocker = adblocker,.arena_pool = &network.app.arena_pool,};}@@ -253,6 +271,7 @@ pub fn deinit(self: *Client) void {}self.clearUrlBlocklist();+ if (self.adblocker) |*blocker| blocker.deinit();self.robots.deinit();self.blocking_requests.deinit(self.allocator);self.transfers.deinit(self.allocator);@@ -375,6 +394,43 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {return blocklist.isBlocked(url);}+fn loadAdblockList(allocator: Allocator, blocker: *AdBlocker, path: []const u8) !void {+ const text = try std.Io.Dir.cwd().readFileAllocOptions(+ lp.io,+ path,+ allocator,+ .limited(64 * 1024 * 1024),+ .of(u8),+ null,+ );+ defer allocator.free(text);++ var reader: std.Io.Reader = .fixed(text);+ const stats = try blocker.parse(&reader);+ log.info(.http, "adblock list loaded", .{+ .path = path,+ .filters = stats.network,+ .invalid = stats.invalid,+ });+}++fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) bool {+ if (internal) return false;+ const blocker: *const AdBlocker = if (self.adblocker) |*b| b else return false;+ const host = URL.getHostname(url);+ if (host.len == 0 or host.len > 253) return false;+ // The trie expects normalized (lowercase) hostnames; URLs aren't+ // guaranteed to arrive that way.+ var buf: [253]u8 = undefined;+ const hostname = std.ascii.lowerString(&buf, host);+ return blocker.matchHostname(hostname) == .blocked;+}++pub fn newHeaders(self: *const Client) !http.Headers {+ const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;+ return http.Headers.init(ua_header);+}+pub fn getUserAgent(self: *const Client) [:0]const u8 {return self.user_agent_override orelse self.network.config.http_headers.user_agent;}@@ -846,6 +902,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {log.warn(.http, "blocked url", .{ .url = transfer.req.url });return transfer.failAsync(error.UrlBlocked);}+ if (self.isHostAdblocked(transfer.req.url, transfer.req.internal)) {+ log.info(.http, "adblock blocked", .{ .url = transfer.req.url });+ return transfer.failAsync(error.AdblockBlocked);+ }if (try self.cacheLookup(transfer)) {// response came from the cache, we're donereturn;@@ -3388,6 +3448,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {client.obey_robots = false;client.robots = .{ .allocator = testing.allocator, .network = undefined };client.url_blocklist = null;+ client.adblocker = null;}test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {diff --git a/src/network/http.zig b/src/network/http.zigindex b2631dd04..8af92c319 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -828,6 +828,7 @@ pub const ErrorReason = enum {too_large,aborted,robots_blocked,+ adblock_blocked,other,};@@ -855,6 +856,7 @@ pub fn errorReason(err: anyerror) ErrorReason {error.SyncWaitInterrupted,=> .aborted,error.RobotsBlocked => .robots_blocked,+ error.AdblockBlocked => .adblock_blocked,else => .other,};}