Lightpanda: the headless browser designed for AI and automation
load and parse block list once and share between HTTP clients
Halil Durakc111fc5675a7
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 29346cf18..34e7c0d7b 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -35,8 +35,9 @@ const Network = @import("Network.zig");const Cache = @import("cache/Cache.zig");const RobotsGate = @import("RobotsGate.zig");const UrlBlocklist = @import("UrlBlocklist.zig");+const AdBlocker = @import("adblock/AdBlocker.zig");+pub const BlockPattern = UrlBlocklist.Pattern;-const AdBlocker = @import("../adblock/AdBlocker.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -186,8 +187,10 @@ obey_robots: bool,robots: RobotsGate,url_blocklist: ?UrlBlocklist,-/// Hostname dictionaries built from `--adblock-lists` filter files.-adblocker: ?AdBlocker,+/// Hostname dictionaries built from `--adblock-lists`. Owned by Network,+/// parsed once at startup and read-only from here on, so every client+/// shares the one copy.+adblocker: ?*const AdBlocker,pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {var handles = try http.Handles.init(network.config);@@ -210,20 +213,6 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)}errdefer if (url_blocklist) |*blocklist| blocklist.deinit();- var adblocker: ?AdBlocker = null;- errdefer if (adblocker) |*blocker| blocker.deinit();- if (network.config.adblockLists()) |initial_paths| {- var it = initial_paths;- while (it.next()) |path| {- if (path.len == 0) continue;- if (adblocker == null) adblocker = try AdBlocker.init(allocator);- loadAdblockList(allocator, &adblocker.?, path) catch |err| {- log.err(.http, "adblock list load failed", .{ .path = path, .err = err });- return err;- };- }- }-self.* = Client{.handles = handles,.network = network,@@ -241,7 +230,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).obey_robots = network.config.obeyRobots(),.robots = .{ .allocator = allocator, .network = network },.url_blocklist = url_blocklist,- .adblocker = adblocker,+ .adblocker = if (network.adblocker) |*blocker| blocker else null,.arena_pool = &network.app.arena_pool,};}@@ -271,7 +260,6 @@ pub fn deinit(self: *Client) void {}self.clearUrlBlocklist();- if (self.adblocker) |*blocker| blocker.deinit();self.robots.deinit();self.blocking_requests.deinit(self.allocator);self.transfers.deinit(self.allocator);@@ -394,29 +382,9 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {return blocklist.isBlocked(url);}-fn loadAdblockList(allocator: Allocator, blocker: *AdBlocker, path: []const u8) !void {- const text = try std.Io.Dir.cwd().readFileAllocOptions(- lp.io,- path,- allocator,- .limited(64 * 1024 * 1024),- .of(u8),- null,- );- defer allocator.free(text);-- var reader: std.Io.Reader = .fixed(text);- const stats = try blocker.parse(&reader);- log.info(.http, "adblock list loaded", .{- .path = path,- .filters = stats.network,- .invalid = stats.invalid,- });-}-fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) bool {if (internal) return false;- const blocker: *const AdBlocker = if (self.adblocker) |*b| b else return false;+ const blocker = self.adblocker orelse return false;const host = URL.getHostname(url);if (host.len == 0 or host.len > 253) return false;// The trie expects normalized (lowercase) hostnames; URLs aren't@@ -426,11 +394,6 @@ fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) boolreturn blocker.matchHostname(hostname) == .blocked;}-pub fn newHeaders(self: *const Client) !http.Headers {- const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;- return http.Headers.init(ua_header);-}-pub fn getUserAgent(self: *const Client) [:0]const u8 {return self.user_agent_override orelse self.network.config.http_headers.user_agent;}@@ -3479,13 +3442,14 @@ test "HttpClient: adblock verdicts apply per request hostname" {var client: Client = undefined;initTestClient(&client, &pool);- client.adblocker = try AdBlocker.init(testing.allocator);- defer if (client.adblocker) |*blocker| blocker.deinit();+ var blocker: AdBlocker = try .init(testing.allocator);+ defer blocker.deinit();var list: std.Io.Reader = .fixed(\\||ads.example.com^\\@@||good.ads.example.com^);- _ = try client.adblocker.?.parse(&list);+ try blocker.parse(&list);+ client.adblocker = &blocker;try testing.expect(client.isHostAdblocked("https://ads.example.com/pixel.gif", false));// Hostnames are matched case-insensitively and without the port.diff --git a/src/network/Network.zig b/src/network/Network.zigindex a48205d0b..6a7d7d85c 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -36,6 +36,7 @@ const CurlDebugAllocator = @import("CurlDebugAllocator.zig");const Cache = @import("cache/Cache.zig");const SqliteCache = @import("cache/SqliteCache.zig");+const AdBlocker = @import("adblock/AdBlocker.zig");const log = lp.log;const posix = std.posix;@@ -90,6 +91,9 @@ config: *const Config,x509_store: *crypto.X509_STORE,robot_store: RobotStore,web_bot_auth: ?WebBotAuth,+/// Hostname dictionaries built from `--adblock-lists`. Parsed once here and+/// never mutated afterwards, so every HttpClient can share this one copy.+adblocker: ?AdBlocker,connections: []http.Connection,available: DoublyLinkedList = .{},@@ -229,6 +233,10 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {try WebBotAuth.fromConfig(allocator, &wba_cfg)elsenull;+ errdefer if (web_bot_auth) |wba| wba.deinit(allocator);++ var adblocker = try initAdBlocker(allocator, config);+ errdefer if (adblocker) |*blocker| blocker.deinit();const cache = if (config.httpCacheDir()) |cache_dir_path|Cache{@@ -268,6 +276,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {.cache = cache,.robot_store = RobotStore.init(allocator),.web_bot_auth = web_bot_auth,+ .adblocker = adblocker,.ws_pool = .empty,.ws_max = config.wsMaxConcurrent(),@@ -301,6 +310,8 @@ pub fn deinit(self: *Network) void {wba.deinit(self.allocator);}+ if (self.adblocker) |*blocker| blocker.deinit();+if (self.cache) |*cache| cache.deinit();if (self.ip_filter) |f| {@@ -311,6 +322,41 @@ pub fn deinit(self: *Network) void {globalDeinit();}+/// Read buffer for filter lists, and therefore the longest line we can see.+/// Real-world rules are well under 1KB; the parser skips anything longer.+const ADBLOCK_LINE_MAX = 8 * 1024;++fn initAdBlocker(allocator: Allocator, config: *const Config) !?AdBlocker {+ var paths = config.adblockLists() orelse return null;++ var adblocker: ?AdBlocker = null;+ errdefer if (adblocker) |*blocker| blocker.deinit();++ const buf = try allocator.alloc(u8, ADBLOCK_LINE_MAX);+ defer allocator.free(buf);++ while (paths.next()) |path| {+ if (path.len == 0) continue;+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);+ loadAdblockList(&adblocker.?, path, buf) catch |err| {+ log.err(.app, "adblock list load failed", .{ .path = path, .err = err });+ return err;+ };+ }+ return adblocker;+}++/// Streams `path` into `blocker`. The list is consumed line by line, so a+/// 100MB list never needs 100MB of memory.+fn loadAdblockList(blocker: *AdBlocker, path: []const u8, buf: []u8) !void {+ const file = try std.Io.Dir.cwd().openFile(lp.io, path, .{});+ defer file.close(lp.io);++ var file_reader = file.reader(lp.io, buf);+ try blocker.parse(&file_reader.interface);+ log.info(.app, "adblock list loaded", .{ .path = path });+}+pub fn bind(self: *Network,address: *sys_net.IpAddress,