Skip to content

Lightpanda: the headless browser designed for AI and automation

load and parse block list once and share between HTTP clients

Halil Durakc111fc5675a7
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 29346cf18..34e7c0d7b 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -35,8 +35,9 @@ const Network = @import("Network.zig");
const Cache = @import("cache/Cache.zig");
const RobotsGate = @import("RobotsGate.zig");
const UrlBlocklist = @import("UrlBlocklist.zig");
+const AdBlocker = @import("adblock/AdBlocker.zig");
+
pub const BlockPattern = UrlBlocklist.Pattern;
-const AdBlocker = @import("../adblock/AdBlocker.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -186,8 +187,10 @@ obey_robots: bool,
robots: RobotsGate,
url_blocklist: ?UrlBlocklist,
-/// Hostname dictionaries built from `--adblock-lists` filter files.
-adblocker: ?AdBlocker,
+/// Hostname dictionaries built from `--adblock-lists`. Owned by Network,
+/// parsed once at startup and read-only from here on, so every client
+/// shares the one copy.
+adblocker: ?*const AdBlocker,
pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {
var handles = try http.Handles.init(network.config);
@@ -210,20 +213,6 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
}
errdefer if (url_blocklist) |*blocklist| blocklist.deinit();
- var adblocker: ?AdBlocker = null;
- errdefer if (adblocker) |*blocker| blocker.deinit();
- if (network.config.adblockLists()) |initial_paths| {
- var it = initial_paths;
- while (it.next()) |path| {
- if (path.len == 0) continue;
- if (adblocker == null) adblocker = try AdBlocker.init(allocator);
- loadAdblockList(allocator, &adblocker.?, path) catch |err| {
- log.err(.http, "adblock list load failed", .{ .path = path, .err = err });
- return err;
- };
- }
- }
-
self.* = Client{
.handles = handles,
.network = network,
@@ -241,7 +230,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.obey_robots = network.config.obeyRobots(),
.robots = .{ .allocator = allocator, .network = network },
.url_blocklist = url_blocklist,
- .adblocker = adblocker,
+ .adblocker = if (network.adblocker) |*blocker| blocker else null,
.arena_pool = &network.app.arena_pool,
};
}
@@ -271,7 +260,6 @@ pub fn deinit(self: *Client) void {
}
self.clearUrlBlocklist();
- if (self.adblocker) |*blocker| blocker.deinit();
self.robots.deinit();
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
@@ -394,29 +382,9 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {
return blocklist.isBlocked(url);
}
-fn loadAdblockList(allocator: Allocator, blocker: *AdBlocker, path: []const u8) !void {
- const text = try std.Io.Dir.cwd().readFileAllocOptions(
- lp.io,
- path,
- allocator,
- .limited(64 * 1024 * 1024),
- .of(u8),
- null,
- );
- defer allocator.free(text);
-
- var reader: std.Io.Reader = .fixed(text);
- const stats = try blocker.parse(&reader);
- log.info(.http, "adblock list loaded", .{
- .path = path,
- .filters = stats.network,
- .invalid = stats.invalid,
- });
-}
-
fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) bool {
if (internal) return false;
- const blocker: *const AdBlocker = if (self.adblocker) |*b| b else return false;
+ const blocker = self.adblocker orelse return false;
const host = URL.getHostname(url);
if (host.len == 0 or host.len > 253) return false;
// The trie expects normalized (lowercase) hostnames; URLs aren't
@@ -426,11 +394,6 @@ fn isHostAdblocked(self: *const Client, url: [:0]const u8, internal: bool) bool
return blocker.matchHostname(hostname) == .blocked;
}
-pub fn newHeaders(self: *const Client) !http.Headers {
- const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;
- return http.Headers.init(ua_header);
-}
-
pub fn getUserAgent(self: *const Client) [:0]const u8 {
return self.user_agent_override orelse self.network.config.http_headers.user_agent;
}
@@ -3479,13 +3442,14 @@ test "HttpClient: adblock verdicts apply per request hostname" {
var client: Client = undefined;
initTestClient(&client, &pool);
- client.adblocker = try AdBlocker.init(testing.allocator);
- defer if (client.adblocker) |*blocker| blocker.deinit();
+ var blocker: AdBlocker = try .init(testing.allocator);
+ defer blocker.deinit();
var list: std.Io.Reader = .fixed(
\\||ads.example.com^
\\@@||good.ads.example.com^
);
- _ = try client.adblocker.?.parse(&list);
+ try blocker.parse(&list);
+ client.adblocker = &blocker;
try testing.expect(client.isHostAdblocked("https://ads.example.com/pixel.gif", false));
// Hostnames are matched case-insensitively and without the port.
diff --git a/src/network/Network.zig b/src/network/Network.zig
index a48205d0b..6a7d7d85c 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -36,6 +36,7 @@ const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
const Cache = @import("cache/Cache.zig");
const SqliteCache = @import("cache/SqliteCache.zig");
+const AdBlocker = @import("adblock/AdBlocker.zig");
const log = lp.log;
const posix = std.posix;
@@ -90,6 +91,9 @@ config: *const Config,
x509_store: *crypto.X509_STORE,
robot_store: RobotStore,
web_bot_auth: ?WebBotAuth,
+/// Hostname dictionaries built from `--adblock-lists`. Parsed once here and
+/// never mutated afterwards, so every HttpClient can share this one copy.
+adblocker: ?AdBlocker,
connections: []http.Connection,
available: DoublyLinkedList = .{},
@@ -229,6 +233,10 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
try WebBotAuth.fromConfig(allocator, &wba_cfg)
else
null;
+ errdefer if (web_bot_auth) |wba| wba.deinit(allocator);
+
+ var adblocker = try initAdBlocker(allocator, config);
+ errdefer if (adblocker) |*blocker| blocker.deinit();
const cache = if (config.httpCacheDir()) |cache_dir_path|
Cache{
@@ -268,6 +276,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
.cache = cache,
.robot_store = RobotStore.init(allocator),
.web_bot_auth = web_bot_auth,
+ .adblocker = adblocker,
.ws_pool = .empty,
.ws_max = config.wsMaxConcurrent(),
@@ -301,6 +310,8 @@ pub fn deinit(self: *Network) void {
wba.deinit(self.allocator);
}
+ if (self.adblocker) |*blocker| blocker.deinit();
+
if (self.cache) |*cache| cache.deinit();
if (self.ip_filter) |f| {
@@ -311,6 +322,41 @@ pub fn deinit(self: *Network) void {
globalDeinit();
}
+/// Read buffer for filter lists, and therefore the longest line we can see.
+/// Real-world rules are well under 1KB; the parser skips anything longer.
+const ADBLOCK_LINE_MAX = 8 * 1024;
+
+fn initAdBlocker(allocator: Allocator, config: *const Config) !?AdBlocker {
+ var paths = config.adblockLists() orelse return null;
+
+ var adblocker: ?AdBlocker = null;
+ errdefer if (adblocker) |*blocker| blocker.deinit();
+
+ const buf = try allocator.alloc(u8, ADBLOCK_LINE_MAX);
+ defer allocator.free(buf);
+
+ while (paths.next()) |path| {
+ if (path.len == 0) continue;
+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);
+ loadAdblockList(&adblocker.?, path, buf) catch |err| {
+ log.err(.app, "adblock list load failed", .{ .path = path, .err = err });
+ return err;
+ };
+ }
+ return adblocker;
+}
+
+/// Streams `path` into `blocker`. The list is consumed line by line, so a
+/// 100MB list never needs 100MB of memory.
+fn loadAdblockList(blocker: *AdBlocker, path: []const u8, buf: []u8) !void {
+ const file = try std.Io.Dir.cwd().openFile(lp.io, path, .{});
+ defer file.close(lp.io);
+
+ var file_reader = file.reader(lp.io, buf);
+ try blocker.parse(&file_reader.interface);
+ log.info(.app, "adblock list loaded", .{ .path = path });
+}
+
pub fn bind(
self: *Network,
address: *sys_net.IpAddress,