Lightpanda: the headless browser designed for AI and automation
`AdBlocker`: tracking of `rules_loaded` and `rules_skipped`
Halil Durak60cd7e63dba3
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zigindex 2077e34d6..ebe9b052b 100644--- a/src/network/adblock/AdBlocker.zig+++ b/src/network/adblock/AdBlocker.zig@@ -17,13 +17,19 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.const std = @import("std");+const lp = @import("lightpanda");+const Io = std.Io;const Allocator = std.mem.Allocator;+const Config = @import("../../Config.zig");+const Parser = @import("Parser.zig");const HostnameTrie = @import("HostnameTrie.zig");const NetworkFilter = @import("NetworkFilter.zig");+const log = lp.log;+const AdBlocker = @This();allocator: Allocator,@@ -33,6 +39,18 @@ blocked: u32,blocked_important: u32,/// Pure-hostname `@@` exceptions.allowed: u32,+suppressed: u32,+/// Rules that reached a trie, across every list parsed so far.+rules_loaded: usize,+/// Rules we understood but cannot represent, plus lines that were not valid+/// filters at all. Only a fraction of a real list survives (see `trieRoot`),+/// so this is what separates "my rule does nothing" from "my rule was never+/// loaded".+rules_skipped: usize,++/// Read buffer for filter lists, and therefore the longest line we can see.+/// Real-world rules are well under 1KB; the parser skips anything longer.+const LINE_MAX = 8 * 1024;pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {var trie: HostnameTrie = try .init(allocator);@@ -40,6 +58,7 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {const blocked = try trie.createTrie(allocator);const blocked_important = try trie.createTrie(allocator);const allowed = try trie.createTrie(allocator);+ const suppressed = try trie.createTrie(allocator);return .{.allocator = allocator,@@ -47,9 +66,51 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {.blocked = blocked,.blocked_important = blocked_important,.allowed = allowed,+ .suppressed = suppressed,+ .rules_loaded = 0,+ .rules_skipped = 0,};}+/// Builds the blocker from `--adblock-lists`, or null when the option is+/// unset. Lists accumulate into the one instance.+pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {+ var paths = config.adblockLists() orelse return null;++ var adblocker: ?AdBlocker = null;+ errdefer if (adblocker) |*blocker| blocker.deinit();++ const buf = try allocator.alloc(u8, LINE_MAX);+ defer allocator.free(buf);++ while (paths.next()) |path| {+ if (path.len == 0) continue;+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);+ loadList(&adblocker.?, path, buf) catch |err| {+ log.err(.app, "adblock list load failed", .{ .path = path, .err = err });+ return err;+ };+ }++ if (adblocker) |*blocker| {+ log.info(.app, "adblock lists loaded", .{+ .loaded = blocker.rules_loaded,+ .skipped = blocker.rules_skipped,+ });+ }+ return adblocker;+}++/// Streams `path` into `blocker`. The list is consumed line by line, so a+/// 100MB list never needs 100MB of memory.+fn loadList(blocker: *AdBlocker, path: []const u8, buf: []u8) !void {+ const file = try std.Io.Dir.cwd().openFile(lp.io, path, .{});+ defer file.close(lp.io);++ var file_reader = file.reader(lp.io, buf);+ try blocker.parse(&file_reader.interface);+}+pub fn deinit(self: *AdBlocker) void {self.trie.deinit(self.allocator);}@@ -66,51 +127,86 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {defer _ = scratch_instance.reset(.retain_capacity);// Filters the tries cannot express (patterns, type/party/domain- // constraints, badfilter directives) are dropped: deciding those- // needs a request engine we do not have yet, and retaining them- // costs tens of MB on a list like EasyList.- const root = self.trieRoot(&filter) orelse continue;+ // constraints) are dropped: deciding those needs a request engine+ // we do not have yet, and retaining them costs tens of MB on a+ // list like EasyList.+ const root = self.trieRoot(&filter) orelse {+ self.rules_skipped += 1;+ continue;+ };// Duplicate and subdomain-of-existing entries drop.self.trie.add(self.allocator, root, filter.hostname) catch |err| switch (err) {error.OutOfMemory, error.TrieFull => |e| return e,- // The parser never yields a .hostname filter without one.+ // trieRoot never routes a filter without one.error.InvalidHostname => unreachable,};+ self.rules_loaded += 1;}+ self.rules_skipped += parser.skipped;}pub const Verdict = enum { none, allowed, blocked };-/// `.none` means no hostname-wide filter applies. Filters that need more+/// `.none` means no hostname-wide verdict applies. Filters that need more/// than a hostname match are not represented here at all.pub fn matchHostname(self: *const AdBlocker, hostname: []const u8) Verdict {+ // A hostname something might unblock stays undecided, whatever else+ // matches it.+ if (self.trie.matches(self.suppressed, hostname) != null) return .none;if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;if (self.trie.matches(self.allowed, hostname) != null) return .allowed;if (self.trie.matches(self.blocked, hostname) != null) return .blocked;return .none;}-/// The trie holding this filter, or null when the filter's behavior is-/// more than a hostname-wide match.+/// The trie holding this filter, or null when it carries no hostname-wide+/// meaning we can represent.fn trieRoot(self: *const AdBlocker, filter: *const NetworkFilter) ?u32 {- if (filter.kind != .hostname) return null;- // `||host` without '^' also matches hostnames merely *starting* with- // host ("example.com.evil.org"): broader than a suffix match.- if (!filter.require_separator) return null;- // `||host^|` pins the URL end to the separator: narrower.- if (filter.left_anchor or filter.right_anchor) return null;- if (filter.badfilter) return null;- if (!filter.first_party or !filter.third_party) return null;- if (!filter.domains.isEmpty()) return null;- if (filter.types.bits() != NetworkFilter.ResourceTypes.all.bits()) return null;- if (filter.exception) {+ // An exception or a $badfilter can only ever *unblock*. Where one is+ // narrower than a whole hostname we cannot evaluate it here — but+ // ignoring it is not neutral either, because the rule it cancels may+ // well be a plain block we did keep:+ //+ // ||adsafeprotected.com^+ // @@||adsafeprotected.com/iasPET.$script,domain=reuters.com|...+ //+ // uBO blocks that host everywhere but those sites; keeping only the+ // first line blocks it on those sites too, and breaks them. So the+ // hostname goes to `suppressed` and stops being blockable at all. We+ // lose most of the block's value to save the pages it would break,+ // which is the direction we want to fail in.+ if (filter.exception or filter.badfilter) {// Cosmetic-realm exceptions never unblock network requests.- if (filter.generichide or filter.specifichide or filter.elemhide) return null;- return self.allowed;+ if (filter.exception and (filter.generichide or filter.specifichide or filter.elemhide)) {+ return null;+ }+ if (filter.exception and !filter.badfilter and isWholeHostname(filter)) {+ return self.allowed;+ }+ // `@@/ads/^$script` and friends name no hostname to key on: there+ // is nothing we can do with them until there is an engine.+ if (filter.hostname.len == 0) return null;+ return self.suppressed;}++ if (!isWholeHostname(filter)) return null;return if (filter.important) self.blocked_important else self.blocked;}+/// Whether the filter's whole effect is "every request to this hostname and+/// its subdomains", which is all a trie can express.+fn isWholeHostname(filter: *const NetworkFilter) bool {+ if (filter.kind != .hostname) return false;+ // `||host` without '^' also matches hostnames merely *starting* with+ // host ("example.com.evil.org"): broader than a suffix match.+ if (!filter.require_separator) return false;+ // `||host^|` pins the URL end to the separator: narrower.+ if (filter.left_anchor or filter.right_anchor) return false;+ if (filter.has_domains) return false;+ if (!filter.first_party or !filter.third_party) return false;+ return filter.types.bits() == NetworkFilter.ResourceTypes.all.bits();+}+const testing = @import("../../testing.zig");test "adblock.AdBlocker: parse accumulates across lists" {