Skip to content

Lightpanda: the headless browser designed for AI and automation

`AdBlocker`: tracking of `rules_loaded` and `rules_skipped`

Halil Durak60cd7e63dba3
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zig
index 2077e34d6..ebe9b052b 100644
--- a/src/network/adblock/AdBlocker.zig
+++ b/src/network/adblock/AdBlocker.zig
@@ -17,13 +17,19 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
+const lp = @import("lightpanda");
+
const Io = std.Io;
const Allocator = std.mem.Allocator;
+const Config = @import("../../Config.zig");
+
const Parser = @import("Parser.zig");
const HostnameTrie = @import("HostnameTrie.zig");
const NetworkFilter = @import("NetworkFilter.zig");
+const log = lp.log;
+
const AdBlocker = @This();
allocator: Allocator,
@@ -33,6 +39,18 @@ blocked: u32,
blocked_important: u32,
/// Pure-hostname `@@` exceptions.
allowed: u32,
+suppressed: u32,
+/// Rules that reached a trie, across every list parsed so far.
+rules_loaded: usize,
+/// Rules we understood but cannot represent, plus lines that were not valid
+/// filters at all. Only a fraction of a real list survives (see `trieRoot`),
+/// so this is what separates "my rule does nothing" from "my rule was never
+/// loaded".
+rules_skipped: usize,
+
+/// Read buffer for filter lists, and therefore the longest line we can see.
+/// Real-world rules are well under 1KB; the parser skips anything longer.
+const LINE_MAX = 8 * 1024;
pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
var trie: HostnameTrie = try .init(allocator);
@@ -40,6 +58,7 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
const blocked = try trie.createTrie(allocator);
const blocked_important = try trie.createTrie(allocator);
const allowed = try trie.createTrie(allocator);
+ const suppressed = try trie.createTrie(allocator);
return .{
.allocator = allocator,
@@ -47,9 +66,51 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
.blocked = blocked,
.blocked_important = blocked_important,
.allowed = allowed,
+ .suppressed = suppressed,
+ .rules_loaded = 0,
+ .rules_skipped = 0,
};
}
+/// Builds the blocker from `--adblock-lists`, or null when the option is
+/// unset. Lists accumulate into the one instance.
+pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {
+ var paths = config.adblockLists() orelse return null;
+
+ var adblocker: ?AdBlocker = null;
+ errdefer if (adblocker) |*blocker| blocker.deinit();
+
+ const buf = try allocator.alloc(u8, LINE_MAX);
+ defer allocator.free(buf);
+
+ while (paths.next()) |path| {
+ if (path.len == 0) continue;
+ if (adblocker == null) adblocker = try AdBlocker.init(allocator);
+ loadList(&adblocker.?, path, buf) catch |err| {
+ log.err(.app, "adblock list load failed", .{ .path = path, .err = err });
+ return err;
+ };
+ }
+
+ if (adblocker) |*blocker| {
+ log.info(.app, "adblock lists loaded", .{
+ .loaded = blocker.rules_loaded,
+ .skipped = blocker.rules_skipped,
+ });
+ }
+ return adblocker;
+}
+
+/// Streams `path` into `blocker`. The list is consumed line by line, so a
+/// 100MB list never needs 100MB of memory.
+fn loadList(blocker: *AdBlocker, path: []const u8, buf: []u8) !void {
+ const file = try std.Io.Dir.cwd().openFile(lp.io, path, .{});
+ defer file.close(lp.io);
+
+ var file_reader = file.reader(lp.io, buf);
+ try blocker.parse(&file_reader.interface);
+}
+
pub fn deinit(self: *AdBlocker) void {
self.trie.deinit(self.allocator);
}
@@ -66,51 +127,86 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
defer _ = scratch_instance.reset(.retain_capacity);
// Filters the tries cannot express (patterns, type/party/domain
- // constraints, badfilter directives) are dropped: deciding those
- // needs a request engine we do not have yet, and retaining them
- // costs tens of MB on a list like EasyList.
- const root = self.trieRoot(&filter) orelse continue;
+ // constraints) are dropped: deciding those needs a request engine
+ // we do not have yet, and retaining them costs tens of MB on a
+ // list like EasyList.
+ const root = self.trieRoot(&filter) orelse {
+ self.rules_skipped += 1;
+ continue;
+ };
// Duplicate and subdomain-of-existing entries drop.
self.trie.add(self.allocator, root, filter.hostname) catch |err| switch (err) {
error.OutOfMemory, error.TrieFull => |e| return e,
- // The parser never yields a .hostname filter without one.
+ // trieRoot never routes a filter without one.
error.InvalidHostname => unreachable,
};
+ self.rules_loaded += 1;
}
+ self.rules_skipped += parser.skipped;
}
pub const Verdict = enum { none, allowed, blocked };
-/// `.none` means no hostname-wide filter applies. Filters that need more
+/// `.none` means no hostname-wide verdict applies. Filters that need more
/// than a hostname match are not represented here at all.
pub fn matchHostname(self: *const AdBlocker, hostname: []const u8) Verdict {
+ // A hostname something might unblock stays undecided, whatever else
+ // matches it.
+ if (self.trie.matches(self.suppressed, hostname) != null) return .none;
if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;
if (self.trie.matches(self.allowed, hostname) != null) return .allowed;
if (self.trie.matches(self.blocked, hostname) != null) return .blocked;
return .none;
}
-/// The trie holding this filter, or null when the filter's behavior is
-/// more than a hostname-wide match.
+/// The trie holding this filter, or null when it carries no hostname-wide
+/// meaning we can represent.
fn trieRoot(self: *const AdBlocker, filter: *const NetworkFilter) ?u32 {
- if (filter.kind != .hostname) return null;
- // `||host` without '^' also matches hostnames merely *starting* with
- // host ("example.com.evil.org"): broader than a suffix match.
- if (!filter.require_separator) return null;
- // `||host^|` pins the URL end to the separator: narrower.
- if (filter.left_anchor or filter.right_anchor) return null;
- if (filter.badfilter) return null;
- if (!filter.first_party or !filter.third_party) return null;
- if (!filter.domains.isEmpty()) return null;
- if (filter.types.bits() != NetworkFilter.ResourceTypes.all.bits()) return null;
- if (filter.exception) {
+ // An exception or a $badfilter can only ever *unblock*. Where one is
+ // narrower than a whole hostname we cannot evaluate it here — but
+ // ignoring it is not neutral either, because the rule it cancels may
+ // well be a plain block we did keep:
+ //
+ // ||adsafeprotected.com^
+ // @@||adsafeprotected.com/iasPET.$script,domain=reuters.com|...
+ //
+ // uBO blocks that host everywhere but those sites; keeping only the
+ // first line blocks it on those sites too, and breaks them. So the
+ // hostname goes to `suppressed` and stops being blockable at all. We
+ // lose most of the block's value to save the pages it would break,
+ // which is the direction we want to fail in.
+ if (filter.exception or filter.badfilter) {
// Cosmetic-realm exceptions never unblock network requests.
- if (filter.generichide or filter.specifichide or filter.elemhide) return null;
- return self.allowed;
+ if (filter.exception and (filter.generichide or filter.specifichide or filter.elemhide)) {
+ return null;
+ }
+ if (filter.exception and !filter.badfilter and isWholeHostname(filter)) {
+ return self.allowed;
+ }
+ // `@@/ads/^$script` and friends name no hostname to key on: there
+ // is nothing we can do with them until there is an engine.
+ if (filter.hostname.len == 0) return null;
+ return self.suppressed;
}
+
+ if (!isWholeHostname(filter)) return null;
return if (filter.important) self.blocked_important else self.blocked;
}
+/// Whether the filter's whole effect is "every request to this hostname and
+/// its subdomains", which is all a trie can express.
+fn isWholeHostname(filter: *const NetworkFilter) bool {
+ if (filter.kind != .hostname) return false;
+ // `||host` without '^' also matches hostnames merely *starting* with
+ // host ("example.com.evil.org"): broader than a suffix match.
+ if (!filter.require_separator) return false;
+ // `||host^|` pins the URL end to the separator: narrower.
+ if (filter.left_anchor or filter.right_anchor) return false;
+ if (filter.has_domains) return false;
+ if (!filter.first_party or !filter.third_party) return false;
+ return filter.types.bits() == NetworkFilter.ResourceTypes.all.bits();
+}
+
const testing = @import("../../testing.zig");
test "adblock.AdBlocker: parse accumulates across lists" {