Lightpanda: the headless browser designed for AI and automation
`Adblocker`: parser semantics: follow uBO's behavior
Halil Durak085b4836acd0
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zigindex 7c7ede9fe..94a8d9dd6 100644--- a/src/network/adblock/AdBlocker.zig+++ b/src/network/adblock/AdBlocker.zig@@ -59,10 +59,6 @@ suppressed: u32,blocking: Engine,blocking_important: Engine,exceptions: Engine,-/// `@@…$important` exceptions, indexed apart because they outrank the-/// `$important` blocks while plain exceptions do not — one engine returning-/// its first match could not tell the two apart.-exceptions_important: Engine,/// Rules that reached a trie or an index, across every list parsed so far.rules_loaded: usize,@@ -96,7 +92,6 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {.blocking = .empty,.blocking_important = .empty,.exceptions = .empty,- .exceptions_important = .empty,.rules_loaded = 0,.rules_skipped = 0,};@@ -106,7 +101,6 @@ pub fn deinit(self: *AdBlocker) void {self.blocking.deinit(self.allocator);self.blocking_important.deinit(self.allocator);self.exceptions.deinit(self.allocator);- self.exceptions_important.deinit(self.allocator);self.badfilters.deinit(self.allocator);self.filters.deinit(self.allocator);self.trie.deinit(self.allocator);@@ -251,8 +245,6 @@ pub fn build(self: *AdBlocker) !void {defer important.deinit(self.allocator);var exceptions: std.ArrayList(u32) = .empty;defer exceptions.deinit(self.allocator);- var exceptions_important: std.ArrayList(u32) = .empty;- defer exceptions_important.deinit(self.allocator);var kept: usize = 0;for (self.filters.items) |filter| {@@ -278,7 +270,7 @@ pub fn build(self: *AdBlocker) !void {kept += 1;const list = if (filter.exception)- (if (filter.important) &exceptions_important else &exceptions)+ &exceptionselse if (filter.important)&importantelse@@ -299,7 +291,6 @@ pub fn build(self: *AdBlocker) !void {self.blocking = try .build(self.allocator, arena, filters, blocking.items, &histogram);self.blocking_important = try .build(self.allocator, arena, filters, important.items, &histogram);self.exceptions = try .build(self.allocator, arena, filters, exceptions.items, &histogram);- self.exceptions_important = try .build(self.allocator, arena, filters, exceptions_important.items, &histogram);}fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {@@ -313,8 +304,8 @@ fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {pub const Verdict = enum { none, allowed, blocked };-/// The verdict for one request. uBO's order: `@@…$important` outranks-/// `$important` blocks, which outrank exceptions, which outrank plain blocks...+/// The verdict for one request.+/// `$important` blocks outrank exceptions, which outrank plain blocks; same as uBO.pub fn match(self: *const AdBlocker, request: Request) Verdict {std.debug.assert(self.built);@@ -323,10 +314,6 @@ pub fn match(self: *const AdBlocker, request: Request) Verdict {// matches it.if (self.trie.matches(self.suppressed, hostname) != null) return .none;- // `@@…$important` is what a list writes to beat a `$important` block, so- // it has to be looked at before one.- if (self.exceptions_important.match(request) != null) return .allowed;-if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;if (self.blocking_important.match(request) != null) return .blocked;@@ -352,14 +339,11 @@ fn isSupported(filter: *const NetworkFilter) bool {/// its subdomains", which is all a trie can express.fn isWholeHostname(filter: *const NetworkFilter) bool {if (filter.kind != .hostname) return false;- // `||host` without '^' also matches hostnames merely *starting* with- // host ("example.com.evil.org"): broader than a suffix match.+ // A prefix like `||adform.` also matches hostnames merely *starting*+ // with it, broader than a suffix match.if (!filter.require_separator) return false;// `||host^|` pins the URL end to the separator: narrower.if (filter.left_anchor or filter.right_anchor) return false;- // `@@…$important` has to be consulted before the `$important` blocks,- // which is an order the fixed trie sequence cannot express.- if (filter.exception and filter.important) return false;if (!filter.domains.isEmpty()) return false;if (!filter.first_party or !filter.third_party) return false;return filter.types.bits() == ResourceTypes.all.bits();@@ -570,11 +554,6 @@ test "adblock.AdBlocker: verdict precedence" {\\@@||evil.com^\\||strict.example.com^$important,script\\@@||strict.example.com^$script- \\||both.example.com^$important,script- \\@@||both.example.com^$important,script- // Both sides pure hostnames, so both would otherwise be tries.- \\||pure.example.com^$important- \\@@||pure.example.com^$important);try expectVerdict(&blocker, .blocked, "https://ads.example.com/x", "a.com", script);@@ -585,9 +564,6 @@ test "adblock.AdBlocker: verdict precedence" {// ...but $important beats the exception, whichever side is indexed.try expectVerdict(&blocker, .blocked, "https://evil.com/x", "a.com", script);try expectVerdict(&blocker, .blocked, "https://strict.example.com/x.js", "a.com", script);- // ...and $important on the exception takes it back.- try expectVerdict(&blocker, .allowed, "https://both.example.com/x.js", "a.com", script);- try expectVerdict(&blocker, .allowed, "https://pure.example.com/x.js", "a.com", script);try expectVerdict(&blocker, .none, "https://example.com/x", "a.com", script);}@@ -650,32 +626,41 @@ test "adblock.AdBlocker: trie absorbs every pure-hostname form" {try testLoad(&blocker,\\||anchored.example.com^+ \\||caretless.example.com+ \\||pinned.example.com|\\bare-hostname.example.com\\0.0.0.0 hosts-style.example.io);try expectVerdict(&blocker, .blocked, "https://anchored.example.com/x", "a.com", script);+ // `||host` and `||host|` are the same filter as `||host^`.+ try expectVerdict(&blocker, .blocked, "https://sub.caretless.example.com/x", "a.com", script);+ try expectVerdict(&blocker, .none, "https://caretless.example.com.evil.org/x", "a.com", script);+ try expectVerdict(&blocker, .blocked, "https://pinned.example.com/x?q=1", "a.com", script);try expectVerdict(&blocker, .blocked, "https://bare-hostname.example.com/x", "a.com", script);try expectVerdict(&blocker, .blocked, "https://hosts-style.example.io/x", "a.com", script);// Pure-hostname filters block top-level documents too.try expectVerdict(&blocker, .blocked, "https://anchored.example.com/", "a.com", document);++ // Absorbs: every rule above went to a trie, none stayed indexed.+ try testing.expectEqual(0, blocker.filters.items.len);}-test "adblock.AdBlocker: an important exception is found whatever matches first" {+test "adblock.AdBlocker: $important on an exception is invalid" {var blocker: AdBlocker = try .init(testing.allocator);defer blocker.deinit();- // The plain exception is written first, so a single first-match engine- // would return it and never see the important one behind it.+ // uBO rejects `@@…$important`, so nothing outranks a `$important`+ // block — and being a rule uBO drops too, the invalid exception does+ // not suppress its hostname either.try testLoad(&blocker,\\||tracker.com^$important,script- \\@@||tracker.com^$script\\@@||tracker.com/x.js$important,script);- try expectVerdict(&blocker, .allowed, "https://tracker.com/x.js", "a.com", script);- // Off the important exception's path, the $important block still wins.- try expectVerdict(&blocker, .blocked, "https://tracker.com/other.js", "a.com", script);+ try expectVerdict(&blocker, .blocked, "https://tracker.com/x.js", "a.com", script);+ try testing.expectEqual(1, blocker.rules_loaded);+ try testing.expectEqual(1, blocker.rules_skipped);}test "adblock.AdBlocker: an empty blocker decides nothing" {diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zigindex fc63f858b..4cf9493f2 100644--- a/src/network/adblock/NetworkFilter.zig+++ b/src/network/adblock/NetworkFilter.zig@@ -416,7 +416,8 @@ fn parseOptions(self.domains = try domain.parse(arena, v);},.important => {- if (negated) return error.InvalidOption;+ // Importance is a property of blocks; uBO rejects `@@…$important`.+ if (negated or self.exception) return error.InvalidOption;self.important = true;},.badfilter => {@@ -588,12 +589,14 @@ fn parsePattern(pattern = pattern[0 .. pattern.len - 1];}- // Trim pointless wildcards touching the (now removed) ends.- while (std.mem.startsWith(u8, pattern, "*")) {- pattern = pattern[1..];+ // Trim pointless wildcards touching the (now removed) ends. A wildcard bordering+ // a short word stays; which is what keeps the word a substring rather than a hostname.+ const stars = std.mem.indexOfNone(u8, pattern, "*") orelse pattern.len;+ if (stars > 0 and (stars == pattern.len or !isPatternWordChar(pattern[stars]))) {+ pattern = pattern[stars..];self.left_anchor = false;}- while (std.mem.endsWith(u8, pattern, "*")) {+ while (pointlessTrailingWildcard(pattern)) {pattern = pattern[0 .. pattern.len - 1];self.right_anchor = false;}@@ -623,6 +626,10 @@ fn parsePattern(const remainder = pattern[host_end..];if (remainder.len == 0) {self.kind = .hostname;+ if (isHostnameShaped(self.hostname)) {+ self.require_separator = true;+ self.right_anchor = false;+ }} else if (std.mem.eql(u8, remainder, "^")) {self.kind = .hostname;self.require_separator = true;@@ -633,14 +640,14 @@ fn parsePattern(return;}- // A pattern that reads as a bare hostname is a hostname filter- // (`ads.example.com` == `||ads.example.com^`)- if (!self.left_anchor and !self.right_anchor and isHostnameShaped(pattern)) {+ if (isHostnameShaped(pattern)) {if (isRedirectHostName(pattern)) return error.Ignored;self.kind = .hostname;self.hostname = pattern;self.hostname_anchor = true;self.require_separator = true;+ self.left_anchor = false;+ self.right_anchor = false;return;}@@ -652,6 +659,19 @@ fn parsePattern(self.pattern = pattern;}+/// Ditto uBO's `rePointlessTrailingWildcards`.+fn pointlessTrailingWildcard(pattern: []const u8) bool {+ if (pattern.len == 0 or pattern[pattern.len - 1] != '*') return false;+ const before = std.mem.trimEnd(u8, pattern, "*");+ var run: usize = 0;+ while (run < before.len and isPatternWordChar(before[before.len - 1 - run])) run += 1;+ return run == 0 or run >= 7;+}++inline fn isPatternWordChar(c: u8) bool {+ return std.ascii.isAlphanumeric(c) or c == '%';+}+/// Matches uBO's hostname flavor: dot-separated labels of [a-z0-9_-], each/// starting alphanumeric; the last label is [a-z0-9][a-z0-9-]*[a-z0-9]/// (len >= 2, no '_'). Accepts IPv4 literals. Input must be lowercase.@@ -705,12 +725,17 @@ test "adblock.NetworkFilter: pure hostname forms" {// Implicit "strict" blocking: documents included.try testing.expectEqual(ResourceTypes.all.bits(), f.types.bits());- // Without '^' there is no separator requirement and no implicit- // document blocking.+ // `||host` and `||host|` are, in uBO, the same filter as `||host^`:+ // the request hostname is `host` or a subdomain of it.f = try testParse(arena, "||ads.example.com");try testing.expectEqual(.hostname, f.kind);- try testing.expect(!f.require_separator);- try testing.expectEqual(ResourceTypes.all_network.bits(), f.types.bits());+ try testing.expect(f.require_separator);+ try testing.expectEqual(ResourceTypes.all.bits(), f.types.bits());++ f = try testParse(arena, "||ads.example.com|");+ try testing.expectEqual(.hostname, f.kind);+ try testing.expect(f.require_separator);+ try testing.expect(!f.right_anchor);// Bare hostname line == ||host^ (uBO divergence from ABP).f = try testParse(arena, "tracker.example.net");@@ -718,6 +743,18 @@ test "adblock.NetworkFilter: pure hostname forms" {try testing.expectString("tracker.example.net", f.hostname);try testing.expect(f.require_separator);+ // An anchor on a bare hostname is part of the same reading.+ f = try testParse(arena, "tracker.example.net|");+ try testing.expectEqual(.hostname, f.kind);+ try testing.expectString("tracker.example.net", f.hostname);+ try testing.expect(f.require_separator);+ try testing.expect(!f.right_anchor);++ f = try testParse(arena, "|tracker.example.net");+ try testing.expectEqual(.hostname, f.kind);+ try testing.expectString("tracker.example.net", f.hostname);+ try testing.expect(!f.left_anchor);+// Raw IPv4 lines (URLhaus style).f = try testParse(arena, "101.126.11.168");try testing.expectEqual(.hostname, f.kind);@@ -778,11 +815,23 @@ test "adblock.NetworkFilter: anchors and pattern kinds" {try testing.expectEqual(.plain, f.kind);try testing.expect(!f.right_anchor);- // A pattern that trims down to a bare hostname shape gets promoted- // (uBO flavor rules), even a single label.+ // A '*' bordering a short alphanumeric run is meaningful (uBO's+ // pointless-wildcard rules), so no trimming and no hostname promotion.f = try testParse(arena, "*ads*|");+ try testing.expectEqual(.wildcard, f.kind);+ try testing.expectString("*ads*", f.pattern);++ // A trailing '*' after 7+ chars drops, and a pattern that trims down+ // to a bare hostname shape gets promoted (uBO flavor rules)...+ f = try testParse(arena, "ads.example*");try testing.expectEqual(.hostname, f.kind);- try testing.expectString("ads", f.hostname);+ try testing.expectString("ads.example", f.hostname);++ // ...but a leading '*' before a word survives, and keeps the pattern a+ // substring match rather than a hostname.+ f = try testParse(arena, "*ads.example*");+ try testing.expectEqual(.wildcard, f.kind);+ try testing.expectString("*ads.example", f.pattern);// '||' hostname region containing '*' stays a generic pattern.f = try testParse(arena, "||example.*/ads");@@ -964,6 +1013,9 @@ test "adblock.NetworkFilter: important, badfilter, noop" {var f = try testParse(arena, "||ads.com^$important");try testing.expect(f.important);+ // Importance is a property of blocks; uBO rejects it on an exception.+ try testing.expectError(error.InvalidOption, testParse(arena, "@@||ads.com^$important"));+f = try testParse(arena, "||ads.com^$badfilter");try testing.expect(f.badfilter);diff --git a/src/network/adblock/pattern.zig b/src/network/adblock/pattern.zigindex 012875a98..6bd2d085d 100644--- a/src/network/adblock/pattern.zig+++ b/src/network/adblock/pattern.zig@@ -271,21 +271,31 @@ test "adblock.pattern: hostname filters cover subdomains" {try testing.expect(!try testMatch(arena, "||ads.example.com^", "https://example.com/ads.example.com"));}-test "adblock.pattern: caret-less hostname filters end on a label boundary" {+test "adblock.pattern: caret-less hostname filters match the hostname or a subdomain" {var arena_state = std.heap.ArenaAllocator.init(testing.allocator);defer arena_state.deinit();const arena = arena_state.allocator();- // The real EasyPrivacy shape "||coinimp.com$third-party": without a '^'- // the next label may continue ("coinimp.com.evil.org") but the last one- // may not ("coinimp.community" is someone else's site).+ // The real EasyPrivacy shape "||coinimp.com$third-party": uBO reads it+ // as "||coinimp.com^", so "coinimp.com.evil.org" — evil.org's hostname,+ // not a subdomain of coinimp.com — is out.try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com/miner.js"));try testing.expect(try testMatch(arena, "||coinimp.com", "https://sub.coinimp.com/x"));- try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com.evil.org/x"));try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com:8080/x"));+ try testing.expect(!try testMatch(arena, "||coinimp.com", "https://coinimp.com.evil.org/x"));try testing.expect(!try testMatch(arena, "||coinimp.com", "https://coinimp.community/miner.js"));- // A filter hostname ending in '.' carries its own boundary.+ // `||host|` is that same filter once more.+ try testing.expect(try testMatch(arena, "||coinimp.com|", "https://sub.coinimp.com/x"));+ try testing.expect(!try testMatch(arena, "||coinimp.com|", "https://coinimp.com.evil.org/x"));++ // `||host*` is NOT: uBO keeps a trailing '*' after a short run, and the+ // pattern stays a label-anchored prefix.+ try testing.expect(try testMatch(arena, "||coinimp.com*", "https://coinimp.com.evil.org/x"));+ try testing.expect(try testMatch(arena, "||coinimp.com*", "https://coinimp.community/miner.js"));++ // A filter hostname ending in '.' names no hostname; it stays a prefix+ // and carries its own boundary.try testing.expect(try testMatch(arena, "||adform.", "https://adform.net/banner"));try testing.expect(try testMatch(arena, "||adform.", "https://track.adform.co.uk/x"));try testing.expect(!try testMatch(arena, "||adform", "https://adformat.example/x"));