Skip to content

Lightpanda: the headless browser designed for AI and automation

`Adblocker`: parser semantics: follow uBO's behavior

Halil Durak085b4836acd0
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zig
index 7c7ede9fe..94a8d9dd6 100644
--- a/src/network/adblock/AdBlocker.zig
+++ b/src/network/adblock/AdBlocker.zig
@@ -59,10 +59,6 @@ suppressed: u32,
blocking: Engine,
blocking_important: Engine,
exceptions: Engine,
-/// `@@…$important` exceptions, indexed apart because they outrank the
-/// `$important` blocks while plain exceptions do not — one engine returning
-/// its first match could not tell the two apart.
-exceptions_important: Engine,
/// Rules that reached a trie or an index, across every list parsed so far.
rules_loaded: usize,
@@ -96,7 +92,6 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
.blocking = .empty,
.blocking_important = .empty,
.exceptions = .empty,
- .exceptions_important = .empty,
.rules_loaded = 0,
.rules_skipped = 0,
};
@@ -106,7 +101,6 @@ pub fn deinit(self: *AdBlocker) void {
self.blocking.deinit(self.allocator);
self.blocking_important.deinit(self.allocator);
self.exceptions.deinit(self.allocator);
- self.exceptions_important.deinit(self.allocator);
self.badfilters.deinit(self.allocator);
self.filters.deinit(self.allocator);
self.trie.deinit(self.allocator);
@@ -251,8 +245,6 @@ pub fn build(self: *AdBlocker) !void {
defer important.deinit(self.allocator);
var exceptions: std.ArrayList(u32) = .empty;
defer exceptions.deinit(self.allocator);
- var exceptions_important: std.ArrayList(u32) = .empty;
- defer exceptions_important.deinit(self.allocator);
var kept: usize = 0;
for (self.filters.items) |filter| {
@@ -278,7 +270,7 @@ pub fn build(self: *AdBlocker) !void {
kept += 1;
const list = if (filter.exception)
- (if (filter.important) &exceptions_important else &exceptions)
+ &exceptions
else if (filter.important)
&important
else
@@ -299,7 +291,6 @@ pub fn build(self: *AdBlocker) !void {
self.blocking = try .build(self.allocator, arena, filters, blocking.items, &histogram);
self.blocking_important = try .build(self.allocator, arena, filters, important.items, &histogram);
self.exceptions = try .build(self.allocator, arena, filters, exceptions.items, &histogram);
- self.exceptions_important = try .build(self.allocator, arena, filters, exceptions_important.items, &histogram);
}
fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {
@@ -313,8 +304,8 @@ fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {
pub const Verdict = enum { none, allowed, blocked };
-/// The verdict for one request. uBO's order: `@@…$important` outranks
-/// `$important` blocks, which outrank exceptions, which outrank plain blocks...
+/// The verdict for one request.
+/// `$important` blocks outrank exceptions, which outrank plain blocks; same as uBO.
pub fn match(self: *const AdBlocker, request: Request) Verdict {
std.debug.assert(self.built);
@@ -323,10 +314,6 @@ pub fn match(self: *const AdBlocker, request: Request) Verdict {
// matches it.
if (self.trie.matches(self.suppressed, hostname) != null) return .none;
- // `@@…$important` is what a list writes to beat a `$important` block, so
- // it has to be looked at before one.
- if (self.exceptions_important.match(request) != null) return .allowed;
-
if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;
if (self.blocking_important.match(request) != null) return .blocked;
@@ -352,14 +339,11 @@ fn isSupported(filter: *const NetworkFilter) bool {
/// its subdomains", which is all a trie can express.
fn isWholeHostname(filter: *const NetworkFilter) bool {
if (filter.kind != .hostname) return false;
- // `||host` without '^' also matches hostnames merely *starting* with
- // host ("example.com.evil.org"): broader than a suffix match.
+ // A prefix like `||adform.` also matches hostnames merely *starting*
+ // with it, broader than a suffix match.
if (!filter.require_separator) return false;
// `||host^|` pins the URL end to the separator: narrower.
if (filter.left_anchor or filter.right_anchor) return false;
- // `@@…$important` has to be consulted before the `$important` blocks,
- // which is an order the fixed trie sequence cannot express.
- if (filter.exception and filter.important) return false;
if (!filter.domains.isEmpty()) return false;
if (!filter.first_party or !filter.third_party) return false;
return filter.types.bits() == ResourceTypes.all.bits();
@@ -570,11 +554,6 @@ test "adblock.AdBlocker: verdict precedence" {
\\@@||evil.com^
\\||strict.example.com^$important,script
\\@@||strict.example.com^$script
- \\||both.example.com^$important,script
- \\@@||both.example.com^$important,script
- // Both sides pure hostnames, so both would otherwise be tries.
- \\||pure.example.com^$important
- \\@@||pure.example.com^$important
);
try expectVerdict(&blocker, .blocked, "https://ads.example.com/x", "a.com", script);
@@ -585,9 +564,6 @@ test "adblock.AdBlocker: verdict precedence" {
// ...but $important beats the exception, whichever side is indexed.
try expectVerdict(&blocker, .blocked, "https://evil.com/x", "a.com", script);
try expectVerdict(&blocker, .blocked, "https://strict.example.com/x.js", "a.com", script);
- // ...and $important on the exception takes it back.
- try expectVerdict(&blocker, .allowed, "https://both.example.com/x.js", "a.com", script);
- try expectVerdict(&blocker, .allowed, "https://pure.example.com/x.js", "a.com", script);
try expectVerdict(&blocker, .none, "https://example.com/x", "a.com", script);
}
@@ -650,32 +626,41 @@ test "adblock.AdBlocker: trie absorbs every pure-hostname form" {
try testLoad(&blocker,
\\||anchored.example.com^
+ \\||caretless.example.com
+ \\||pinned.example.com|
\\bare-hostname.example.com
\\0.0.0.0 hosts-style.example.io
);
try expectVerdict(&blocker, .blocked, "https://anchored.example.com/x", "a.com", script);
+ // `||host` and `||host|` are the same filter as `||host^`.
+ try expectVerdict(&blocker, .blocked, "https://sub.caretless.example.com/x", "a.com", script);
+ try expectVerdict(&blocker, .none, "https://caretless.example.com.evil.org/x", "a.com", script);
+ try expectVerdict(&blocker, .blocked, "https://pinned.example.com/x?q=1", "a.com", script);
try expectVerdict(&blocker, .blocked, "https://bare-hostname.example.com/x", "a.com", script);
try expectVerdict(&blocker, .blocked, "https://hosts-style.example.io/x", "a.com", script);
// Pure-hostname filters block top-level documents too.
try expectVerdict(&blocker, .blocked, "https://anchored.example.com/", "a.com", document);
+
+ // Absorbs: every rule above went to a trie, none stayed indexed.
+ try testing.expectEqual(0, blocker.filters.items.len);
}
-test "adblock.AdBlocker: an important exception is found whatever matches first" {
+test "adblock.AdBlocker: $important on an exception is invalid" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
- // The plain exception is written first, so a single first-match engine
- // would return it and never see the important one behind it.
+ // uBO rejects `@@…$important`, so nothing outranks a `$important`
+ // block — and being a rule uBO drops too, the invalid exception does
+ // not suppress its hostname either.
try testLoad(&blocker,
\\||tracker.com^$important,script
- \\@@||tracker.com^$script
\\@@||tracker.com/x.js$important,script
);
- try expectVerdict(&blocker, .allowed, "https://tracker.com/x.js", "a.com", script);
- // Off the important exception's path, the $important block still wins.
- try expectVerdict(&blocker, .blocked, "https://tracker.com/other.js", "a.com", script);
+ try expectVerdict(&blocker, .blocked, "https://tracker.com/x.js", "a.com", script);
+ try testing.expectEqual(1, blocker.rules_loaded);
+ try testing.expectEqual(1, blocker.rules_skipped);
}
test "adblock.AdBlocker: an empty blocker decides nothing" {
diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zig
index fc63f858b..4cf9493f2 100644
--- a/src/network/adblock/NetworkFilter.zig
+++ b/src/network/adblock/NetworkFilter.zig
@@ -416,7 +416,8 @@ fn parseOptions(
self.domains = try domain.parse(arena, v);
},
.important => {
- if (negated) return error.InvalidOption;
+ // Importance is a property of blocks; uBO rejects `@@…$important`.
+ if (negated or self.exception) return error.InvalidOption;
self.important = true;
},
.badfilter => {
@@ -588,12 +589,14 @@ fn parsePattern(
pattern = pattern[0 .. pattern.len - 1];
}
- // Trim pointless wildcards touching the (now removed) ends.
- while (std.mem.startsWith(u8, pattern, "*")) {
- pattern = pattern[1..];
+ // Trim pointless wildcards touching the (now removed) ends. A wildcard bordering
+ // a short word stays; which is what keeps the word a substring rather than a hostname.
+ const stars = std.mem.indexOfNone(u8, pattern, "*") orelse pattern.len;
+ if (stars > 0 and (stars == pattern.len or !isPatternWordChar(pattern[stars]))) {
+ pattern = pattern[stars..];
self.left_anchor = false;
}
- while (std.mem.endsWith(u8, pattern, "*")) {
+ while (pointlessTrailingWildcard(pattern)) {
pattern = pattern[0 .. pattern.len - 1];
self.right_anchor = false;
}
@@ -623,6 +626,10 @@ fn parsePattern(
const remainder = pattern[host_end..];
if (remainder.len == 0) {
self.kind = .hostname;
+ if (isHostnameShaped(self.hostname)) {
+ self.require_separator = true;
+ self.right_anchor = false;
+ }
} else if (std.mem.eql(u8, remainder, "^")) {
self.kind = .hostname;
self.require_separator = true;
@@ -633,14 +640,14 @@ fn parsePattern(
return;
}
- // A pattern that reads as a bare hostname is a hostname filter
- // (`ads.example.com` == `||ads.example.com^`)
- if (!self.left_anchor and !self.right_anchor and isHostnameShaped(pattern)) {
+ if (isHostnameShaped(pattern)) {
if (isRedirectHostName(pattern)) return error.Ignored;
self.kind = .hostname;
self.hostname = pattern;
self.hostname_anchor = true;
self.require_separator = true;
+ self.left_anchor = false;
+ self.right_anchor = false;
return;
}
@@ -652,6 +659,19 @@ fn parsePattern(
self.pattern = pattern;
}
+/// Ditto uBO's `rePointlessTrailingWildcards`.
+fn pointlessTrailingWildcard(pattern: []const u8) bool {
+ if (pattern.len == 0 or pattern[pattern.len - 1] != '*') return false;
+ const before = std.mem.trimEnd(u8, pattern, "*");
+ var run: usize = 0;
+ while (run < before.len and isPatternWordChar(before[before.len - 1 - run])) run += 1;
+ return run == 0 or run >= 7;
+}
+
+inline fn isPatternWordChar(c: u8) bool {
+ return std.ascii.isAlphanumeric(c) or c == '%';
+}
+
/// Matches uBO's hostname flavor: dot-separated labels of [a-z0-9_-], each
/// starting alphanumeric; the last label is [a-z0-9][a-z0-9-]*[a-z0-9]
/// (len >= 2, no '_'). Accepts IPv4 literals. Input must be lowercase.
@@ -705,12 +725,17 @@ test "adblock.NetworkFilter: pure hostname forms" {
// Implicit "strict" blocking: documents included.
try testing.expectEqual(ResourceTypes.all.bits(), f.types.bits());
- // Without '^' there is no separator requirement and no implicit
- // document blocking.
+ // `||host` and `||host|` are, in uBO, the same filter as `||host^`:
+ // the request hostname is `host` or a subdomain of it.
f = try testParse(arena, "||ads.example.com");
try testing.expectEqual(.hostname, f.kind);
- try testing.expect(!f.require_separator);
- try testing.expectEqual(ResourceTypes.all_network.bits(), f.types.bits());
+ try testing.expect(f.require_separator);
+ try testing.expectEqual(ResourceTypes.all.bits(), f.types.bits());
+
+ f = try testParse(arena, "||ads.example.com|");
+ try testing.expectEqual(.hostname, f.kind);
+ try testing.expect(f.require_separator);
+ try testing.expect(!f.right_anchor);
// Bare hostname line == ||host^ (uBO divergence from ABP).
f = try testParse(arena, "tracker.example.net");
@@ -718,6 +743,18 @@ test "adblock.NetworkFilter: pure hostname forms" {
try testing.expectString("tracker.example.net", f.hostname);
try testing.expect(f.require_separator);
+ // An anchor on a bare hostname is part of the same reading.
+ f = try testParse(arena, "tracker.example.net|");
+ try testing.expectEqual(.hostname, f.kind);
+ try testing.expectString("tracker.example.net", f.hostname);
+ try testing.expect(f.require_separator);
+ try testing.expect(!f.right_anchor);
+
+ f = try testParse(arena, "|tracker.example.net");
+ try testing.expectEqual(.hostname, f.kind);
+ try testing.expectString("tracker.example.net", f.hostname);
+ try testing.expect(!f.left_anchor);
+
// Raw IPv4 lines (URLhaus style).
f = try testParse(arena, "101.126.11.168");
try testing.expectEqual(.hostname, f.kind);
@@ -778,11 +815,23 @@ test "adblock.NetworkFilter: anchors and pattern kinds" {
try testing.expectEqual(.plain, f.kind);
try testing.expect(!f.right_anchor);
- // A pattern that trims down to a bare hostname shape gets promoted
- // (uBO flavor rules), even a single label.
+ // A '*' bordering a short alphanumeric run is meaningful (uBO's
+ // pointless-wildcard rules), so no trimming and no hostname promotion.
f = try testParse(arena, "*ads*|");
+ try testing.expectEqual(.wildcard, f.kind);
+ try testing.expectString("*ads*", f.pattern);
+
+ // A trailing '*' after 7+ chars drops, and a pattern that trims down
+ // to a bare hostname shape gets promoted (uBO flavor rules)...
+ f = try testParse(arena, "ads.example*");
try testing.expectEqual(.hostname, f.kind);
- try testing.expectString("ads", f.hostname);
+ try testing.expectString("ads.example", f.hostname);
+
+ // ...but a leading '*' before a word survives, and keeps the pattern a
+ // substring match rather than a hostname.
+ f = try testParse(arena, "*ads.example*");
+ try testing.expectEqual(.wildcard, f.kind);
+ try testing.expectString("*ads.example", f.pattern);
// '||' hostname region containing '*' stays a generic pattern.
f = try testParse(arena, "||example.*/ads");
@@ -964,6 +1013,9 @@ test "adblock.NetworkFilter: important, badfilter, noop" {
var f = try testParse(arena, "||ads.com^$important");
try testing.expect(f.important);
+ // Importance is a property of blocks; uBO rejects it on an exception.
+ try testing.expectError(error.InvalidOption, testParse(arena, "@@||ads.com^$important"));
+
f = try testParse(arena, "||ads.com^$badfilter");
try testing.expect(f.badfilter);
diff --git a/src/network/adblock/pattern.zig b/src/network/adblock/pattern.zig
index 012875a98..6bd2d085d 100644
--- a/src/network/adblock/pattern.zig
+++ b/src/network/adblock/pattern.zig
@@ -271,21 +271,31 @@ test "adblock.pattern: hostname filters cover subdomains" {
try testing.expect(!try testMatch(arena, "||ads.example.com^", "https://example.com/ads.example.com"));
}
-test "adblock.pattern: caret-less hostname filters end on a label boundary" {
+test "adblock.pattern: caret-less hostname filters match the hostname or a subdomain" {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
- // The real EasyPrivacy shape "||coinimp.com$third-party": without a '^'
- // the next label may continue ("coinimp.com.evil.org") but the last one
- // may not ("coinimp.community" is someone else's site).
+ // The real EasyPrivacy shape "||coinimp.com$third-party": uBO reads it
+ // as "||coinimp.com^", so "coinimp.com.evil.org" — evil.org's hostname,
+ // not a subdomain of coinimp.com — is out.
try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com/miner.js"));
try testing.expect(try testMatch(arena, "||coinimp.com", "https://sub.coinimp.com/x"));
- try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com.evil.org/x"));
try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com:8080/x"));
+ try testing.expect(!try testMatch(arena, "||coinimp.com", "https://coinimp.com.evil.org/x"));
try testing.expect(!try testMatch(arena, "||coinimp.com", "https://coinimp.community/miner.js"));
- // A filter hostname ending in '.' carries its own boundary.
+ // `||host|` is that same filter once more.
+ try testing.expect(try testMatch(arena, "||coinimp.com|", "https://sub.coinimp.com/x"));
+ try testing.expect(!try testMatch(arena, "||coinimp.com|", "https://coinimp.com.evil.org/x"));
+
+ // `||host*` is NOT: uBO keeps a trailing '*' after a short run, and the
+ // pattern stays a label-anchored prefix.
+ try testing.expect(try testMatch(arena, "||coinimp.com*", "https://coinimp.com.evil.org/x"));
+ try testing.expect(try testMatch(arena, "||coinimp.com*", "https://coinimp.community/miner.js"));
+
+ // A filter hostname ending in '.' names no hostname; it stays a prefix
+ // and carries its own boundary.
try testing.expect(try testMatch(arena, "||adform.", "https://adform.net/banner"));
try testing.expect(try testMatch(arena, "||adform.", "https://track.adform.co.uk/x"));
try testing.expect(!try testMatch(arena, "||adform", "https://adformat.example/x"));