Skip to content

Lightpanda: the headless browser designed for AI and automation

`AdBlocker`: initial implementation for request engine

Halil Durak0b2f14194e0d
diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zig
new file mode 100644
index 000000000..8b84705cc
--- /dev/null
+++ b/src/network/adblock/Engine.zig
@@ -0,0 +1,319 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! One indexed set of network filters — the blocking ones, the `$important`
+//! ones or the exceptions — and the lookup that finds which of them a request
+//! matches. A port of adblock-rust's `NetworkFilterList`.
+//!
+//! Checking every filter against every request is not affordable: EasyList
+//! alone carries tens of thousands. Instead each filter is filed under one
+//! token of its pattern — a run of alphanumerics that any URL it matches is
+//! bound to contain — and a request only ever pays for the filters filed
+//! under the tokens its own URL happens to have.
+//!
+//! Which token decides how much that costs, so each filter takes its rarest
+//! one across the whole loaded corpus: "gampad" narrows to a handful of
+//! filters where "com" would narrow to nothing. Filters whose pattern offers
+//! no token that is guaranteed to survive into the URL (see `collectTokens`)
+//! go to `fallback`, which every request pays for.
+
+const std = @import("std");
+
+const domain = @import("domain.zig");
+const pattern = @import("pattern.zig");
+const NetworkFilter = @import("NetworkFilter.zig");
+
+const Allocator = std.mem.Allocator;
+
+const Engine = @This();
+
+/// The shared filter array; `buckets` and `fallback` hold indices into it.
+filters: []const NetworkFilter,
+buckets: std.AutoHashMapUnmanaged(u32, []const u32),
+/// Filters no token can stand for; checked on every request.
+fallback: []const u32,
+
+pub const empty: Engine = .{
+ .filters = &.{},
+ .buckets = .empty,
+ .fallback = &.{},
+};
+
+pub fn deinit(self: *Engine, allocator: Allocator) void {
+ self.buckets.deinit(allocator);
+ self.* = .empty;
+}
+
+/// Everything the filters need to know about a request. Built once per
+/// request and shared by all three engines.
+pub const Request = struct {
+ url: pattern.Url,
+ /// The hostname of the document the request belongs to. Falls back to the
+ /// request's own hostname, which is what uBO does for top-level loads.
+ source_hostname: []const u8,
+ /// Exactly one bit set.
+ kind: NetworkFilter.ResourceTypes,
+ third_party: bool,
+
+ /// `url` must already be lowercased and fragment-free; `source_hostname`
+ /// may be empty when there is no document context.
+ pub fn init(
+ url: []const u8,
+ source_hostname: []const u8,
+ kind: NetworkFilter.ResourceTypes,
+ ) Request {
+ const parsed: pattern.Url = .init(url);
+ const source = if (source_hostname.len == 0) parsed.hostname() else source_hostname;
+ return .{
+ .url = parsed,
+ .source_hostname = source,
+ .kind = kind,
+ .third_party = domain.isThirdParty(parsed.hostname(), source),
+ };
+ }
+};
+
+/// How often each token appears across every indexed filter. Shared by the
+/// engines so that "rarest" means rarest overall, not within one class.
+pub const Histogram = std.AutoHashMapUnmanaged(u32, u32);
+
+/// Tokens considered per filter. Long query-string patterns can exceed this;
+/// taking the first few costs a wider bucket, never a wrong answer.
+const TOKENS_MAX = 32;
+
+/// The first filter that matches, or null.
+pub fn match(self: *const Engine, request: Request) ?*const NetworkFilter {
+ if (self.filters.len != 0) {
+ var it: Tokens = .{ .text = request.url.text };
+ while (it.next()) |token| {
+ const bucket = self.buckets.get(token) orelse continue;
+ if (self.matchIn(bucket, request)) |filter| return filter;
+ }
+ }
+ return self.matchIn(self.fallback, request);
+}
+
+fn matchIn(self: *const Engine, bucket: []const u32, request: Request) ?*const NetworkFilter {
+ for (bucket) |index| {
+ const filter = &self.filters[index];
+ if (matchesFilter(filter, request)) return filter;
+ }
+ return null;
+}
+
+/// Cheapest constraint first: the type and party bits are two comparisons,
+/// the pattern walk is the expensive one.
+pub fn matchesFilter(filter: *const NetworkFilter, request: Request) bool {
+ if (filter.types.bits() & request.kind.bits() == 0) return false;
+ if (request.third_party) {
+ if (!filter.third_party) return false;
+ } else if (!filter.first_party) {
+ return false;
+ }
+ if (!filter.domains.matches(request.source_hostname)) return false;
+ return pattern.matches(filter, request.url);
+}
+
+/// Adds `filter`'s tokens to the corpus counts.
+pub fn count(
+ histogram: *Histogram,
+ allocator: Allocator,
+ filter: *const NetworkFilter,
+) Allocator.Error!void {
+ var buf: [TOKENS_MAX]u32 = undefined;
+ for (collectTokens(filter, &buf)) |token| {
+ const gop = try histogram.getOrPut(allocator, token);
+ gop.value_ptr.* = if (gop.found_existing) gop.value_ptr.* + 1 else 1;
+ }
+}
+
+/// Indexes `members` (indices into `filters`) by their rarest token.
+/// `allocator` backs the bucket map, `arena` the index slices it points at.
+pub fn build(
+ allocator: Allocator,
+ arena: Allocator,
+ filters: []const NetworkFilter,
+ members: []const u32,
+ histogram: *const Histogram,
+) Allocator.Error!Engine {
+ var groups: std.AutoHashMapUnmanaged(u32, std.ArrayList(u32)) = .empty;
+ defer {
+ var it = groups.valueIterator();
+ while (it.next()) |list| list.deinit(allocator);
+ groups.deinit(allocator);
+ }
+
+ var fallback: std.ArrayList(u32) = .empty;
+ defer fallback.deinit(allocator);
+
+ var buf: [TOKENS_MAX]u32 = undefined;
+ for (members) |index| {
+ var rarest: ?u32 = null;
+ var lowest: u32 = std.math.maxInt(u32);
+ for (collectTokens(&filters[index], &buf)) |token| {
+ const seen = histogram.get(token) orelse 0;
+ if (seen < lowest) {
+ lowest = seen;
+ rarest = token;
+ }
+ }
+
+ const token = rarest orelse {
+ try fallback.append(allocator, index);
+ continue;
+ };
+ const gop = try groups.getOrPut(allocator, token);
+ if (!gop.found_existing) gop.value_ptr.* = .empty;
+ try gop.value_ptr.append(allocator, index);
+ }
+
+ var buckets: std.AutoHashMapUnmanaged(u32, []const u32) = .empty;
+ errdefer buckets.deinit(allocator);
+ try buckets.ensureTotalCapacity(allocator, groups.count());
+
+ var it = groups.iterator();
+ while (it.next()) |entry| {
+ buckets.putAssumeCapacity(entry.key_ptr.*, try arena.dupe(u32, entry.value_ptr.items));
+ }
+
+ return .{
+ .filters = filters,
+ .buckets = buckets,
+ .fallback = try arena.dupe(u32, fallback.items),
+ };
+}
+
+/// Runs of alphanumerics. Everything else ('.', '/', '-', '%', '_', '?') is a boundary.
+pub const Tokens = struct {
+ text: []const u8,
+ i: usize = 0,
+
+ pub fn next(self: *Tokens) ?u32 {
+ while (self.i < self.text.len and !isTokenChar(self.text[self.i])) self.i += 1;
+ if (self.i == self.text.len) return null;
+
+ const start = self.i;
+ while (self.i < self.text.len and isTokenChar(self.text[self.i])) self.i += 1;
+ return hash(self.text[start..self.i]);
+ }
+};
+
+fn isTokenChar(c: u8) bool {
+ return std.ascii.isAlphanumeric(c);
+}
+
+fn hash(token: []const u8) u32 {
+ return @truncate(std.hash.Wyhash.hash(0, token));
+}
+
+fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
+ var n: usize = 0;
+
+ if (filter.hostname.len != 0) {
+ var it: Tokens = .{ .text = filter.hostname };
+ while (it.next()) |token| {
+ if (n == buf.len) return buf[0..n];
+ buf[n] = token;
+ n += 1;
+ }
+ }
+
+ // A /regex/ literal is never indexed (nothing can run it), and `.any`
+ // has no pattern at all.
+ if (filter.kind == .regex or filter.pattern.len == 0) return buf[0..n];
+
+ const text = filter.pattern;
+ var i: usize = 0;
+ while (i < text.len) {
+ if (!isTokenChar(text[i])) {
+ i += 1;
+ continue;
+ }
+ const start = i;
+ while (i < text.len and isTokenChar(text[i])) i += 1;
+
+ const left_bounded = if (start == 0)
+ filter.left_anchor or filter.hostname_anchor
+ else
+ text[start - 1] != '*';
+ const right_bounded = if (i == text.len) filter.right_anchor else text[i] != '*';
+
+ if (left_bounded and right_bounded) {
+ if (n == buf.len) return buf[0..n];
+ buf[n] = hash(text[start..i]);
+ n += 1;
+ }
+ }
+
+ return buf[0..n];
+}
+
+const testing = @import("../../testing.zig");
+
+fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 {
+ const filter = try NetworkFilter.parse(arena, line);
+ return collectTokens(&filter, buf);
+}
+
+fn contains(tokens: []const u32, token: []const u8) bool {
+ for (tokens) |t| {
+ if (t == hash(token)) return true;
+ }
+ return false;
+}
+
+test "adblock.Engine: only tokens the URL must reproduce are collected" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ var buf: [TOKENS_MAX]u32 = undefined;
+
+ // Every hostname label is bounded, the last one by the matcher's
+ // label-boundary requirement — with or without '^'.
+ var tokens = try tokensOf(arena, "||ads.example.com^", &buf);
+ try testing.expect(contains(tokens, "ads"));
+ try testing.expect(contains(tokens, "example"));
+ try testing.expect(contains(tokens, "com"));
+
+ tokens = try tokensOf(arena, "||ads.example.com", &buf);
+ try testing.expect(contains(tokens, "example"));
+ try testing.expect(contains(tokens, "com"));
+
+ // A path following the hostname bounds it just as well.
+ tokens = try tokensOf(arena, "||youtube.com/pagead/", &buf);
+ try testing.expect(contains(tokens, "com"));
+ try testing.expect(contains(tokens, "pagead"));
+
+ // A trailing token is unbounded unless the pattern is right-anchored:
+ // "/ads" also matches "/adserver".
+ tokens = try tokensOf(arena, "||example.com/ads", &buf);
+ try testing.expect(!contains(tokens, "ads"));
+ tokens = try tokensOf(arena, "||example.com/ads|", &buf);
+ try testing.expect(contains(tokens, "ads"));
+
+ // A '*' on either side disqualifies the token beside it, so this one is
+ // carried entirely by its hostname.
+ tokens = try tokensOf(arena, "||example.com/a*banner*c^", &buf);
+ try testing.expect(!contains(tokens, "banner"));
+ try testing.expect(contains(tokens, "example"));
+
+ // Nothing to index: option-only filters ride the fallback bucket.
+ tokens = try tokensOf(arena, "$script,domain=example.com", &buf);
+ try testing.expectEqual(0, tokens.len);
+}
diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zig
index dbc2c613a..fc63f858b 100644
--- a/src/network/adblock/NetworkFilter.zig
+++ b/src/network/adblock/NetworkFilter.zig
@@ -26,11 +26,14 @@ kind: PatternKind,
/// for the bare-hostname and hosts-file forms. Empty when the pattern names
/// no hostname (`/ads/banner.`, `|https://…`, `/regex/`).
hostname: []const u8 = "",
+/// Normalized (lowercased) pattern with the anchor markers stripped, and
+/// with `hostname` removed when the filter is `||`-anchored — so
+/// `||ads.com/x*.js` keeps `/x*.js` here. Empty for `.hostname` and `.any`;
+/// the raw literal, slashes included, for `.regex`.
+pattern: []const u8 = "",
types: ResourceTypes = .none,
-/// Whether a `$domain=` list constrains this filter. The entries themselves
-/// are not kept: any of them already makes the filter narrower than the
-/// whole-hostname verdict we can represent.
-has_domains: bool = false,
+/// The `$domain=` constraint; `.none` when the filter carries none.
+domains: domain.List = .none,
exception: bool = false,
important: bool = false,
badfilter: bool = false,
@@ -71,7 +74,8 @@ pub const ResourceTypes = packed struct(u16) {
websocket: bool = false,
ping: bool = false,
other: bool = false,
- _padding: u4 = 0,
+ popup: bool = false,
+ _padding: u3 = 0,
pub const none: ResourceTypes = .{};
@@ -114,6 +118,7 @@ pub const ParseError = error{
UnsupportedOption,
UnsupportedPattern,
NoSupportedDomains,
+ UnsupportedModifier,
// Hosts-file noise ("127.0.0.1 localhost"), skip silently.
Ignored,
} || std.mem.Allocator.Error;
@@ -132,6 +137,8 @@ const Option = enum {
websocket,
ping,
other,
+ popup,
+ popunder,
all,
// Party.
first_party,
@@ -145,8 +152,6 @@ const Option = enum {
specifichide,
elemhide,
// Recognized but unsupported (rule dropped)...
- popup,
- popunder,
inline_script,
inline_font,
genericblock,
@@ -255,7 +260,7 @@ pub fn parse(arena: std.mem.Allocator, line: []const u8) ParseError!NetworkFilte
const split = splitOptions(rest);
var explicit_types = false;
if (split.options) |options| {
- try filter.parseOptions(options, &explicit_types);
+ try filter.parseOptions(arena, options, &explicit_types);
}
if (std.mem.indexOfAny(u8, split.pattern, &std.ascii.whitespace) != null) {
@@ -335,6 +340,7 @@ fn isNoop(name: []const u8) bool {
fn parseOptions(
self: *NetworkFilter,
+ arena: std.mem.Allocator,
options: []const u8,
explicit_types: *bool,
) ParseError!void {
@@ -383,6 +389,8 @@ fn parseOptions(
.websocket,
.ping,
.other,
+ .popup,
+ .popunder,
=> {
if (negated and option == .document) return error.InvalidOption;
explicit_types.* = true;
@@ -405,8 +413,7 @@ fn parseOptions(
if (negated) return error.InvalidOption;
const v = value orelse return error.InvalidOption;
if (v.len == 0) return error.InvalidOption;
- try domain.validate(v);
- self.has_domains = true;
+ self.domains = try domain.parse(arena, v);
},
.important => {
if (negated) return error.InvalidOption;
@@ -443,8 +450,8 @@ fn parseOptions(
explicit_types.* = true;
positive.media = true;
},
- .popup,
- .popunder,
+ // Options that narrow or cancel blocking: an `@@` rule carrying
+ // one can unblock something we do block.
.inline_script,
.inline_font,
.genericblock,
@@ -456,6 +463,9 @@ fn parseOptions(
.strict1p,
.strict3p,
.ipaddress,
+ => return error.UnsupportedOption,
+ // Options that only rewrite a request that was going through
+ // regardless.
.csp,
.permissions,
.removeparam,
@@ -463,7 +473,7 @@ fn parseOptions(
.urlskip,
.uritransform,
.redirect_rule,
- => return error.UnsupportedOption,
+ => return error.UnsupportedModifier,
.webrtc => return error.InvalidOption,
}
}
@@ -494,6 +504,7 @@ fn setType(set: *ResourceTypes, option: Option) void {
.websocket => set.websocket = true,
.ping => set.ping = true,
.other => set.other = true,
+ .popup, .popunder => set.popup = true,
else => unreachable,
}
}
@@ -551,6 +562,7 @@ fn parsePattern(
// Whole-pattern regex literal: /.../ with anything between the slashes.
if (raw.len > 2 and raw[0] == '/' and raw[raw.len - 1] == '/') {
self.kind = .regex;
+ self.pattern = raw;
return;
}
@@ -603,6 +615,7 @@ fn parsePattern(
// Wildcard inside the hostname region (`||example.*/ads`):
// no hostname split, the whole thing is a generic pattern.
self.kind = .wildcard;
+ self.pattern = pattern;
return;
}
if (host_end == 0) return error.InvalidPattern;
@@ -615,6 +628,7 @@ fn parsePattern(
self.require_separator = true;
} else {
self.kind = if (std.mem.indexOfAny(u8, remainder, "*^") != null) .wildcard else .plain;
+ self.pattern = remainder;
}
return;
}
@@ -635,6 +649,7 @@ fn parsePattern(
}
self.kind = if (std.mem.indexOfAny(u8, pattern, "*^") != null) .wildcard else .plain;
+ self.pattern = pattern;
}
/// Matches uBO's hostname flavor: dot-separated labels of [a-z0-9_-], each
@@ -845,6 +860,63 @@ test "adblock.NetworkFilter: type options, aliases and negation" {
try testing.expectError(error.InvalidOption, testParse(arena, "||ads.com^$~document"));
}
+test "adblock.NetworkFilter: $popup is a type no request carries" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // We issue no popup requests, so these load and never fire. Dropping them
+ // instead would leave their `@@` counterparts with nothing to except.
+ var f = try testParse(arena, "||doubleclick.net^$popup");
+ try testing.expect(f.types.popup);
+ try testing.expect(!f.types.script);
+ try testing.expect(!f.types.document);
+
+ f = try testParse(arena, "@@||ad.doubleclick.net/ddm/$popup,domain=nytimes.com");
+ try testing.expect(f.exception);
+ try testing.expect(f.types.popup);
+ try testing.expectString("/ddm/", f.pattern);
+
+ // $popunder is the same bit, and the implicit type sets exclude both.
+ f = try testParse(arena, "||ads.com^$popunder");
+ try testing.expect(f.types.popup);
+ f = try testParse(arena, "||ads.com^");
+ try testing.expect(!f.types.popup);
+ try testing.expect(!ResourceTypes.all.popup);
+}
+
+test "adblock.NetworkFilter: the pattern survives parsing" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // A `||`-anchored pattern keeps only what follows the hostname.
+ var f = try testParse(arena, "||youtube.com/pagead/");
+ try testing.expectString("youtube.com", f.hostname);
+ try testing.expectString("/pagead/", f.pattern);
+
+ f = try testParse(arena, "||g.doubleclick.net/gampad/ads*%20Web%20Player$domain=imasdk.googleapis.com");
+ try testing.expectEqual(.wildcard, f.kind);
+ try testing.expectString("g.doubleclick.net", f.hostname);
+ try testing.expectString("/gampad/ads*%20web%20player", f.pattern);
+
+ // Pure hostname filters carry no pattern at all.
+ f = try testParse(arena, "||ads.example.com^");
+ try testing.expectString("", f.pattern);
+
+ // Unanchored patterns keep the whole thing, anchors stripped.
+ f = try testParse(arena, "|https://ads.");
+ try testing.expectString("https://ads.", f.pattern);
+
+ f = try testParse(arena, "-Ad-300x250.gif|");
+ try testing.expectString("-ad-300x250.gif", f.pattern);
+
+ // A '*' in the hostname region leaves the hostname unsplit.
+ f = try testParse(arena, "||example.*/ads");
+ try testing.expectString("", f.hostname);
+ try testing.expectString("example.*/ads", f.pattern);
+}
+
test "adblock.NetworkFilter: party options" {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
@@ -871,7 +943,12 @@ test "adblock.NetworkFilter: domain option" {
const arena = arena_state.allocator();
const f = try testParse(arena, "||ads.com^$script,domain=news.com|~sports.news.com|google.*");
- try testing.expect(f.has_domains);
+ try testing.expectEqual(2, f.domains.included.len);
+ try testing.expectString("news.com", f.domains.included[0].name);
+ try testing.expectString("google", f.domains.included[1].name);
+ try testing.expect(f.domains.included[1].entity);
+ try testing.expectEqual(1, f.domains.excluded.len);
+ try testing.expectString("sports.news.com", f.domains.excluded[0].name);
try testing.expectError(error.InvalidOption, testParse(arena, "||ads.com^$domain="));
try testing.expectError(error.NoSupportedDomains, testParse(arena, "||ads.com^$domain=/re/"));
@@ -902,10 +979,15 @@ test "adblock.NetworkFilter: unsupported and modifier options" {
// Modifier/rewrite rules are dropped: keeping them as plain blocks
// would over-block (a $removeparam rule matches nearly everything).
- try testing.expectError(error.UnsupportedOption, testParse(arena, "$removeparam=utm_source"));
- try testing.expectError(error.UnsupportedOption, testParse(arena, "||ads.com^$csp=script-src 'none'"));
- try testing.expectError(error.UnsupportedOption, testParse(arena, "||ads.com^$popup"));
- try testing.expectError(error.UnsupportedOption, testParse(arena, "||ads.com^$redirect-rule=noopjs"));
+ // They report a distinct error because they never blocked anything, so
+ // their `@@` form has nothing to unblock either.
+ try testing.expectError(error.UnsupportedModifier, testParse(arena, "$removeparam=utm_source"));
+ try testing.expectError(error.UnsupportedModifier, testParse(arena, "||ads.com^$csp=script-src 'none'"));
+ try testing.expectError(error.UnsupportedModifier, testParse(arena, "||ads.com^$redirect-rule=noopjs"));
+
+ // Options that do narrow blocking keep the blunter error.
+ try testing.expectError(error.UnsupportedOption, testParse(arena, "||ads.com^$denyallow=cdn.com"));
+ try testing.expectError(error.UnsupportedOption, testParse(arena, "||ads.com^$method=get"));
// $redirect keeps its blocking half; the directive itself is ignored.
var f = try testParse(arena, "||ads.com/ad.js$script,redirect=noopjs");
diff --git a/src/network/adblock/Parser.zig b/src/network/adblock/Parser.zig
index 8a1291bba..29c56c3fd 100644
--- a/src/network/adblock/Parser.zig
+++ b/src/network/adblock/Parser.zig
@@ -37,14 +37,30 @@ skipped: usize = 0,
pub const Error = error{ OutOfMemory, ReadFailed };
+/// One rule-shaped line. Dropped lines are surfaced rather than swallowed
+/// because an `@@` rule we cannot read still tells the caller that something
+/// on that hostname is not meant to be blocked.
+pub const Item = union(enum) {
+ filter: NetworkFilter,
+ /// Already counted in `skipped`; the caller only inspects it.
+ dropped: Dropped,
+};
+
+pub const Dropped = struct {
+ line: []const u8,
+ /// Why the filter parser refused it. Null when the line never reached it
+ /// (AdGuard `$$` syntax).
+ reason: ?NetworkFilter.ParseError,
+};
+
pub fn init(reader: *std.Io.Reader) Parser {
return .{ .reader = reader };
}
-/// Returns the next network filter, or null at end of list. Allocations come
-/// from `arena`; the returned filter borrows from it and from the reader's
+/// Returns the next rule-shaped line, or null at end of list. Allocations
+/// come from `arena`; the result borrows from it and from the reader's
/// buffer, so it only stays valid until the following call.
-pub fn next(self: *Parser, arena: Allocator) Error!?NetworkFilter {
+pub fn next(self: *Parser, arena: Allocator) Error!?Item {
while (try self.takeLine()) |raw_line| {
var stripped: []const u8 = raw_line;
if (self.first_line) {
@@ -57,15 +73,21 @@ pub fn next(self: *Parser, arena: Allocator) Error!?NetworkFilter {
switch (LineClass.fromLine(line)) {
.empty, .comment => {},
- .unsupported => self.skipped += 1,
+ .unsupported => {
+ self.skipped += 1;
+ return .{ .dropped = .{ .line = line, .reason = null } };
+ },
.network => {
if (NetworkFilter.parse(arena, line)) |filter| {
- return filter;
+ return .{ .filter = filter };
} else |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
// Anything else is a line outside the supported subset
// or malformed; either way it is not ours to enforce.
- else => self.skipped += 1,
+ else => {
+ self.skipped += 1;
+ return .{ .dropped = .{ .line = line, .reason = err } };
+ },
}
},
}
@@ -154,7 +176,18 @@ test "adblock.Parser: line classification" {
try testing.expectEqual(.unsupported, LineClass.fromLine("example.com$$script[data-x]"));
}
-test "adblock.Parser: yields one filter per next() call" {
+/// The next item that is a filter, skipping over dropped lines.
+fn nextFilter(parser: *Parser, arena: Allocator) !?NetworkFilter {
+ while (try parser.next(arena)) |item| {
+ switch (item) {
+ .filter => |filter| return filter,
+ .dropped => {},
+ }
+ }
+ return null;
+}
+
+test "adblock.Parser: yields one item per next() call" {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
@@ -167,13 +200,19 @@ test "adblock.Parser: yields one filter per next() call" {
var parser: Parser = .init(&reader);
const first = (try parser.next(arena)).?;
- try testing.expectString("ads.example.com", first.hostname);
+ try testing.expectString("ads.example.com", first.filter.hostname);
+ // The cosmetic line comes back as dropped rather than being swallowed,
+ // with the reason the filter parser gave.
const second = (try parser.next(arena)).?;
- try testing.expectString("tracker.net", second.hostname);
+ try testing.expectString("example.com##.ad-banner", second.dropped.line);
+ try testing.expectEqual(error.UnsupportedPattern, second.dropped.reason.?);
- // The header, the cosmetic line and the end of the list all yield
- // nothing, and next() keeps returning null once drained.
+ const third = (try parser.next(arena)).?;
+ try testing.expectString("tracker.net", third.filter.hostname);
+
+ // The header and the end of the list yield nothing, and next() keeps
+ // returning null once drained.
try testing.expect(try parser.next(arena) == null);
try testing.expect(try parser.next(arena) == null);
}
@@ -195,7 +234,7 @@ test "adblock.Parser: only rule-shaped lines count as skipped" {
var parser: Parser = .init(&reader);
var count: usize = 0;
- while (try parser.next(arena)) |_| count += 1;
+ while (try nextFilter(&parser, arena)) |_| count += 1;
// The header, the comment and the blank line are not rules.
try testing.expectEqual(1, count);
@@ -220,14 +259,14 @@ test "adblock.Parser: a line too long for the buffer is skipped" {
var reader = text_reader.limited(.unlimited, &buf);
var parser: Parser = .init(&reader.interface);
- const first = (try parser.next(arena)).?;
+ const first = (try nextFilter(&parser, arena)).?;
try testing.expectString("ads.example.com", first.hostname);
// The oversized line was stepped over, not treated as end of list.
- const second = (try parser.next(arena)).?;
+ const second = (try nextFilter(&parser, arena)).?;
try testing.expectString("tracker.net", second.hostname);
- try testing.expect(try parser.next(arena) == null);
+ try testing.expect(try nextFilter(&parser, arena) == null);
// It is still a line we could not load, and it says so.
try testing.expectEqual(1, parser.skipped);
}
@@ -265,7 +304,7 @@ test "adblock.Parser: full list" {
// A filter only borrows the reader's buffer until the next call, so
// assert on each one as it comes out.
var count: usize = 0;
- while (try parser.next(arena)) |filter| : (count += 1) {
+ while (try nextFilter(&parser, arena)) |filter| : (count += 1) {
switch (count) {
// "-banner-468x60." is not hostname-shaped (leading '-'), so it
// stays a plain pattern rather than becoming a hostname filter.
diff --git a/src/network/adblock/domain.zig b/src/network/adblock/domain.zig
index 90b4d500d..ab0ba3c9c 100644
--- a/src/network/adblock/domain.zig
+++ b/src/network/adblock/domain.zig
@@ -18,10 +18,77 @@
const std = @import("std");
-/// Checks a `$domain=` value. Nothing keeps the entries: a filter carrying
-/// any of them is narrower than a hostname, which is as much as we can
-/// represent, so only "is it well formed" and "did anything survive" matter.
-pub fn validate(raw: []const u8) error{ InvalidDomainList, NoSupportedDomains }!void {
+const public_suffix_list = @import("../../data/public_suffix_list.zig");
+
+const Allocator = std.mem.Allocator;
+
+/// One `$domain=` entry. `entity` marks the "google.*" form, whose `name` is
+/// the hostname with its public suffix removed.
+pub const Entry = struct {
+ name: []const u8,
+ entity: bool = false,
+};
+
+/// A parsed `$domain=` value. Both lists empty means the filter is
+/// unconstrained; `included` empty with `excluded` set means "everywhere but".
+pub const List = struct {
+ included: []const Entry = &.{},
+ excluded: []const Entry = &.{},
+
+ pub const none: List = .{};
+
+ pub fn isEmpty(self: List) bool {
+ return self.included.len == 0 and self.excluded.len == 0;
+ }
+
+ /// uBO semantics.
+ pub fn matches(self: List, hostname: []const u8) bool {
+ if (self.isEmpty()) return true;
+ // Only computed if an entity entry asks for it.
+ const ent = if (hasEntity(self.excluded) or hasEntity(self.included))
+ entity(hostname)
+ else
+ "";
+
+ for (self.excluded) |e| {
+ if (matchEntry(e, hostname, ent)) return false;
+ }
+ if (self.included.len == 0) return true;
+ for (self.included) |e| {
+ if (matchEntry(e, hostname, ent)) return true;
+ }
+ return false;
+ }
+};
+
+fn hasEntity(entries: []const Entry) bool {
+ for (entries) |e| {
+ if (e.entity) return true;
+ }
+ return false;
+}
+
+fn matchEntry(e: Entry, hostname: []const u8, ent: []const u8) bool {
+ if (e.entity) return std.mem.eql(u8, e.name, ent);
+ return isSubdomainOf(hostname, e.name);
+}
+
+/// Whether `hostname` is `suffix` or one of its subdomains.
+pub fn isSubdomainOf(hostname: []const u8, suffix: []const u8) bool {
+ if (hostname.len == suffix.len) return std.mem.eql(u8, hostname, suffix);
+ if (hostname.len < suffix.len + 1) return false;
+ const at = hostname.len - suffix.len;
+ return hostname[at - 1] == '.' and std.mem.eql(u8, hostname[at..], suffix);
+}
+
+/// Parses a `$domain=` value into its inclusions and exclusions. Entries
+/// alias `raw`; only the two slices come from `arena`.
+pub fn parse(
+ arena: Allocator,
+ raw: []const u8,
+) error{ InvalidDomainList, NoSupportedDomains, OutOfMemory }!List {
+ var included: std.ArrayList(Entry) = .empty;
+ var excluded: std.ArrayList(Entry) = .empty;
var had_entries = false;
var any_supported = false;
@@ -31,7 +98,9 @@ pub fn validate(raw: []const u8) error{ InvalidDomainList, NoSupportedDomains }!
if (entry.len == 0) continue;
had_entries = true;
+ var negated = false;
if (entry[0] == '~') {
+ negated = true;
entry = entry[1..];
if (entry.len == 0) return error.InvalidDomainList;
}
@@ -40,13 +109,19 @@ pub fn validate(raw: []const u8) error{ InvalidDomainList, NoSupportedDomains }!
if (entry[0] == '/') continue;
// An entity ("google.*") is the hostname without its public suffix.
+ var is_entity = false;
if (std.mem.endsWith(u8, entry, ".*")) {
+ is_entity = true;
entry = entry[0 .. entry.len - 2];
if (entry.len == 0) return error.InvalidDomainList;
}
if (!isValidHost(entry)) return error.InvalidDomainList;
any_supported = true;
+
+ const name = try lowered(arena, entry);
+ const list = if (negated) &excluded else &included;
+ try list.append(arena, .{ .name = name, .entity = is_entity });
}
if (had_entries and !any_supported) {
@@ -54,6 +129,45 @@ pub fn validate(raw: []const u8) error{ InvalidDomainList, NoSupportedDomains }!
// rule rather than let it apply everywhere.
return error.NoSupportedDomains;
}
+
+ return .{
+ .included = included.items,
+ .excluded = excluded.items,
+ };
+}
+
+pub fn registrable(host: []const u8) []const u8 {
+ if (isIpLiteral(host)) return host;
+ var i = std.mem.lastIndexOfScalar(u8, host, '.') orelse return host;
+ while (true) {
+ i = std.mem.lastIndexOfScalar(u8, host[0..i], '.') orelse return host;
+ const start = i + 1;
+ if (public_suffix_list.lookup(host[start..]) == false) {
+ return host[start..];
+ }
+ }
+}
+
+pub fn entity(host: []const u8) []const u8 {
+ if (isIpLiteral(host)) return "";
+ const reg = registrable(host);
+ const dot = std.mem.indexOfScalar(u8, reg, '.') orelse return "";
+ return reg[0..dot];
+}
+
+/// Whether two hostnames belong to different sites.
+pub fn isThirdParty(request_host: []const u8, source_host: []const u8) bool {
+ if (std.mem.eql(u8, request_host, source_host)) return false;
+ return !std.mem.eql(u8, registrable(request_host), registrable(source_host));
+}
+
+fn isIpLiteral(host: []const u8) bool {
+ if (host.len == 0) return false;
+ if (host[0] == '[') return true; // IPv6, already bracketed by the URL parser
+ for (host) |c| {
+ if (!std.ascii.isDigit(c) and c != '.') return false;
+ }
+ return true;
}
fn isValidHost(host: []const u8) bool {
@@ -85,16 +199,94 @@ pub fn lowered(arena: std.mem.Allocator, s: []const u8) ![]const u8 {
const testing = @import("../../testing.zig");
test "adblock.domain: plain, negated and entity entries" {
- try validate("example.com|~sub.Example.com|google.*");
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ const list = try parse(arena, "example.com|~sub.Example.com|google.*");
+ try testing.expectEqual(2, list.included.len);
+ try testing.expectString("example.com", list.included[0].name);
+ try testing.expect(!list.included[0].entity);
+ try testing.expectString("google", list.included[1].name);
+ try testing.expect(list.included[1].entity);
+
+ try testing.expectEqual(1, list.excluded.len);
+ try testing.expectString("sub.example.com", list.excluded[0].name);
+}
+
+test "adblock.domain: matching against a source hostname" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // An entry matches itself and its subdomains.
+ var list = try parse(arena, "example.com");
+ try testing.expect(list.matches("example.com"));
+ try testing.expect(list.matches("www.example.com"));
+ try testing.expect(!list.matches("notexample.com"));
+ try testing.expect(!list.matches("example.org"));
+
+ // An exclusion wins over the inclusion that covers it.
+ list = try parse(arena, "example.com|~ads.example.com");
+ try testing.expect(list.matches("www.example.com"));
+ try testing.expect(!list.matches("ads.example.com"));
+ try testing.expect(!list.matches("x.ads.example.com"));
+
+ // Exclusions alone mean "everywhere but".
+ list = try parse(arena, "~example.com");
+ try testing.expect(list.matches("other.com"));
+ try testing.expect(!list.matches("example.com"));
+
+ // An entity ignores the public suffix.
+ list = try parse(arena, "google.*");
+ try testing.expect(list.matches("google.com"));
+ try testing.expect(list.matches("www.google.com"));
+ try testing.expect(!list.matches("google.example.com"));
+
+ // No list at all constrains nothing.
+ try testing.expect(List.none.matches("anything.com"));
}
test "adblock.domain: regex entries are skipped, all-regex errors" {
- try validate("/foo/|example.com");
- try testing.expectError(error.NoSupportedDomains, validate("/foo/"));
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ const list = try parse(arena, "/foo/|example.com");
+ try testing.expectEqual(1, list.included.len);
+ try testing.expectError(error.NoSupportedDomains, parse(arena, "/foo/"));
}
test "adblock.domain: invalid entries" {
- try testing.expectError(error.InvalidDomainList, validate("exa mple.com"));
- try testing.expectError(error.InvalidDomainList, validate(".example.com"));
- try testing.expectError(error.InvalidDomainList, validate("~"));
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ try testing.expectError(error.InvalidDomainList, parse(arena, "exa mple.com"));
+ try testing.expectError(error.InvalidDomainList, parse(arena, ".example.com"));
+ try testing.expectError(error.InvalidDomainList, parse(arena, "~"));
+}
+
+// The public suffix list is stubbed to {gov.uk, api.gov.uk} in test builds,
+// so these use .com and gov.uk only.
+test "adblock.domain: registrable domain and entity" {
+ try testing.expectString("example.com", registrable("example.com"));
+ try testing.expectString("example.com", registrable("a.b.example.com"));
+ try testing.expectString("dept.gov.uk", registrable("www.dept.gov.uk"));
+ // No registrable domain to find: single labels and IP literals.
+ try testing.expectString("localhost", registrable("localhost"));
+ try testing.expectString("127.0.0.1", registrable("127.0.0.1"));
+ try testing.expectString("[::1]", registrable("[::1]"));
+
+ try testing.expectString("example", entity("a.example.com"));
+ try testing.expectString("dept", entity("www.dept.gov.uk"));
+ try testing.expectString("", entity("127.0.0.1"));
+}
+
+test "adblock.domain: third party" {
+ try testing.expect(!isThirdParty("example.com", "example.com"));
+ try testing.expect(!isThirdParty("cdn.example.com", "www.example.com"));
+ try testing.expect(isThirdParty("ads.com", "example.com"));
+ // Sharing a public suffix is not sharing a site.
+ try testing.expect(isThirdParty("a.gov.uk", "b.gov.uk"));
}
diff --git a/src/network/adblock/pattern.zig b/src/network/adblock/pattern.zig
new file mode 100644
index 000000000..ad6999ef0
--- /dev/null
+++ b/src/network/adblock/pattern.zig
@@ -0,0 +1,402 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Matching of a filter's URL pattern, once the type, party and `$domain=`
+//! constraints have already passed. Everything here works on a lowercased,
+//! fragment-stripped URL; `$match-case` only survives on /regex/ filters,
+//! which never reach this file.
+
+const std = @import("std");
+
+const NetworkFilter = @import("NetworkFilter.zig");
+
+/// The URL a request is matched against, with its hostname located once so
+/// every filter can reuse the offsets.
+pub const Url = struct {
+ /// Lowercased, fragment stripped.
+ text: []const u8,
+ host_start: u32,
+ host_end: u32,
+
+ pub fn hostname(self: Url) []const u8 {
+ return self.text[self.host_start..self.host_end];
+ }
+
+ /// Locates the hostname inside `url`: after "scheme://", up to the port,
+ /// path, query or end. Offsets, not a slice, because the matcher needs to
+ /// resume the pattern right where the hostname stops.
+ pub fn init(url: []const u8) Url {
+ var start: usize = 0;
+ if (std.mem.indexOf(u8, url, "://")) |scheme| start = scheme + 3;
+
+ var end = url.len;
+ for (url[start..], start..) |c, i| {
+ switch (c) {
+ '/', '?', '#' => {
+ end = i;
+ break;
+ },
+ else => {},
+ }
+ }
+
+ // "user:password@host"
+ if (std.mem.lastIndexOfScalar(u8, url[start..end], '@')) |at| {
+ start += at + 1;
+ }
+
+ // A ':' before the path is a port, unless we are inside an IPv6
+ // literal's brackets.
+ const host = url[start..end];
+ var host_end = end;
+ if (host.len > 0 and host[0] == '[') {
+ if (std.mem.indexOfScalar(u8, host, ']')) |close| {
+ host_end = start + close + 1;
+ }
+ } else if (std.mem.indexOfScalar(u8, host, ':')) |colon| {
+ host_end = start + colon;
+ }
+
+ return .{
+ .text = url,
+ .host_start = @intCast(start),
+ .host_end = @intCast(host_end),
+ };
+ }
+};
+
+/// Whether `filter`'s pattern matches `url`. Only the pattern: the caller has
+/// already decided the filter applies to this request.
+pub fn matches(filter: *const NetworkFilter, url: Url) bool {
+ switch (filter.kind) {
+ // Option-only filters ("$script,domain=x") match any URL.
+ .any => return true,
+ // Never indexed: there is no regex engine to run them with.
+ .regex => return false,
+ .hostname, .plain, .wildcard => {},
+ }
+
+ if (filter.hostname.len != 0) {
+ var from: usize = 0;
+ while (hostAnchor(url, filter.hostname, from)) |at| {
+ if (filter.kind == .hostname) {
+ const boundary = at == url.host_end or url.text[at] == '.' or
+ filter.hostname[filter.hostname.len - 1] == '.';
+ if (boundary) {
+ if (!filter.require_separator) return true;
+ if (at == url.text.len) return true;
+ // `||host^|` pins the separator to the end of the URL.
+ if (isSeparator(url.text[at]) and
+ (!filter.right_anchor or at + 1 == url.text.len))
+ {
+ return true;
+ }
+ }
+ } else if (matchFrom(filter, url.text, at, filter.right_anchor)) {
+ return true;
+ }
+ // The label this attempt started on cannot match again.
+ from = at - filter.hostname.len - url.host_start + 1;
+ }
+ return false;
+ }
+
+ if (filter.hostname_anchor) {
+ var i: usize = url.host_start;
+ while (true) {
+ if (matchFrom(filter, url.text, i, filter.right_anchor)) return true;
+ const dot = std.mem.indexOfScalarPos(u8, url.text[0..url.host_end], i, '.') orelse
+ return false;
+ i = dot + 1;
+ }
+ }
+
+ if (filter.left_anchor) {
+ return matchFrom(filter, url.text, 0, filter.right_anchor);
+ }
+ if (filter.kind == .plain) {
+ if (filter.right_anchor) return std.mem.endsWith(u8, url.text, filter.pattern);
+ return std.mem.indexOf(u8, url.text, filter.pattern) != null;
+ }
+
+ var i: usize = 0;
+ while (i <= url.text.len) : (i += 1) {
+ if (matchFrom(filter, url.text, i, filter.right_anchor)) return true;
+ }
+ return false;
+}
+
+/// Finds `needle` starting on a label boundary of the request hostname at or
+/// after `from` (an offset into the hostname), and returns the offset just
+/// past it in the whole URL.
+fn hostAnchor(url: Url, needle: []const u8, from: usize) ?usize {
+ const host = url.hostname();
+ var p = from;
+ while (p <= host.len) {
+ if (p == 0 or host[p - 1] == '.') {
+ if (std.mem.startsWith(u8, host[p..], needle)) {
+ return url.host_start + p + needle.len;
+ }
+ }
+ const dot = std.mem.indexOfScalarPos(u8, host, p, '.') orelse return null;
+ p = dot + 1;
+ }
+ return null;
+}
+
+/// Matches the pattern starting exactly at `start`. `anchor_end` additionally
+/// requires it to run to the end of the URL.
+fn matchFrom(filter: *const NetworkFilter, url: []const u8, start: usize, anchor_end: bool) bool {
+ if (filter.kind == .plain) {
+ // No '*' and no '^': a byte comparison is the whole job.
+ if (start + filter.pattern.len > url.len) return false;
+ if (!std.mem.startsWith(u8, url[start..], filter.pattern)) return false;
+ return !anchor_end or start + filter.pattern.len == url.len;
+ }
+
+ var it = std.mem.splitScalar(u8, filter.pattern, '*');
+ var pos = matchSegment(it.first(), url, start) orelse return false;
+ var pending = it.next();
+ while (pending) |segment| {
+ const following = it.next();
+ if (following == null and anchor_end) {
+ return matchSegmentAtEnd(segment, url, pos);
+ }
+ pos = findSegment(segment, url, pos) orelse return false;
+ pending = following;
+ }
+ return !anchor_end or pos == url.len;
+}
+
+/// Matches one wildcard-free segment at `pos`, returning where it ends.
+fn matchSegment(segment: []const u8, url: []const u8, pos: usize) ?usize {
+ var i = pos;
+ for (segment, 0..) |c, si| {
+ if (i == url.len) {
+ // A '^' at the very end of the pattern also matches the end of
+ // the URL, consuming nothing.
+ return if (c == '^' and si == segment.len - 1) i else null;
+ }
+ if (c == '^') {
+ if (!isSeparator(url[i])) return null;
+ } else if (url[i] != c) {
+ return null;
+ }
+ i += 1;
+ }
+ return i;
+}
+
+/// Leftmost occurrence of `segment` at or after `from`; no backtracking, the
+/// same simplification uBO and adblock-rust make.
+fn findSegment(segment: []const u8, url: []const u8, from: usize) ?usize {
+ var i = from;
+ while (i <= url.len) : (i += 1) {
+ if (matchSegment(segment, url, i)) |end| return end;
+ }
+ return null;
+}
+
+fn matchSegmentAtEnd(segment: []const u8, url: []const u8, from: usize) bool {
+ var i = from;
+ while (i <= url.len) : (i += 1) {
+ if (matchSegment(segment, url, i)) |end| {
+ if (end == url.len) return true;
+ }
+ }
+ return false;
+}
+
+/// uBO's separator class: everything that cannot appear in a hostname or an unescaped path word.
+pub fn isSeparator(c: u8) bool {
+ if (std.ascii.isAlphanumeric(c)) return false;
+ return switch (c) {
+ '_', '%', '.', '-' => false,
+ else => true,
+ };
+}
+
+const testing = @import("../../testing.zig");
+
+fn testMatch(arena: std.mem.Allocator, line: []const u8, url: []const u8) !bool {
+ const filter = try NetworkFilter.parse(arena, line);
+ return matches(&filter, .init(url));
+}
+
+test "adblock.pattern: hostname location" {
+ var u: Url = .init("https://ads.example.com/x?y=1");
+ try testing.expectString("ads.example.com", u.hostname());
+
+ u = .init("https://ads.example.com:8443/x");
+ try testing.expectString("ads.example.com", u.hostname());
+
+ u = .init("https://example.com");
+ try testing.expectString("example.com", u.hostname());
+
+ u = .init("http://[::1]:9222/json");
+ try testing.expectString("[::1]", u.hostname());
+
+ // Credentials are not part of the host.
+ u = .init("https://user:[email protected]/x");
+ try testing.expectString("ads.example.com", u.hostname());
+
+ // A pattern is matched against whatever it is given, even a bare path.
+ u = .init("/relative/path");
+ try testing.expectString("", u.hostname());
+}
+
+test "adblock.pattern: hostname filters cover subdomains" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ try testing.expect(try testMatch(arena, "||ads.example.com^", "https://ads.example.com/a"));
+ try testing.expect(try testMatch(arena, "||ads.example.com^", "https://x.ads.example.com/a"));
+ try testing.expect(!try testMatch(arena, "||ads.example.com^", "https://notads.example.com/a"));
+ try testing.expect(!try testMatch(arena, "||ads.example.com^", "https://example.com/ads.example.com"));
+}
+
+test "adblock.pattern: caret-less hostname filters end on a label boundary" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // The real EasyPrivacy shape "||coinimp.com$third-party": without a '^'
+ // the next label may continue ("coinimp.com.evil.org") but the last one
+ // may not ("coinimp.community" is someone else's site).
+ try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com/miner.js"));
+ try testing.expect(try testMatch(arena, "||coinimp.com", "https://sub.coinimp.com/x"));
+ try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com.evil.org/x"));
+ try testing.expect(try testMatch(arena, "||coinimp.com", "https://coinimp.com:8080/x"));
+ try testing.expect(!try testMatch(arena, "||coinimp.com", "https://coinimp.community/miner.js"));
+
+ // A filter hostname ending in '.' carries its own boundary.
+ try testing.expect(try testMatch(arena, "||adform.", "https://adform.net/banner"));
+ try testing.expect(try testMatch(arena, "||adform.", "https://track.adform.co.uk/x"));
+ try testing.expect(!try testMatch(arena, "||adform", "https://adformat.example/x"));
+}
+
+test "adblock.pattern: right-anchored hostname filters pin the URL end" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // `||host^|`: the separator '^' must also be the end of the URL.
+ try testing.expect(try testMatch(arena, "||ads.com^|", "https://ads.com"));
+ try testing.expect(try testMatch(arena, "||ads.com^|", "https://ads.com/"));
+ try testing.expect(!try testMatch(arena, "||ads.com^|", "https://ads.com/path/x"));
+ try testing.expect(!try testMatch(arena, "||ads.com^|", "https://ads.com/?q=1"));
+}
+
+test "adblock.pattern: hostname-anchored patterns resume after the host" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // The real shape from the report.
+ try testing.expect(try testMatch(
+ arena,
+ "||youtube.com/pagead/",
+ "https://www.youtube.com/pagead/lvz?ei=x",
+ ));
+ try testing.expect(!try testMatch(
+ arena,
+ "||youtube.com/pagead/",
+ "https://www.youtube.com/watch?v=x",
+ ));
+ // The path is anchored to the end of the hostname, not searched for.
+ try testing.expect(!try testMatch(
+ arena,
+ "||youtube.com/pagead/",
+ "https://example.com/www.youtube.com/pagead/x",
+ ));
+
+ try testing.expect(try testMatch(
+ arena,
+ "||g.doubleclick.net/gampad/ads",
+ "https://g.doubleclick.net/gampad/ads?env=vp",
+ ));
+ try testing.expect(!try testMatch(
+ arena,
+ "||g.doubleclick.net/gampad/ads",
+ "https://g.doubleclick.net/pagead/ads",
+ ));
+}
+
+test "adblock.pattern: wildcards" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ try testing.expect(try testMatch(
+ arena,
+ "||g.doubleclick.net/gampad/ads*%20web%20player",
+ "https://g.doubleclick.net/gampad/ads?a=1&b=my%20web%20player",
+ ));
+ try testing.expect(!try testMatch(
+ arena,
+ "||g.doubleclick.net/gampad/ads*%20web%20player",
+ "https://g.doubleclick.net/gampad/ads?a=1",
+ ));
+
+ // A '*' in the hostname region anchors to a label boundary.
+ try testing.expect(try testMatch(arena, "||example.*/ads", "https://example.co.uk/ads/x"));
+ try testing.expect(try testMatch(arena, "||example.*/ads", "https://www.example.com/ads/x"));
+ try testing.expect(!try testMatch(arena, "||example.*/ads", "https://notexample.com/ads/x"));
+
+ try testing.expect(try testMatch(arena, "/banner/*/img", "https://a.com/banner/x/img"));
+}
+
+test "adblock.pattern: separators" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ // '^' matches one separator...
+ try testing.expect(try testMatch(arena, "||example.com/ads^", "https://example.com/ads?x=1"));
+ try testing.expect(try testMatch(arena, "||example.com/ads^", "https://example.com/ads/x"));
+ // ...or the end of the URL.
+ try testing.expect(try testMatch(arena, "||example.com/ads^", "https://example.com/ads"));
+ // Letters, digits, '_', '%', '.' and '-' are not separators.
+ try testing.expect(!try testMatch(arena, "||example.com/ads^", "https://example.com/adsx"));
+ try testing.expect(!try testMatch(arena, "||example.com/ads^", "https://example.com/ads.js"));
+}
+
+test "adblock.pattern: anchors" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ try testing.expect(try testMatch(arena, "|https://ads.", "https://ads.example.com/"));
+ try testing.expect(!try testMatch(arena, "|https://ads.", "http://ads.example.com/"));
+
+ try testing.expect(try testMatch(arena, "-ad-300x250.gif|", "https://a.com/-ad-300x250.gif"));
+ try testing.expect(!try testMatch(arena, "-ad-300x250.gif|", "https://a.com/-ad-300x250.gif?x"));
+
+ // Unanchored patterns are plain substrings.
+ try testing.expect(try testMatch(arena, "/banner/ads.", "https://a.com/x/banner/ads.png"));
+ try testing.expect(!try testMatch(arena, "/banner/ads.", "https://a.com/banner/x.png"));
+}
+
+test "adblock.pattern: option-only filters match every URL" {
+ var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ try testing.expect(try testMatch(arena, "$script,domain=example.com", "https://anything.test/x"));
+}