Skip to content

Lightpanda: the headless browser designed for AI and automation

pass `*const Request` where possible

Halil Durakfb58901604ee
diff --git a/src/Metrics.zig b/src/Metrics.zig
index 120299cec..66c7c6d69 100644
--- a/src/Metrics.zig
+++ b/src/Metrics.zig
@@ -95,7 +95,7 @@ cors_check: CounterEnum("result", enum { same_origin, no_cors, simple, preflight
cors_preflight: CounterEnum("result", enum { allowed, blocked }) = .{},
cors_response: CounterEnum("result", enum { allowed, blocked }) = .{},
adblock_verdicts: CounterEnum("verdict", @import("network/adblock/AdBlocker.zig").Verdict) = .{},
-adblock_rules: GaugeEnum("state", enum { loaded, skipped }) = .{},
+adblock_rules: GaugeEnum("state", enum { loaded, skipped, cosmetic }) = .{},
// Emitted as each metric's "# HELP" line. A field without an entry is a
// compile error.
@@ -131,8 +131,8 @@ const help = .{
.cors_check = "CORS initial classification: same_origin/no_cors need no CORS handling, simple needs response validation only, preflight needs an OPTIONS round-trip first",
.cors_preflight = "CORS preflight (OPTIONS) results, one per request that required one",
.cors_response = "CORS actual-response validation results",
- .adblock_verdicts = "Adblocker decisions for evaluated requests, by verdict (none = no filter matched)",
- .adblock_rules = "Filter-list rules by fate: loaded into the matcher, or skipped as unsupported",
+ .adblock_verdicts = "Adblocker decisions for evaluated requests, by verdict (none = not blocked, allowed = an exception overrode a block)",
+ .adblock_rules = "Filter-list rules by fate: loaded into the matcher, skipped as unsupported, or cosmetic (domain-scoped element hiding, outside the network realm)",
};
pub fn write(self: *const Metrics, writer: *std.Io.Writer) void {
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zig
index a06111cd7..551dd695e 100644
--- a/src/network/adblock/AdBlocker.zig
+++ b/src/network/adblock/AdBlocker.zig
@@ -34,8 +34,8 @@ const log = lp.log;
const AdBlocker = @This();
-pub const Request = Engine.Request;
-pub const ResourceTypes = NetworkFilter.ResourceTypes;
+const Request = Engine.Request;
+const ResourceTypes = NetworkFilter.ResourceTypes;
allocator: Allocator,
/// Owns everything the filters point at.
@@ -66,6 +66,11 @@ rules_loaded: usize,
/// Rules the lists carry and we do not apply, across those same lists: the
/// ones the parser could not read, plus the ones no matcher can express.
rules_skipped: usize,
+/// Domain-scoped element-hiding rules and cosmetic-realm exceptions, across
+/// those same lists. Not ours to apply, but not rules we failed at either, so
+/// they stay out of `rules_skipped`. Generic "##" rules read as comments and
+/// are counted nowhere.
+rules_cosmetic: usize,
/// Read buffer for filter lists, and therefore the longest line we can see.
/// Real-world rules are well under 1KB; the parser skips anything longer.
@@ -95,6 +100,7 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
.exceptions = .empty,
.rules_loaded = 0,
.rules_skipped = 0,
+ .rules_cosmetic = 0,
};
}
@@ -132,9 +138,11 @@ pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {
try blocker.build();
lp.metrics.adblock_rules.add(.loaded, @intCast(blocker.rules_loaded));
lp.metrics.adblock_rules.add(.skipped, @intCast(blocker.rules_skipped));
+ lp.metrics.adblock_rules.add(.cosmetic, @intCast(blocker.rules_cosmetic));
log.info(.app, "adblock lists loaded", .{
.loaded = blocker.rules_loaded,
.skipped = blocker.rules_skipped,
+ .cosmetic = blocker.rules_cosmetic,
.indexed = blocker.filters.items.len,
});
}
@@ -161,7 +169,10 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
var parser: Parser = .init(reader);
// Lines the parser dropped are rules we did not load just as much as the
// ones we drop below, and on a real list they are the bulk of them.
- defer self.rules_skipped += parser.skipped;
+ defer {
+ self.rules_skipped += parser.skipped;
+ self.rules_cosmetic += parser.cosmetic;
+ }
const arena = self.arena.allocator();
while (try parser.next(scratch)) |item| {
@@ -183,6 +194,13 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
.filter => |filter| filter,
};
+ // Cosmetic-realm exceptions ($generichide and friends) modify element
+ // hiding, which no network matcher applies: another realm's rule, not
+ // one we failed at.
+ if (filter.generichide or filter.specifichide or filter.elemhide) {
+ self.rules_cosmetic += 1;
+ continue;
+ }
if (!isSupported(&filter)) {
self.rules_skipped += 1;
continue;
@@ -310,14 +328,14 @@ pub const Verdict = enum { none, allowed, blocked };
pub fn isBlocked(self: *const AdBlocker, transfer: *const HttpClient.Transfer) bool {
var buffers: Request.Buffers = undefined;
const request = Request.fromHttp(transfer, &buffers) orelse return false;
- const verdict = self.match(request);
+ const verdict = self.match(&request);
lp.metrics.adblock_verdicts.incr(verdict);
return verdict == .blocked;
}
/// The verdict for one request.
/// `$important` blocks outrank exceptions, which outrank plain blocks; same as uBO.
-pub fn match(self: *const AdBlocker, request: Request) Verdict {
+pub fn match(self: *const AdBlocker, request: *const Request) Verdict {
std.debug.assert(self.built);
const hostname = request.url.hostname();
@@ -340,10 +358,7 @@ pub fn match(self: *const AdBlocker, request: Request) Verdict {
/// Whether we can evaluate this filter at all.
fn isSupported(filter: *const NetworkFilter) bool {
// No regex engine to run them with.
- if (filter.kind == .regex) return false;
- // Cosmetic-realm exceptions never say anything about network requests.
- if (filter.generichide or filter.specifichide or filter.elemhide) return false;
- return true;
+ return filter.kind != .regex;
}
/// Whether the filter's whole effect is "every request to this hostname and
@@ -434,7 +449,8 @@ fn expectVerdict(
// are lowercased too, so a rule spelled "/embed/C-iDzdvIg1Y" still lands.
var buf: [512]u8 = undefined;
const lowered = std.ascii.lowerString(buf[0..url.len], url);
- try testing.expectEqual(expected, blocker.match(.init(lowered, source, kind)));
+ const request: Request = .init(lowered, source, kind);
+ try testing.expectEqual(expected, blocker.match(&request));
}
const script: ResourceTypes = .{ .script = true };
@@ -548,10 +564,28 @@ test "adblock.AdBlocker: parse accumulates across lists" {
try expectVerdict(&blocker, .none, "https://tracker.net/x", "sports.news.com", script);
try expectVerdict(&blocker, .none, "https://tracker.net/x", "other.com", script);
- // Three rules loaded across the two lists; the cosmetic line is a rule
- // the list carries and we do not apply, so it counts as skipped.
+ // Three rules loaded across the two lists; the cosmetic line is a rule of
+ // another realm, counted as such rather than as one we failed to apply.
try testing.expectEqual(3, blocker.rules_loaded);
+ try testing.expectEqual(0, blocker.rules_skipped);
+ try testing.expectEqual(1, blocker.rules_cosmetic);
+}
+
+test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
+ var blocker: AdBlocker = try .init(testing.allocator);
+ defer blocker.deinit();
+
+ try testLoad(&blocker,
+ \\||ads.example.com^
+ \\@@||example.com^$generichide
+ \\@@||example.com^$elemhide
+ \\example.com##.ad-banner
+ \\/regex-we-cannot-run/
+ );
+
+ try testing.expectEqual(1, blocker.rules_loaded);
try testing.expectEqual(1, blocker.rules_skipped);
+ try testing.expectEqual(3, blocker.rules_cosmetic);
}
test "adblock.AdBlocker: verdict precedence" {
diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zig
index 37ef51cd2..01ef5fc43 100644
--- a/src/network/adblock/Engine.zig
+++ b/src/network/adblock/Engine.zig
@@ -176,7 +176,7 @@ pub const Histogram = std.AutoHashMapUnmanaged(u32, u32);
const TOKENS_MAX = 32;
/// The first filter that matches, or null.
-pub fn match(self: *const Engine, request: Request) ?*const NetworkFilter {
+pub fn match(self: *const Engine, request: *const Request) ?*const NetworkFilter {
if (self.filters.len != 0) {
for (request.tokens()) |token| {
const bucket = self.buckets.get(token) orelse continue;
@@ -186,7 +186,7 @@ pub fn match(self: *const Engine, request: Request) ?*const NetworkFilter {
return self.matchIn(self.fallback, request);
}
-fn matchIn(self: *const Engine, bucket: []const u32, request: Request) ?*const NetworkFilter {
+fn matchIn(self: *const Engine, bucket: []const u32, request: *const Request) ?*const NetworkFilter {
for (bucket) |index| {
const filter = &self.filters[index];
if (matchesFilter(filter, request)) return filter;
@@ -196,7 +196,7 @@ fn matchIn(self: *const Engine, bucket: []const u32, request: Request) ?*const N
/// Cheapest constraint first: the type and party bits are two comparisons,
/// the pattern walk is the expensive one.
-fn matchesFilter(filter: *const NetworkFilter, request: Request) bool {
+fn matchesFilter(filter: *const NetworkFilter, request: *const Request) bool {
if (filter.types.bits() & request.kind.bits() == 0) return false;
if (request.third_party) {
if (!filter.third_party) return false;
@@ -277,7 +277,7 @@ pub fn build(
}
/// Runs of alphanumerics. Everything else ('.', '/', '-', '%', '_', '?') is a boundary.
-pub const Tokens = struct {
+const Tokens = struct {
text: []const u8,
i: usize = 0,
diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zig
index 4cf9493f2..0cae510b6 100644
--- a/src/network/adblock/NetworkFilter.zig
+++ b/src/network/adblock/NetworkFilter.zig
@@ -119,6 +119,9 @@ pub const ParseError = error{
UnsupportedPattern,
NoSupportedDomains,
UnsupportedModifier,
+ // Element hiding ("example.com##.ad"): a rule for another realm, not a
+ // network rule we failed to read.
+ CosmeticFilter,
// Hosts-file noise ("127.0.0.1 localhost"), skip silently.
Ignored,
} || std.mem.Allocator.Error;
@@ -258,6 +261,10 @@ pub fn parse(arena: std.mem.Allocator, line: []const u8) ParseError!NetworkFilte
if (rest.len == 0) return error.InvalidPattern;
const split = splitOptions(rest);
+ // Cosmetic filters classify as network.
+ if (!isRegexLiteral(split.pattern) and hasCosmeticSeparator(split.pattern)) {
+ return error.CosmeticFilter;
+ }
var explicit_types = false;
if (split.options) |options| {
try filter.parseOptions(arena, options, &explicit_types);
@@ -287,6 +294,23 @@ pub fn parse(arena: std.mem.Allocator, line: []const u8) ParseError!NetworkFilte
return filter;
}
+inline fn isRegexLiteral(text: []const u8) bool {
+ return text.len > 2 and text[0] == '/' and text[text.len - 1] == '/';
+}
+
+/// Whether `text` carries a cosmetic separator.
+/// "##", "#@#", "#?#", "#$#", "#%#" and their combinations, like "#@$?#".
+fn hasCosmeticSeparator(text: []const u8) bool {
+ var start: usize = 0;
+ while (std.mem.indexOfScalarPos(u8, text, start, '#')) |pos| {
+ var i = pos + 1;
+ while (i < text.len and std.mem.indexOfScalar(u8, "@$?%", text[i]) != null) i += 1;
+ if (i < text.len and text[i] == '#') return true;
+ start = pos + 1;
+ }
+ return false;
+}
+
/// uBO allows a trailing " # comment" on lines containing whitespace.
fn stripInlineComment(line: []const u8) []const u8 {
var start: usize = 1;
@@ -561,16 +585,12 @@ fn parsePattern(
}
// Whole-pattern regex literal: /.../ with anything between the slashes.
- if (raw.len > 2 and raw[0] == '/' and raw[raw.len - 1] == '/') {
+ if (isRegexLiteral(raw)) {
self.kind = .regex;
self.pattern = raw;
return;
}
- // A literal '#' can never match: request URLs have their fragment
- // stripped before matching. This is also what drops cosmetic filter
- // lines (example.com##.ad-banner), which classify as network now that
- // there is no cosmetic-separator scan.
if (std.mem.indexOfScalar(u8, raw, '#') != null) return error.UnsupportedPattern;
var pattern = raw;
@@ -1070,10 +1090,15 @@ test "adblock.NetworkFilter: option-only and invalid patterns" {
try testing.expectError(error.InvalidPattern, testParse(arena, "foo bar$script"));
try testing.expectError(error.UnsupportedPattern, testParse(arena, "||exämple.com^"));
- // Cosmetic lines reach this parser undetected (no separator scan) and
- // drop here: a '#' never matches a fragment-stripped request URL.
- try testing.expectError(error.UnsupportedPattern, testParse(arena, "example.com##.ad-banner"));
- try testing.expectError(error.UnsupportedPattern, testParse(arena, "example.com#@#.sponsored"));
+ // Cosmetic lines reach this parser as network lines, there being no
+ // separator scan up front, and are told apart by their separator,
+ // whichever flavour...
+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com##.ad-banner"));
+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#@#.sponsored"));
+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#?#.ad:has(> a)"));
+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#@$#body { background: none }"));
+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com##+js(no-fetch-if, ads)"));
+ // ...while a lone '#' is a fragment, which never matches a request URL.
try testing.expectError(error.UnsupportedPattern, testParse(arena, "|https://a.com/x#y|"));
// ... but '#' inside a /regex/ body is untouched.
const regex = try testParse(arena, "/ads#[0-9]+/");
diff --git a/src/network/adblock/Parser.zig b/src/network/adblock/Parser.zig
index 29c56c3fd..8f0024d4a 100644
--- a/src/network/adblock/Parser.zig
+++ b/src/network/adblock/Parser.zig
@@ -34,6 +34,7 @@ first_line: bool = true,
/// not rules, so they never count. The caller adds this to whatever it drops
/// afterwards, so that "skipped" means every rule the list has and we do not.
skipped: usize = 0,
+cosmetic: usize = 0,
pub const Error = error{ OutOfMemory, ReadFailed };
@@ -42,7 +43,7 @@ pub const Error = error{ OutOfMemory, ReadFailed };
/// on that hostname is not meant to be blocked.
pub const Item = union(enum) {
filter: NetworkFilter,
- /// Already counted in `skipped`; the caller only inspects it.
+ /// Already counted in `skipped` or `cosmetic`; the caller only inspects it.
dropped: Dropped,
};
@@ -82,6 +83,10 @@ pub fn next(self: *Parser, arena: Allocator) Error!?Item {
return .{ .filter = filter };
} else |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
+ error.CosmeticFilter => {
+ self.cosmetic += 1;
+ return .{ .dropped = .{ .line = line, .reason = err } };
+ },
// Anything else is a line outside the supported subset
// or malformed; either way it is not ours to enforce.
else => {
@@ -167,8 +172,8 @@ test "adblock.Parser: line classification" {
try testing.expectEqual(.network, LineClass.fromLine("||ads.example.com^"));
try testing.expectEqual(.network, LineClass.fromLine("@@|https://example.com/path#frag|"));
try testing.expectEqual(.network, LineClass.fromLine("0.0.0.0 tracker.com"));
- // Domain-prefixed cosmetic lines classify as network; the filter
- // parser drops them via their '#' (see NetworkFilter tests).
+ // Domain-prefixed cosmetic lines classify as network; the filter parser
+ // tells them apart by their separator (see NetworkFilter tests).
try testing.expectEqual(.network, LineClass.fromLine("example.com#@#.ad"));
try testing.expectEqual(.network, LineClass.fromLine("example.com#?#.ad:has-text(x)"));
try testing.expectEqual(.network, LineClass.fromLine("example.com#$#body { padding: 0 }"));
@@ -206,7 +211,7 @@ test "adblock.Parser: yields one item per next() call" {
// with the reason the filter parser gave.
const second = (try parser.next(arena)).?;
try testing.expectString("example.com##.ad-banner", second.dropped.line);
- try testing.expectEqual(error.UnsupportedPattern, second.dropped.reason.?);
+ try testing.expectEqual(error.CosmeticFilter, second.dropped.reason.?);
const third = (try parser.next(arena)).?;
try testing.expectString("tracker.net", third.filter.hostname);
@@ -238,8 +243,10 @@ test "adblock.Parser: only rule-shaped lines count as skipped" {
// The header, the comment and the blank line are not rules.
try testing.expectEqual(1, count);
- // The cosmetic filter, the unknown option and the AdGuard line are.
- try testing.expectEqual(3, parser.skipped);
+ // The unknown option and the AdGuard line are rules we do not apply; the
+ // cosmetic filter is a rule for another realm, counted as such.
+ try testing.expectEqual(2, parser.skipped);
+ try testing.expectEqual(1, parser.cosmetic);
}
test "adblock.Parser: a line too long for the buffer is skipped" {
@@ -319,7 +326,11 @@ test "adblock.Parser: full list" {
// 5 direct network filters + both branches of the !#if block: the
// directives are comments, their contents parse unconditionally.
- // Everything else in the list — cosmetic lines, the scriptlet, the
- // hosts-file noise, $removeparam and the unknown option — is skipped.
+ // Everything else in the list is either cosmetic (the domain-prefixed
+ // element-hiding lines and the scriptlet; the generic "##.ad-banner" is
+ // a comment) or skipped (the hosts-file noise, $removeparam and the
+ // unknown option).
try testing.expectEqual(7, count);
+ try testing.expectEqual(3, parser.cosmetic);
+ try testing.expectEqual(3, parser.skipped);
}