Lightpanda: the headless browser designed for AI and automation
pass `*const Request` where possible
Halil Durakfb58901604ee
diff --git a/src/Metrics.zig b/src/Metrics.zigindex 120299cec..66c7c6d69 100644--- a/src/Metrics.zig+++ b/src/Metrics.zig@@ -95,7 +95,7 @@ cors_check: CounterEnum("result", enum { same_origin, no_cors, simple, preflightcors_preflight: CounterEnum("result", enum { allowed, blocked }) = .{},cors_response: CounterEnum("result", enum { allowed, blocked }) = .{},adblock_verdicts: CounterEnum("verdict", @import("network/adblock/AdBlocker.zig").Verdict) = .{},-adblock_rules: GaugeEnum("state", enum { loaded, skipped }) = .{},+adblock_rules: GaugeEnum("state", enum { loaded, skipped, cosmetic }) = .{},// Emitted as each metric's "# HELP" line. A field without an entry is a// compile error.@@ -131,8 +131,8 @@ const help = .{.cors_check = "CORS initial classification: same_origin/no_cors need no CORS handling, simple needs response validation only, preflight needs an OPTIONS round-trip first",.cors_preflight = "CORS preflight (OPTIONS) results, one per request that required one",.cors_response = "CORS actual-response validation results",- .adblock_verdicts = "Adblocker decisions for evaluated requests, by verdict (none = no filter matched)",- .adblock_rules = "Filter-list rules by fate: loaded into the matcher, or skipped as unsupported",+ .adblock_verdicts = "Adblocker decisions for evaluated requests, by verdict (none = not blocked, allowed = an exception overrode a block)",+ .adblock_rules = "Filter-list rules by fate: loaded into the matcher, skipped as unsupported, or cosmetic (domain-scoped element hiding, outside the network realm)",};pub fn write(self: *const Metrics, writer: *std.Io.Writer) void {diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zigindex a06111cd7..551dd695e 100644--- a/src/network/adblock/AdBlocker.zig+++ b/src/network/adblock/AdBlocker.zig@@ -34,8 +34,8 @@ const log = lp.log;const AdBlocker = @This();-pub const Request = Engine.Request;-pub const ResourceTypes = NetworkFilter.ResourceTypes;+const Request = Engine.Request;+const ResourceTypes = NetworkFilter.ResourceTypes;allocator: Allocator,/// Owns everything the filters point at.@@ -66,6 +66,11 @@ rules_loaded: usize,/// Rules the lists carry and we do not apply, across those same lists: the/// ones the parser could not read, plus the ones no matcher can express.rules_skipped: usize,+/// Domain-scoped element-hiding rules and cosmetic-realm exceptions, across+/// those same lists. Not ours to apply, but not rules we failed at either, so+/// they stay out of `rules_skipped`. Generic "##" rules read as comments and+/// are counted nowhere.+rules_cosmetic: usize,/// Read buffer for filter lists, and therefore the longest line we can see./// Real-world rules are well under 1KB; the parser skips anything longer.@@ -95,6 +100,7 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {.exceptions = .empty,.rules_loaded = 0,.rules_skipped = 0,+ .rules_cosmetic = 0,};}@@ -132,9 +138,11 @@ pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {try blocker.build();lp.metrics.adblock_rules.add(.loaded, @intCast(blocker.rules_loaded));lp.metrics.adblock_rules.add(.skipped, @intCast(blocker.rules_skipped));+ lp.metrics.adblock_rules.add(.cosmetic, @intCast(blocker.rules_cosmetic));log.info(.app, "adblock lists loaded", .{.loaded = blocker.rules_loaded,.skipped = blocker.rules_skipped,+ .cosmetic = blocker.rules_cosmetic,.indexed = blocker.filters.items.len,});}@@ -161,7 +169,10 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {var parser: Parser = .init(reader);// Lines the parser dropped are rules we did not load just as much as the// ones we drop below, and on a real list they are the bulk of them.- defer self.rules_skipped += parser.skipped;+ defer {+ self.rules_skipped += parser.skipped;+ self.rules_cosmetic += parser.cosmetic;+ }const arena = self.arena.allocator();while (try parser.next(scratch)) |item| {@@ -183,6 +194,13 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {.filter => |filter| filter,};+ // Cosmetic-realm exceptions ($generichide and friends) modify element+ // hiding, which no network matcher applies: another realm's rule, not+ // one we failed at.+ if (filter.generichide or filter.specifichide or filter.elemhide) {+ self.rules_cosmetic += 1;+ continue;+ }if (!isSupported(&filter)) {self.rules_skipped += 1;continue;@@ -310,14 +328,14 @@ pub const Verdict = enum { none, allowed, blocked };pub fn isBlocked(self: *const AdBlocker, transfer: *const HttpClient.Transfer) bool {var buffers: Request.Buffers = undefined;const request = Request.fromHttp(transfer, &buffers) orelse return false;- const verdict = self.match(request);+ const verdict = self.match(&request);lp.metrics.adblock_verdicts.incr(verdict);return verdict == .blocked;}/// The verdict for one request./// `$important` blocks outrank exceptions, which outrank plain blocks; same as uBO.-pub fn match(self: *const AdBlocker, request: Request) Verdict {+pub fn match(self: *const AdBlocker, request: *const Request) Verdict {std.debug.assert(self.built);const hostname = request.url.hostname();@@ -340,10 +358,7 @@ pub fn match(self: *const AdBlocker, request: Request) Verdict {/// Whether we can evaluate this filter at all.fn isSupported(filter: *const NetworkFilter) bool {// No regex engine to run them with.- if (filter.kind == .regex) return false;- // Cosmetic-realm exceptions never say anything about network requests.- if (filter.generichide or filter.specifichide or filter.elemhide) return false;- return true;+ return filter.kind != .regex;}/// Whether the filter's whole effect is "every request to this hostname and@@ -434,7 +449,8 @@ fn expectVerdict(// are lowercased too, so a rule spelled "/embed/C-iDzdvIg1Y" still lands.var buf: [512]u8 = undefined;const lowered = std.ascii.lowerString(buf[0..url.len], url);- try testing.expectEqual(expected, blocker.match(.init(lowered, source, kind)));+ const request: Request = .init(lowered, source, kind);+ try testing.expectEqual(expected, blocker.match(&request));}const script: ResourceTypes = .{ .script = true };@@ -548,10 +564,28 @@ test "adblock.AdBlocker: parse accumulates across lists" {try expectVerdict(&blocker, .none, "https://tracker.net/x", "sports.news.com", script);try expectVerdict(&blocker, .none, "https://tracker.net/x", "other.com", script);- // Three rules loaded across the two lists; the cosmetic line is a rule- // the list carries and we do not apply, so it counts as skipped.+ // Three rules loaded across the two lists; the cosmetic line is a rule of+ // another realm, counted as such rather than as one we failed to apply.try testing.expectEqual(3, blocker.rules_loaded);+ try testing.expectEqual(0, blocker.rules_skipped);+ try testing.expectEqual(1, blocker.rules_cosmetic);+}++test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {+ var blocker: AdBlocker = try .init(testing.allocator);+ defer blocker.deinit();++ try testLoad(&blocker,+ \\||ads.example.com^+ \\@@||example.com^$generichide+ \\@@||example.com^$elemhide+ \\example.com##.ad-banner+ \\/regex-we-cannot-run/+ );++ try testing.expectEqual(1, blocker.rules_loaded);try testing.expectEqual(1, blocker.rules_skipped);+ try testing.expectEqual(3, blocker.rules_cosmetic);}test "adblock.AdBlocker: verdict precedence" {diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zigindex 37ef51cd2..01ef5fc43 100644--- a/src/network/adblock/Engine.zig+++ b/src/network/adblock/Engine.zig@@ -176,7 +176,7 @@ pub const Histogram = std.AutoHashMapUnmanaged(u32, u32);const TOKENS_MAX = 32;/// The first filter that matches, or null.-pub fn match(self: *const Engine, request: Request) ?*const NetworkFilter {+pub fn match(self: *const Engine, request: *const Request) ?*const NetworkFilter {if (self.filters.len != 0) {for (request.tokens()) |token| {const bucket = self.buckets.get(token) orelse continue;@@ -186,7 +186,7 @@ pub fn match(self: *const Engine, request: Request) ?*const NetworkFilter {return self.matchIn(self.fallback, request);}-fn matchIn(self: *const Engine, bucket: []const u32, request: Request) ?*const NetworkFilter {+fn matchIn(self: *const Engine, bucket: []const u32, request: *const Request) ?*const NetworkFilter {for (bucket) |index| {const filter = &self.filters[index];if (matchesFilter(filter, request)) return filter;@@ -196,7 +196,7 @@ fn matchIn(self: *const Engine, bucket: []const u32, request: Request) ?*const N/// Cheapest constraint first: the type and party bits are two comparisons,/// the pattern walk is the expensive one.-fn matchesFilter(filter: *const NetworkFilter, request: Request) bool {+fn matchesFilter(filter: *const NetworkFilter, request: *const Request) bool {if (filter.types.bits() & request.kind.bits() == 0) return false;if (request.third_party) {if (!filter.third_party) return false;@@ -277,7 +277,7 @@ pub fn build(}/// Runs of alphanumerics. Everything else ('.', '/', '-', '%', '_', '?') is a boundary.-pub const Tokens = struct {+const Tokens = struct {text: []const u8,i: usize = 0,diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zigindex 4cf9493f2..0cae510b6 100644--- a/src/network/adblock/NetworkFilter.zig+++ b/src/network/adblock/NetworkFilter.zig@@ -119,6 +119,9 @@ pub const ParseError = error{UnsupportedPattern,NoSupportedDomains,UnsupportedModifier,+ // Element hiding ("example.com##.ad"): a rule for another realm, not a+ // network rule we failed to read.+ CosmeticFilter,// Hosts-file noise ("127.0.0.1 localhost"), skip silently.Ignored,} || std.mem.Allocator.Error;@@ -258,6 +261,10 @@ pub fn parse(arena: std.mem.Allocator, line: []const u8) ParseError!NetworkFilteif (rest.len == 0) return error.InvalidPattern;const split = splitOptions(rest);+ // Cosmetic filters classify as network.+ if (!isRegexLiteral(split.pattern) and hasCosmeticSeparator(split.pattern)) {+ return error.CosmeticFilter;+ }var explicit_types = false;if (split.options) |options| {try filter.parseOptions(arena, options, &explicit_types);@@ -287,6 +294,23 @@ pub fn parse(arena: std.mem.Allocator, line: []const u8) ParseError!NetworkFiltereturn filter;}+inline fn isRegexLiteral(text: []const u8) bool {+ return text.len > 2 and text[0] == '/' and text[text.len - 1] == '/';+}++/// Whether `text` carries a cosmetic separator.+/// "##", "#@#", "#?#", "#$#", "#%#" and their combinations, like "#@$?#".+fn hasCosmeticSeparator(text: []const u8) bool {+ var start: usize = 0;+ while (std.mem.indexOfScalarPos(u8, text, start, '#')) |pos| {+ var i = pos + 1;+ while (i < text.len and std.mem.indexOfScalar(u8, "@$?%", text[i]) != null) i += 1;+ if (i < text.len and text[i] == '#') return true;+ start = pos + 1;+ }+ return false;+}+/// uBO allows a trailing " # comment" on lines containing whitespace.fn stripInlineComment(line: []const u8) []const u8 {var start: usize = 1;@@ -561,16 +585,12 @@ fn parsePattern(}// Whole-pattern regex literal: /.../ with anything between the slashes.- if (raw.len > 2 and raw[0] == '/' and raw[raw.len - 1] == '/') {+ if (isRegexLiteral(raw)) {self.kind = .regex;self.pattern = raw;return;}- // A literal '#' can never match: request URLs have their fragment- // stripped before matching. This is also what drops cosmetic filter- // lines (example.com##.ad-banner), which classify as network now that- // there is no cosmetic-separator scan.if (std.mem.indexOfScalar(u8, raw, '#') != null) return error.UnsupportedPattern;var pattern = raw;@@ -1070,10 +1090,15 @@ test "adblock.NetworkFilter: option-only and invalid patterns" {try testing.expectError(error.InvalidPattern, testParse(arena, "foo bar$script"));try testing.expectError(error.UnsupportedPattern, testParse(arena, "||exämple.com^"));- // Cosmetic lines reach this parser undetected (no separator scan) and- // drop here: a '#' never matches a fragment-stripped request URL.- try testing.expectError(error.UnsupportedPattern, testParse(arena, "example.com##.ad-banner"));- try testing.expectError(error.UnsupportedPattern, testParse(arena, "example.com#@#.sponsored"));+ // Cosmetic lines reach this parser as network lines, there being no+ // separator scan up front, and are told apart by their separator,+ // whichever flavour...+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com##.ad-banner"));+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#@#.sponsored"));+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#?#.ad:has(> a)"));+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com#@$#body { background: none }"));+ try testing.expectError(error.CosmeticFilter, testParse(arena, "example.com##+js(no-fetch-if, ads)"));+ // ...while a lone '#' is a fragment, which never matches a request URL.try testing.expectError(error.UnsupportedPattern, testParse(arena, "|https://a.com/x#y|"));// ... but '#' inside a /regex/ body is untouched.const regex = try testParse(arena, "/ads#[0-9]+/");diff --git a/src/network/adblock/Parser.zig b/src/network/adblock/Parser.zigindex 29c56c3fd..8f0024d4a 100644--- a/src/network/adblock/Parser.zig+++ b/src/network/adblock/Parser.zig@@ -34,6 +34,7 @@ first_line: bool = true,/// not rules, so they never count. The caller adds this to whatever it drops/// afterwards, so that "skipped" means every rule the list has and we do not.skipped: usize = 0,+cosmetic: usize = 0,pub const Error = error{ OutOfMemory, ReadFailed };@@ -42,7 +43,7 @@ pub const Error = error{ OutOfMemory, ReadFailed };/// on that hostname is not meant to be blocked.pub const Item = union(enum) {filter: NetworkFilter,- /// Already counted in `skipped`; the caller only inspects it.+ /// Already counted in `skipped` or `cosmetic`; the caller only inspects it.dropped: Dropped,};@@ -82,6 +83,10 @@ pub fn next(self: *Parser, arena: Allocator) Error!?Item {return .{ .filter = filter };} else |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,+ error.CosmeticFilter => {+ self.cosmetic += 1;+ return .{ .dropped = .{ .line = line, .reason = err } };+ },// Anything else is a line outside the supported subset// or malformed; either way it is not ours to enforce.else => {@@ -167,8 +172,8 @@ test "adblock.Parser: line classification" {try testing.expectEqual(.network, LineClass.fromLine("||ads.example.com^"));try testing.expectEqual(.network, LineClass.fromLine("@@|https://example.com/path#frag|"));try testing.expectEqual(.network, LineClass.fromLine("0.0.0.0 tracker.com"));- // Domain-prefixed cosmetic lines classify as network; the filter- // parser drops them via their '#' (see NetworkFilter tests).+ // Domain-prefixed cosmetic lines classify as network; the filter parser+ // tells them apart by their separator (see NetworkFilter tests).try testing.expectEqual(.network, LineClass.fromLine("example.com#@#.ad"));try testing.expectEqual(.network, LineClass.fromLine("example.com#?#.ad:has-text(x)"));try testing.expectEqual(.network, LineClass.fromLine("example.com#$#body { padding: 0 }"));@@ -206,7 +211,7 @@ test "adblock.Parser: yields one item per next() call" {// with the reason the filter parser gave.const second = (try parser.next(arena)).?;try testing.expectString("example.com##.ad-banner", second.dropped.line);- try testing.expectEqual(error.UnsupportedPattern, second.dropped.reason.?);+ try testing.expectEqual(error.CosmeticFilter, second.dropped.reason.?);const third = (try parser.next(arena)).?;try testing.expectString("tracker.net", third.filter.hostname);@@ -238,8 +243,10 @@ test "adblock.Parser: only rule-shaped lines count as skipped" {// The header, the comment and the blank line are not rules.try testing.expectEqual(1, count);- // The cosmetic filter, the unknown option and the AdGuard line are.- try testing.expectEqual(3, parser.skipped);+ // The unknown option and the AdGuard line are rules we do not apply; the+ // cosmetic filter is a rule for another realm, counted as such.+ try testing.expectEqual(2, parser.skipped);+ try testing.expectEqual(1, parser.cosmetic);}test "adblock.Parser: a line too long for the buffer is skipped" {@@ -319,7 +326,11 @@ test "adblock.Parser: full list" {// 5 direct network filters + both branches of the !#if block: the// directives are comments, their contents parse unconditionally.- // Everything else in the list — cosmetic lines, the scriptlet, the- // hosts-file noise, $removeparam and the unknown option — is skipped.+ // Everything else in the list is either cosmetic (the domain-prefixed+ // element-hiding lines and the scriptlet; the generic "##.ad-banner" is+ // a comment) or skipped (the hosts-file noise, $removeparam and the+ // unknown option).try testing.expectEqual(7, count);+ try testing.expectEqual(3, parser.cosmetic);+ try testing.expectEqual(3, parser.skipped);}