Lightpanda: the headless browser designed for AI and automation
first 128 tokens cached at init (common), rest streamed by each engine
Halil Durak860c4950e2f6
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zigindex 551dd695e..44ebcb265 100644--- a/src/network/adblock/AdBlocker.zig+++ b/src/network/adblock/AdBlocker.zig@@ -323,7 +323,14 @@ fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {};}-pub const Verdict = enum { none, allowed, blocked };+pub const Verdict = enum {+ /// Nothing blocks the request, whether or not an exception names it.+ none,+ /// An exception overrode a block.+ allowed,+ /// A block no exception overrides, or a `$important` one.+ blocked,+};pub fn isBlocked(self: *const AdBlocker, transfer: *const HttpClient.Transfer) bool {var buffers: Request.Buffers = undefined;@@ -346,13 +353,12 @@ pub fn match(self: *const AdBlocker, request: *const Request) Verdict {if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;if (self.blocking_important.match(request) != null) return .blocked;- if (self.exceptions.match(request) != null) return .allowed;+ const blocked = self.trie.matches(self.blocked, hostname) != null or+ self.blocking.match(request) != null;+ if (!blocked) return .none;if (self.trie.matches(self.allowed, hostname) != null) return .allowed;-- if (self.trie.matches(self.blocked, hostname) != null) return .blocked;- if (self.blocking.match(request) != null) return .blocked;-- return .none;+ if (self.exceptions.match(request) != null) return .allowed;+ return .blocked;}/// Whether we can evaluate this filter at all.@@ -488,7 +494,9 @@ test "adblock.AdBlocker: the doubleclick rules from EasyList" {// Right everything, wrong path.try expectVerdict(&blocker, .blocked, "https://g.doubleclick.net/pagead/ads", "bloomberg.com", xhr);- try expectVerdict(&blocker, .allowed, "https://static.doubleclick.net/instream/ad_status.js", "ignboards.com", script);+ // An exception with nothing to override decides nothing: `$popup` never+ // applies to a script, so no block covers this request.+ try expectVerdict(&blocker, .none, "https://static.doubleclick.net/instream/ad_status.js", "ignboards.com", script);// The unconditional exception carves one path out of the block covering// the whole subtree, and leaves the rest of it blocked.try expectVerdict(&blocker, .allowed, "https://pubads.g.doubleclick.net/ssai/x", "history.com", xhr);@@ -542,6 +550,7 @@ test "adblock.AdBlocker: parse accumulates across lists" {var first: Io.Reader = .fixed(\\! Title: First List\\||ads.example.com^+ \\||cdn.example.com^\\@@||cdn.example.com^$script\\example.com##.ad-banner);@@ -555,22 +564,41 @@ test "adblock.AdBlocker: parse accumulates across lists" {try expectVerdict(&blocker, .blocked, "https://ads.example.com/x", "news.com", script);try expectVerdict(&blocker, .blocked, "https://sub.ads.example.com/x", "news.com", script);- // The type-restricted exception now applies to scripts only.+ // The type-restricted exception carves scripts out of the block.try expectVerdict(&blocker, .allowed, "https://cdn.example.com/x.js", "news.com", script);- try expectVerdict(&blocker, .none, "https://cdn.example.com/x.png", "news.com", image);+ try expectVerdict(&blocker, .blocked, "https://cdn.example.com/x.png", "news.com", image);// The party and $domain= constrained block, from the second list.try expectVerdict(&blocker, .blocked, "https://tracker.net/x", "news.com", script);try expectVerdict(&blocker, .none, "https://tracker.net/x", "sports.news.com", script);try expectVerdict(&blocker, .none, "https://tracker.net/x", "other.com", script);- // Three rules loaded across the two lists; the cosmetic line is a rule of+ // Four rules loaded across the two lists; the cosmetic line is a rule of// another realm, counted as such rather than as one we failed to apply.- try testing.expectEqual(3, blocker.rules_loaded);+ try testing.expectEqual(4, blocker.rules_loaded);try testing.expectEqual(0, blocker.rules_skipped);try testing.expectEqual(1, blocker.rules_cosmetic);}+test "adblock.AdBlocker: tokens past the request buffer still match" {+ var blocker: AdBlocker = try .init(testing.allocator);+ defer blocker.deinit();++ try testLoad(&blocker,+ \\&utm_tracker=+ );++ // 140 tokens of query noise push the filter's token ("utm", its rarest)+ // past what the request holds; the engine walks the rest of the URL.+ const noise = "https://example.com/?" ++ "a=1&" ** 70;+ const overflowing: Request = .init(noise ++ "utm_tracker=1", "a.com", script);+ try testing.expect(overflowing.tail.len != 0);+ try testing.expectEqual(.blocked, blocker.match(&overflowing));+ // A token past the buffer finds its bucket, but the pattern does not fit.+ try expectVerdict(&blocker, .none, noise ++ "utm_other=1", "a.com", script);+ try expectVerdict(&blocker, .none, noise ++ "b=2", "a.com", script);+}+test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {var blocker: AdBlocker = try .init(testing.allocator);defer blocker.deinit();diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zigindex 01ef5fc43..4b8184511 100644--- a/src/network/adblock/Engine.zig+++ b/src/network/adblock/Engine.zig@@ -72,10 +72,13 @@ pub const Request = struct {/// Exactly one bit set.kind: NetworkFilter.ResourceTypes,third_party: bool,- /// The URL's tokens, hashed once here so no engine retokenizes. A URL- /// long enough to overflow loses its last tokens.+ /// The URL's first tokens, hashed once here so no engine retokenizes.tokens_buf: [URL_TOKENS_MAX]u32,tokens_len: usize,+ /// The rest of the URL, past the last token that fit.+ /// Each engine tokenizes it itself, so no token is ever lost. Token-free for+ /// all but the longest URLs.+ tail: []const u8,const URL_TOKENS_MAX = 128;@@ -108,13 +111,15 @@ pub const Request = struct {.third_party = domain.isThirdParty(parsed.hostname(), source),.tokens_buf = undefined,.tokens_len = 0,+ .tail = "",};var it: Tokens = .{ .text = url };- while (it.next()) |token| {- if (request.tokens_len == request.tokens_buf.len) break;+ while (request.tokens_len < request.tokens_buf.len) {+ const token = it.next() orelse break;request.tokens_buf[request.tokens_len] = token;request.tokens_len += 1;}+ request.tail = url[it.i..];return request;}@@ -177,15 +182,24 @@ const TOKENS_MAX = 32;/// The first filter that matches, or null.pub fn match(self: *const Engine, request: *const Request) ?*const NetworkFilter {- if (self.filters.len != 0) {+ if (self.buckets.count() != 0) {for (request.tokens()) |token| {- const bucket = self.buckets.get(token) orelse continue;- if (self.matchIn(bucket, request)) |filter| return filter;+ if (self.matchToken(token, request)) |filter| return filter;+ }+ // Whatever the request could not hold is hashed again.+ var it: Tokens = .{ .text = request.tail };+ while (it.next()) |token| {+ if (self.matchToken(token, request)) |filter| return filter;}}return self.matchIn(self.fallback, request);}+fn matchToken(self: *const Engine, token: u32, request: *const Request) ?*const NetworkFilter {+ const bucket = self.buckets.get(token) orelse return null;+ return self.matchIn(bucket, request);+}+fn matchIn(self: *const Engine, bucket: []const u32, request: *const Request) ?*const NetworkFilter {for (bucket) |index| {const filter = &self.filters[index];@@ -355,6 +369,30 @@ fn contains(tokens: []const u32, token: []const u8) bool {return false;}+test "adblock.Engine: a request keeps its first tokens, the rest as tail" {+ const max = Request.URL_TOKENS_MAX;+ const kind: NetworkFilter.ResourceTypes = .{ .script = true };++ // Exactly as many tokens as the buffer holds: nothing is left to walk...+ const full = "x/" ** (max - 1) ++ "x";+ var request: Request = .init(full, "", kind);+ try testing.expectEqual(max, request.tokens_len);+ try testing.expectEqual(0, request.tail.len);++ // ...one more, and only that one is in the tail.+ request = .init(full ++ "/y", "", kind);+ try testing.expectEqual(max, request.tokens_len);+ try testing.expectString("/y", request.tail);+ var it: Tokens = .{ .text = request.tail };+ try testing.expectEqual(hash("y"), it.next().?);+ try testing.expect(it.next() == null);++ // Fewer than the buffer holds: the tail is empty.+ request = .init("https://example.com/a", "", kind);+ try testing.expectEqual(4, request.tokens_len);+ try testing.expectEqual(0, request.tail.len);+}+test "adblock.Engine: only tokens the URL must reproduce are collected" {var arena_state = std.heap.ArenaAllocator.init(testing.allocator);defer arena_state.deinit();