Skip to content

Lightpanda: the headless browser designed for AI and automation

first 128 tokens cached at init (common), rest streamed by each engine

Halil Durak860c4950e2f6
diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zig
index 551dd695e..44ebcb265 100644
--- a/src/network/adblock/AdBlocker.zig
+++ b/src/network/adblock/AdBlocker.zig
@@ -323,7 +323,14 @@ fn addToTrie(self: *AdBlocker, root: u32, hostname: []const u8) !void {
};
}
-pub const Verdict = enum { none, allowed, blocked };
+pub const Verdict = enum {
+ /// Nothing blocks the request, whether or not an exception names it.
+ none,
+ /// An exception overrode a block.
+ allowed,
+ /// A block no exception overrides, or a `$important` one.
+ blocked,
+};
pub fn isBlocked(self: *const AdBlocker, transfer: *const HttpClient.Transfer) bool {
var buffers: Request.Buffers = undefined;
@@ -346,13 +353,12 @@ pub fn match(self: *const AdBlocker, request: *const Request) Verdict {
if (self.trie.matches(self.blocked_important, hostname) != null) return .blocked;
if (self.blocking_important.match(request) != null) return .blocked;
- if (self.exceptions.match(request) != null) return .allowed;
+ const blocked = self.trie.matches(self.blocked, hostname) != null or
+ self.blocking.match(request) != null;
+ if (!blocked) return .none;
if (self.trie.matches(self.allowed, hostname) != null) return .allowed;
-
- if (self.trie.matches(self.blocked, hostname) != null) return .blocked;
- if (self.blocking.match(request) != null) return .blocked;
-
- return .none;
+ if (self.exceptions.match(request) != null) return .allowed;
+ return .blocked;
}
/// Whether we can evaluate this filter at all.
@@ -488,7 +494,9 @@ test "adblock.AdBlocker: the doubleclick rules from EasyList" {
// Right everything, wrong path.
try expectVerdict(&blocker, .blocked, "https://g.doubleclick.net/pagead/ads", "bloomberg.com", xhr);
- try expectVerdict(&blocker, .allowed, "https://static.doubleclick.net/instream/ad_status.js", "ignboards.com", script);
+ // An exception with nothing to override decides nothing: `$popup` never
+ // applies to a script, so no block covers this request.
+ try expectVerdict(&blocker, .none, "https://static.doubleclick.net/instream/ad_status.js", "ignboards.com", script);
// The unconditional exception carves one path out of the block covering
// the whole subtree, and leaves the rest of it blocked.
try expectVerdict(&blocker, .allowed, "https://pubads.g.doubleclick.net/ssai/x", "history.com", xhr);
@@ -542,6 +550,7 @@ test "adblock.AdBlocker: parse accumulates across lists" {
var first: Io.Reader = .fixed(
\\! Title: First List
\\||ads.example.com^
+ \\||cdn.example.com^
\\@@||cdn.example.com^$script
\\example.com##.ad-banner
);
@@ -555,22 +564,41 @@ test "adblock.AdBlocker: parse accumulates across lists" {
try expectVerdict(&blocker, .blocked, "https://ads.example.com/x", "news.com", script);
try expectVerdict(&blocker, .blocked, "https://sub.ads.example.com/x", "news.com", script);
- // The type-restricted exception now applies to scripts only.
+ // The type-restricted exception carves scripts out of the block.
try expectVerdict(&blocker, .allowed, "https://cdn.example.com/x.js", "news.com", script);
- try expectVerdict(&blocker, .none, "https://cdn.example.com/x.png", "news.com", image);
+ try expectVerdict(&blocker, .blocked, "https://cdn.example.com/x.png", "news.com", image);
// The party and $domain= constrained block, from the second list.
try expectVerdict(&blocker, .blocked, "https://tracker.net/x", "news.com", script);
try expectVerdict(&blocker, .none, "https://tracker.net/x", "sports.news.com", script);
try expectVerdict(&blocker, .none, "https://tracker.net/x", "other.com", script);
- // Three rules loaded across the two lists; the cosmetic line is a rule of
+ // Four rules loaded across the two lists; the cosmetic line is a rule of
// another realm, counted as such rather than as one we failed to apply.
- try testing.expectEqual(3, blocker.rules_loaded);
+ try testing.expectEqual(4, blocker.rules_loaded);
try testing.expectEqual(0, blocker.rules_skipped);
try testing.expectEqual(1, blocker.rules_cosmetic);
}
+test "adblock.AdBlocker: tokens past the request buffer still match" {
+ var blocker: AdBlocker = try .init(testing.allocator);
+ defer blocker.deinit();
+
+ try testLoad(&blocker,
+ \\&utm_tracker=
+ );
+
+ // 140 tokens of query noise push the filter's token ("utm", its rarest)
+ // past what the request holds; the engine walks the rest of the URL.
+ const noise = "https://example.com/?" ++ "a=1&" ** 70;
+ const overflowing: Request = .init(noise ++ "utm_tracker=1", "a.com", script);
+ try testing.expect(overflowing.tail.len != 0);
+ try testing.expectEqual(.blocked, blocker.match(&overflowing));
+ // A token past the buffer finds its bucket, but the pattern does not fit.
+ try expectVerdict(&blocker, .none, noise ++ "utm_other=1", "a.com", script);
+ try expectVerdict(&blocker, .none, noise ++ "b=2", "a.com", script);
+}
+
test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zig
index 01ef5fc43..4b8184511 100644
--- a/src/network/adblock/Engine.zig
+++ b/src/network/adblock/Engine.zig
@@ -72,10 +72,13 @@ pub const Request = struct {
/// Exactly one bit set.
kind: NetworkFilter.ResourceTypes,
third_party: bool,
- /// The URL's tokens, hashed once here so no engine retokenizes. A URL
- /// long enough to overflow loses its last tokens.
+ /// The URL's first tokens, hashed once here so no engine retokenizes.
tokens_buf: [URL_TOKENS_MAX]u32,
tokens_len: usize,
+ /// The rest of the URL, past the last token that fit.
+ /// Each engine tokenizes it itself, so no token is ever lost. Token-free for
+ /// all but the longest URLs.
+ tail: []const u8,
const URL_TOKENS_MAX = 128;
@@ -108,13 +111,15 @@ pub const Request = struct {
.third_party = domain.isThirdParty(parsed.hostname(), source),
.tokens_buf = undefined,
.tokens_len = 0,
+ .tail = "",
};
var it: Tokens = .{ .text = url };
- while (it.next()) |token| {
- if (request.tokens_len == request.tokens_buf.len) break;
+ while (request.tokens_len < request.tokens_buf.len) {
+ const token = it.next() orelse break;
request.tokens_buf[request.tokens_len] = token;
request.tokens_len += 1;
}
+ request.tail = url[it.i..];
return request;
}
@@ -177,15 +182,24 @@ const TOKENS_MAX = 32;
/// The first filter that matches, or null.
pub fn match(self: *const Engine, request: *const Request) ?*const NetworkFilter {
- if (self.filters.len != 0) {
+ if (self.buckets.count() != 0) {
for (request.tokens()) |token| {
- const bucket = self.buckets.get(token) orelse continue;
- if (self.matchIn(bucket, request)) |filter| return filter;
+ if (self.matchToken(token, request)) |filter| return filter;
+ }
+ // Whatever the request could not hold is hashed again.
+ var it: Tokens = .{ .text = request.tail };
+ while (it.next()) |token| {
+ if (self.matchToken(token, request)) |filter| return filter;
}
}
return self.matchIn(self.fallback, request);
}
+fn matchToken(self: *const Engine, token: u32, request: *const Request) ?*const NetworkFilter {
+ const bucket = self.buckets.get(token) orelse return null;
+ return self.matchIn(bucket, request);
+}
+
fn matchIn(self: *const Engine, bucket: []const u32, request: *const Request) ?*const NetworkFilter {
for (bucket) |index| {
const filter = &self.filters[index];
@@ -355,6 +369,30 @@ fn contains(tokens: []const u32, token: []const u8) bool {
return false;
}
+test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
+ const max = Request.URL_TOKENS_MAX;
+ const kind: NetworkFilter.ResourceTypes = .{ .script = true };
+
+ // Exactly as many tokens as the buffer holds: nothing is left to walk...
+ const full = "x/" ** (max - 1) ++ "x";
+ var request: Request = .init(full, "", kind);
+ try testing.expectEqual(max, request.tokens_len);
+ try testing.expectEqual(0, request.tail.len);
+
+ // ...one more, and only that one is in the tail.
+ request = .init(full ++ "/y", "", kind);
+ try testing.expectEqual(max, request.tokens_len);
+ try testing.expectString("/y", request.tail);
+ var it: Tokens = .{ .text = request.tail };
+ try testing.expectEqual(hash("y"), it.next().?);
+ try testing.expect(it.next() == null);
+
+ // Fewer than the buffer holds: the tail is empty.
+ request = .init("https://example.com/a", "", kind);
+ try testing.expectEqual(4, request.tokens_len);
+ try testing.expectEqual(0, request.tail.len);
+}
+
test "adblock.Engine: only tokens the URL must reproduce are collected" {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();