Lightpanda: the headless browser designed for AI and automation
`Regex`: share the PCRE2 wrapper beyond adblock
Adrià Arrufatd2f72a9fa996
diff --git a/src/App.zig b/src/App.zigindex fe5a0effa..798bd4b43 100644--- a/src/App.zig+++ b/src/App.zig@@ -20,6 +20,7 @@ const std = @import("std");const lp = @import("lightpanda");const Config = @import("Config.zig");+const Regex = @import("Regex.zig");const Snapshot = @import("browser/js/Snapshot.zig");const Platform = @import("browser/js/Platform.zig");const Telemetry = @import("telemetry/telemetry.zig").Telemetry;@@ -43,6 +44,10 @@ allocator: Allocator,arena_pool: ArenaPool,app_dir_path: ?[]const u8,+// Compiles every pattern the process runs: adblock lists, tool arguments.+// Heap-held because PCRE2 keeps its address.+regex_context: *Regex.Context,+pub fn init(allocator: Allocator, config: *const Config) !*App {const platform = try Platform.init(.{.v8_flags = config.v8Flags(),@@ -54,6 +59,9 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {const snapshot = try Snapshot.load();errdefer snapshot.deinit();+ const regex_context: *Regex.Context = try .init(allocator);+ errdefer regex_context.deinit();+const app = try allocator.create(App);errdefer allocator.destroy(app);@@ -62,6 +70,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {.allocator = allocator,.platform = platform,.snapshot = snapshot,+ .regex_context = regex_context,.network = undefined,.app_dir_path = undefined,.telemetry = undefined,@@ -96,6 +105,8 @@ pub fn deinit(self: *App) void {}self.telemetry.deinit(allocator);self.network.deinit();+ // Compiled patterns free through the context, so it goes after them.+ self.regex_context.deinit();self.snapshot.deinit();self.platform.deinit();self.arena_pool.deinit();diff --git a/src/network/adblock/Regex.zig b/src/Regex.zigsimilarity index 63%rename from src/network/adblock/Regex.zigrename to src/Regex.zigindex 76806f15d..640cbe69d 100644--- a/src/network/adblock/Regex.zig+++ b/src/Regex.zig@@ -16,23 +16,19 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-//! A compiled `/regex/` filter body. Filter lists write them in JavaScript-//! `RegExp` syntax and uBO runs them with `new RegExp(src, 'i')` against the-//! raw request URL (no flag under `$match-case`); PCRE2 reads that syntax-//! as-is, escapes like `\/` included.+//! A compiled pattern in JavaScript `RegExp` syntax, run by PCRE2. Adblock+//! lists and agent tool arguments both write regexes that way, and PCRE2+//! reads the syntax as-is, escapes like `\/` included.//!//! A compiled pattern and its `Context` are never modified after `compile`,-//! so one `Regex` can be shared by every HTTP client thread; the per-call-//! match data is what PCRE2 requires to be private.+//! so one `Regex` can be shared by every thread; the per-call match data is+//! what PCRE2 requires to be private.const std = @import("std");-const lp = @import("lightpanda");const pcre2 = @import("pcre2");const Allocator = std.mem.Allocator;-const log = lp.log;-const Regex = @This();code: *pcre2.pcre2_code_8,@@ -40,21 +36,40 @@ context: *const Context,pub const Error = error{ InvalidRegex, OutOfMemory };+pub const Options = struct {+ case_insensitive: bool = false,+ /// UTF-8 aware matching: `.` consumes a code point and caseless folding+ /// works beyond ASCII. An invalid sequence in the subject fails to match+ /// rather than erroring. `\b` and `\w` stay ASCII, as in JavaScript.+ unicode: bool = false,+};++/// Why a compile failed, for the caller to log or show.+pub const Diagnostic = struct {+ offset: usize = 0,+ len: usize = 0,+ buf: [256]u8 = undefined,++ pub fn message(self: *const Diagnostic) []const u8 {+ return self.buf[0..self.len];+ }+};+/// What every `Regex` compiled through it shares: the allocator PCRE2 draws/// from, and the compile and match settings. Outlives the regexes.///-/// PCRE2 would happily use libc's malloc; it is handed the blocker's-/// allocator so that a compiled pattern nobody freed fails a test the way-/// any other leak does.+/// PCRE2 would happily use libc's malloc; it is handed the owner's allocator+/// so that a compiled pattern nobody freed fails a test the way any other+/// leak does.pub const Context = struct {allocator: Allocator,general: *pcre2.pcre2_general_context_8,compile_context: *pcre2.pcre2_compile_context_8,match_context: *pcre2.pcre2_match_context_8,- // A pattern from a list that backtracks this much on one URL is broken,- // not slow; giving up costs a false negative on that request, nothing- // more.+ // Patterns come from lists and prompts, never from code: one that+ // backtracks this much on a subject is broken, not slow, and giving up+ // costs a false negative on that subject, nothing more.const MATCH_LIMIT = 100_000;const DEPTH_LIMIT = 10_000;@@ -72,7 +87,7 @@ pub const Context = struct {const compile_context = pcre2.pcre2_compile_context_create_8(general) orelse return error.OutOfMemory;errdefer pcre2.pcre2_compile_context_free_8(compile_context);// JavaScript without the `u` flag reads an unknown escape as the- // literal character, and that is the mode uBO compiles filters in.+ // literal character._ = pcre2.pcre2_set_compile_extra_options_8(compile_context, pcre2.PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL);const match_context = pcre2.pcre2_match_context_create_8(general) orelse return error.OutOfMemory;@@ -114,28 +129,30 @@ pub const Context = struct {}};-pub fn compile(context: *const Context, pattern: []const u8, case_insensitive: bool) Error!Regex {- const options: u32 = if (case_insensitive) pcre2.PCRE2_CASELESS else 0;+/// A failed compile fills `diag`, when given, with PCRE2's message and the+/// offset of the offending character.+pub fn compile(context: *const Context, pattern: []const u8, options: Options, diag: ?*Diagnostic) Error!Regex {+ var flags: u32 = 0;+ if (options.case_insensitive) flags |= pcre2.PCRE2_CASELESS;+ if (options.unicode) flags |= pcre2.PCRE2_UTF | pcre2.PCRE2_MATCH_INVALID_UTF;+var err_code: c_int = 0;var err_offset: usize = 0;const code = pcre2.pcre2_compile_8(pattern.ptr,pattern.len,- options,+ flags,&err_code,&err_offset,context.compile_context,) orelse {// A failed allocation is ours, not the pattern's.if (err_code == pcre2.PCRE2_ERROR_HEAP_FAILED) return error.OutOfMemory;- var buf: [256]u8 = undefined;- const len = pcre2.pcre2_get_error_message_8(err_code, &buf, buf.len);- const message: []const u8 = if (len < 0) "unknown error" else buf[0..@intCast(len)];- log.debug(.app, "adblock regex rejected", .{- .pattern = pattern,- .err = message,- .offset = err_offset,- });+ if (diag) |d| {+ const len = pcre2.pcre2_get_error_message_8(err_code, &d.buf, d.buf.len);+ d.len = if (len < 0) 0 else @intCast(len);+ d.offset = err_offset;+ }return error.InvalidRegex;};return .{ .code = code, .context = context };@@ -152,7 +169,8 @@ const MATCH_SCRATCH = 24 * 1024;/// Whether the pattern matches anywhere in `text`, as `RegExp.test` would/// answer. A match that hits the backtracking limits counts as no match.pub fn matches(self: Regex, text: []const u8) bool {- // This runs per request; the scratch keeps the common case off the heap.+ // This can run per request or per DOM node; the scratch keeps the common+ // case off the heap.var scratch = std.heap.stackFallback(MATCH_SCRATCH, self.context.allocator);var allocator = scratch.get();const general = pcre2.pcre2_general_context_create_8(Context.cMalloc, Context.cFree, &allocator) orelse return false;@@ -167,13 +185,13 @@ pub fn matches(self: Regex, text: []const u8) bool {return rc >= 0;}-const testing = @import("../../testing.zig");+const testing = @import("testing.zig");-test "adblock.Regex: JavaScript escapes and unanchored search" {+test "Regex: JavaScript escapes and unanchored search" {const context: *Context = try .init(testing.allocator);defer context.deinit();- const regex = try Regex.compile(context, "^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*", true);+ const regex = try Regex.compile(context, "^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*", .{ .case_insensitive = true }, null);defer regex.deinit();try testing.expect(regex.matches("https://abcde.com/x"));@@ -181,44 +199,85 @@ test "adblock.Regex: JavaScript escapes and unanchored search" {try testing.expect(!regex.matches("https://abcd.com/x"));try testing.expect(!regex.matches("https://abcde.org/x"));- const invoke = try Regex.compile(context, "\\/[0-9a-f]{32}\\/invoke\\.js", true);+ const invoke = try Regex.compile(context, "\\/[0-9a-f]{32}\\/invoke\\.js", .{ .case_insensitive = true }, null);defer invoke.deinit();try testing.expect(invoke.matches("https://host.com/0123456789abcdef0123456789abcdef/invoke.js"));try testing.expect(!invoke.matches("https://host.com/0123456789abcdef0123456789abcde/invoke.js"));- const dash = try Regex.compile(context, "[a-z\\-]+\\?s=", true);+ const dash = try Regex.compile(context, "[a-z\\-]+\\?s=", .{ .case_insensitive = true }, null);defer dash.deinit();try testing.expect(dash.matches("https://x.com/a-b?s=1"));try testing.expect(!dash.matches("https://x.com/?s=1"));}-test "adblock.Regex: $match-case keeps the case" {+test "Regex: case is kept by default" {const context: *Context = try .init(testing.allocator);defer context.deinit();- const exact = try Regex.compile(context, "\\/[a-z0-9]{12}\\/[a-zA-Z0-9]{20,}$", false);+ const exact = try Regex.compile(context, "\\/[a-z0-9]{12}\\/[a-zA-Z0-9]{20,}$", .{}, null);defer exact.deinit();try testing.expect(exact.matches("https://x.com/abcdef123456/aBcDeFgHiJkLmNoPqRsTuV"));try testing.expect(!exact.matches("https://x.com/ABCDEF123456/aBcDeFgHiJkLmNoPqRsTuV"));}-test "adblock.Regex: invalid patterns are errors, runaway ones no match" {+test "Regex: invalid patterns are errors, runaway ones no match" {const context: *Context = try .init(testing.allocator);defer context.deinit();- try testing.expectError(error.InvalidRegex, Regex.compile(context, "(", true));- try testing.expectError(error.InvalidRegex, Regex.compile(context, "a{2,1}", true));+ try testing.expectError(error.InvalidRegex, Regex.compile(context, "(", .{}, null));+ try testing.expectError(error.InvalidRegex, Regex.compile(context, "a{2,1}", .{}, null));// An unknown alphanumeric escape is the literal, as in JavaScript.- const literal = try Regex.compile(context, "\\q", true);+ const literal = try Regex.compile(context, "\\q", .{ .case_insensitive = true }, null);defer literal.deinit();try testing.expect(literal.matches("https://x.com/q"));// Exponential backtracking stops at the match limit instead of stalling- // the request.- const runaway = try Regex.compile(context, "^(a+)+$", true);+ // the caller.+ const runaway = try Regex.compile(context, "^(a+)+$", .{ .case_insensitive = true }, null);defer runaway.deinit();const subject = "a" ** 64 ++ "b";try testing.expect(!runaway.matches(subject));try testing.expect(runaway.matches("a" ** 64));}++test "Regex: a diagnostic names the fault and where it is" {+ const context: *Context = try .init(testing.allocator);+ defer context.deinit();++ var diag: Diagnostic = .{};+ try testing.expectError(error.InvalidRegex, Regex.compile(context, "ab(", .{}, &diag));+ try testing.expectString("missing closing parenthesis", diag.message());+ try testing.expectEqual(3, diag.offset);+}++test "Regex: unicode folds case beyond ASCII and tolerates invalid bytes" {+ const context: *Context = try .init(testing.allocator);+ defer context.deinit();++ const ascii = try Regex.compile(context, "^реклама$", .{ .case_insensitive = true }, null);+ defer ascii.deinit();+ try testing.expect(ascii.matches("реклама"));+ try testing.expect(!ascii.matches("Реклама"));++ const unicode = try Regex.compile(context, "^реклама$", .{ .case_insensitive = true, .unicode = true }, null);+ defer unicode.deinit();+ try testing.expect(unicode.matches("Реклама"));+ try testing.expect(unicode.matches("РЕКЛАМА"));++ // One code point, not one byte.+ const single = try Regex.compile(context, "^.$", .{ .unicode = true }, null);+ defer single.deinit();+ try testing.expect(single.matches("é"));+ try testing.expect(!single.matches("ab"));++ // Word boundaries stay ASCII, as in JavaScript.+ const word = try Regex.compile(context, "\\bshare\\b", .{ .unicode = true }, null);+ defer word.deinit();+ try testing.expect(word.matches("éshare"));++ const sidebar = try Regex.compile(context, "sidebar", .{ .unicode = true }, null);+ defer sidebar.deinit();+ try testing.expect(sidebar.matches("sidebar\xFF"));+ try testing.expect(!sidebar.matches("\xFF"));+}diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 7cb7c9c83..eaa13ed26 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -22,6 +22,7 @@ pub const log = @import("log.zig");pub const mcp = @import("mcp.zig");pub const App = @import("App.zig");pub const Arena = @import("Arena.zig");+pub const Regex = @import("Regex.zig");pub const Config = @import("Config.zig");pub const cookies = @import("cookies.zig");pub const datetime = @import("datetime.zig");diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex fd469377e..55c07ecf8 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -4427,7 +4427,7 @@ test "HttpClient: adblock verdicts apply per request" {var client: Client = undefined;initTestClient(&client, &pool);- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();var list: std.Io.Reader = .fixed(\\||ads.example.com^diff --git a/src/network/Network.zig b/src/network/Network.zigindex b180c4936..71ba1fe3f 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -106,7 +106,7 @@ pub fn init(app: *App) !Network {null;errdefer if (web_bot_auth) |wba| wba.deinit(allocator);- var adblocker = try AdBlocker.fromConfig(allocator, config);+ var adblocker = try AdBlocker.fromConfig(allocator, config, app.regex_context);errdefer if (adblocker) |*blocker| blocker.deinit();var cache = try Cache.init(allocator, config);diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zigindex 9699783aa..255b4c206 100644--- a/src/network/adblock/AdBlocker.zig+++ b/src/network/adblock/AdBlocker.zig@@ -29,7 +29,7 @@ const Parser = @import("Parser.zig");const Engine = @import("Engine.zig");const HostnameTrie = @import("HostnameTrie.zig");const NetworkFilter = @import("NetworkFilter.zig");-const Regex = @import("Regex.zig");+const Regex = lp.Regex;const log = lp.log;@@ -52,7 +52,7 @@ badfilters: std.AutoHashMapUnmanaged(u64, void),/// filter; they are freed here, not through `filters`.regexes: std.ArrayList(*const Regex),/// What the regexes are compiled and run with; PCRE2 allocates through it.-regex_context: *Regex.Context,+regex_context: *const Regex.Context,built: bool,trie: HostnameTrie,blocked: u32,@@ -82,9 +82,7 @@ rules_cosmetic: usize,/// Real-world rules are well under 1KB; the parser skips anything longer.const LINE_MAX = 8 * 1024;-pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {- const regex_context: *Regex.Context = try .init(allocator);- errdefer regex_context.deinit();+pub fn init(allocator: Allocator, regex_context: *const Regex.Context) Allocator.Error!AdBlocker {var trie: HostnameTrie = try .init(allocator);errdefer trie.deinit(allocator);const blocked = try trie.createTrie(allocator);@@ -121,7 +119,6 @@ pub fn deinit(self: *AdBlocker) void {self.badfilters.deinit(self.allocator);for (self.regexes.items) |regex| regex.deinit();self.regexes.deinit(self.allocator);- self.regex_context.deinit();self.filters.deinit(self.allocator);self.trie.deinit(self.allocator);self.arena.deinit();@@ -129,7 +126,7 @@ pub fn deinit(self: *AdBlocker) void {/// Builds the blocker from `--adblock-lists`, or null when the option is/// unset. Lists accumulate into the one instance.-pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {+pub fn fromConfig(allocator: Allocator, config: *const Config, regex_context: *const Regex.Context) !?AdBlocker {var paths = config.adblockLists() orelse return null;var adblocker: ?AdBlocker = null;@@ -140,7 +137,7 @@ pub fn fromConfig(allocator: Allocator, config: *const Config) !?AdBlocker {while (paths.next()) |path| {if (path.len == 0) continue;- if (adblocker == null) adblocker = try AdBlocker.init(allocator);+ if (adblocker == null) adblocker = try AdBlocker.init(allocator, regex_context);loadList(&adblocker.?, path, buf) catch |err| {log.err(.app, "adblock list load failed", .{ .path = path, .err = err });return err;@@ -225,9 +222,15 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {if (filter.kind == .regex) {const body = filter.pattern[1 .. filter.pattern.len - 1];- const compiled = Regex.compile(self.regex_context, body, !filter.match_case) catch |err| switch (err) {+ var diag: Regex.Diagnostic = .{};+ const compiled = Regex.compile(self.regex_context, body, .{ .case_insensitive = !filter.match_case }, &diag) catch |err| switch (err) {error.OutOfMemory => return error.OutOfMemory,error.InvalidRegex => {+ log.debug(.app, "adblock regex rejected", .{+ .pattern = body,+ .err = diag.message(),+ .offset = diag.offset,+ });self.rules_skipped += 1;continue;},@@ -481,7 +484,7 @@ const document: ResourceTypes = .{ .document = true };const frame: ResourceTypes = .{ .subdocument = true };test "adblock.AdBlocker: the doubleclick rules from EasyList" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -523,7 +526,7 @@ test "adblock.AdBlocker: the doubleclick rules from EasyList" {}test "adblock.AdBlocker: the youtube rules from EasyList" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -559,7 +562,7 @@ test "adblock.AdBlocker: the youtube rules from EasyList" {}test "adblock.AdBlocker: parse accumulates across lists" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();var first: Io.Reader = .fixed(@@ -596,7 +599,7 @@ test "adblock.AdBlocker: parse accumulates across lists" {}test "adblock.AdBlocker: tokens past the request buffer still match" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -616,7 +619,7 @@ test "adblock.AdBlocker: tokens past the request buffer still match" {}test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -633,7 +636,7 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {}test "adblock.AdBlocker: the regex rules from EasyList" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -666,7 +669,7 @@ test "adblock.AdBlocker: the regex rules from EasyList" {}test "adblock.AdBlocker: a regex is found under every token it may match" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -679,7 +682,7 @@ test "adblock.AdBlocker: a regex is found under every token it may match" {}test "adblock.AdBlocker: $badfilter removes a regex rule" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -692,7 +695,7 @@ test "adblock.AdBlocker: $badfilter removes a regex rule" {}test "adblock.AdBlocker: verdict precedence" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -717,7 +720,7 @@ test "adblock.AdBlocker: verdict precedence" {}test "adblock.AdBlocker: $badfilter removes its target" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -743,7 +746,7 @@ test "adblock.AdBlocker: $badfilter removes its target" {}test "adblock.AdBlocker: exceptions we cannot read suppress their hostname" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -769,7 +772,7 @@ test "adblock.AdBlocker: exceptions we cannot read suppress their hostname" {}test "adblock.AdBlocker: trie absorbs every pure-hostname form" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try testLoad(&blocker,@@ -795,7 +798,7 @@ test "adblock.AdBlocker: trie absorbs every pure-hostname form" {}test "adblock.AdBlocker: $important on an exception is invalid" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();// uBO rejects `@@…$important`, so nothing outranks a `$important`@@ -812,7 +815,7 @@ test "adblock.AdBlocker: $important on an exception is invalid" {}test "adblock.AdBlocker: an empty blocker decides nothing" {- var blocker: AdBlocker = try .init(testing.allocator);+ var blocker: AdBlocker = try .init(testing.allocator, testing.test_app.regex_context);defer blocker.deinit();try blocker.build();diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zigindex 6140c5832..9e23e2862 100644--- a/src/network/adblock/NetworkFilter.zig+++ b/src/network/adblock/NetworkFilter.zig@@ -18,7 +18,7 @@const std = @import("std");const domain = @import("domain.zig");-const Regex = @import("Regex.zig");+const Regex = @import("../../Regex.zig");const NetworkFilter = @This();