Lightpanda: the headless browser designed for AI and automation
cdp: support request interception patterns
Karl Seguinb7d790561f5f
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 1efc61efb..62a387e94 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -40,13 +40,15 @@ const Label = @import("../../browser/webapi/element/html/Label.zig");const Connection = @import("../Connection.zig");const Driver = @import("../Driver.zig");const Incrementing = @import("id.zig").Incrementing;++const fetch = @import("domains/fetch.zig");const network_domain = @import("domains/network.zig");-const InterceptState = @import("domains/fetch.zig").InterceptState;const log = lp.log;const json = std.json;const posix = std.posix;const Allocator = std.mem.Allocator;+const InterceptState = fetch.InterceptState;pub const URL_BASE = "chrome://newtab/";@@ -824,8 +826,10 @@ pub const BrowserContext = struct {}}- pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {+ pub fn fetchEnable(self: *BrowserContext, patterns: ?[]const fetch.RequestPattern, authRequests: bool, session_id: []const u8) !void {self.fetchDisable(); //in case of multiple calls+ errdefer self.fetchDisable();+ try self.intercept_state.setPatterns(patterns);self.fetch_session_id = session_id;try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);if (authRequests) {@@ -843,6 +847,7 @@ pub const BrowserContext = struct {pub fn fetchDisable(self: *BrowserContext) void {self.notification.unregister(.http_request_intercept, self);self.notification.unregister(.http_request_auth_required, self);+ self.intercept_state.clearPatterns();self.fetch_session_id = null;}diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zigindex 5ada67fa5..f4c4798d8 100644--- a/src/server/cdp/domains/fetch.zig+++ b/src/server/cdp/domains/fetch.zig@@ -21,6 +21,7 @@ const lp = @import("lightpanda");const http = @import("../../../network/http.zig");const Notification = @import("../../../Notification.zig");+const HttpClient = @import("../../../network/HttpClient.zig");const id = @import("../id.zig");const CDP = @import("../CDP.zig");@@ -50,46 +51,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {}}-// Stored in CDP. Maps intercept ids to *transfer ids* (not *Transfer-// pointers) of paused transfers waiting for CDP continueRequest/-// fulfillRequest/failRequest/continueWithAuth. Anyone resolving an entry must-// look the transfer up via `Client.findTransfer(id)` — if the transfer has been-// destroyed out-of-band (e.g. frame shutdown), the lookup returns null.-pub const InterceptState = struct {- allocator: Allocator,- next_id: u32 = 1,- waiting: std.AutoArrayHashMapUnmanaged(u32, u32),-- pub fn init(allocator: Allocator) !InterceptState {- return .{- .waiting = .empty,- .allocator = allocator,- };- }-- pub fn put(self: *InterceptState, transfer_id: u32) !u32 {- const intercept_id = self.next_id;- try self.waiting.put(self.allocator, intercept_id, transfer_id);- self.next_id +%= 1;- return intercept_id;- }-- // Returns the transfer id if the intercept id was present and removed.- pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {- const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;- return entry.value;- }-- pub fn deinit(self: *InterceptState) void {- self.waiting.deinit(self.allocator);- }-- pub fn pendingIntercepts(self: *const InterceptState) []u32 {- return self.waiting.values();- }-};--const RequestPattern = struct {+pub const RequestPattern = struct {// Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed.// Escape character is backslash. Omitting is equivalent to "*".urlPattern: []const u8 = "*",@@ -123,7 +85,8 @@ const RequestStage = enum {};const EnableParam = struct {- patterns: []RequestPattern = &.{},+ // null (omitted) means every request; an explicit empty list means none.+ patterns: ?[]RequestPattern = null,handleAuthRequests: bool = false,};const ErrorReason = enum {@@ -158,51 +121,29 @@ fn disable(cmd: *CDP.Command) !void {fn enable(cmd: *CDP.Command) !void {const params = (try cmd.params(EnableParam)) orelse EnableParam{};- if (!arePatternsSupported(params.patterns)) {- log.warn(.not_implemented, "Fetch.enable", .{ .params = "pattern" });- return cmd.sendResult(null, .{});+ if (params.patterns) |patterns| {+ if (patterns.len == 0 and params.handleAuthRequests) {+ return cmd.sendError(-32602, "Can't specify empty patterns with handleAuth set", .{});+ }}-const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;- try bc.fetchEnable(params.handleAuthRequests, try commandSessionId(cmd, bc));-+ try bc.fetchEnable(params.patterns, params.handleAuthRequests, try commandSessionId(cmd, bc));return cmd.sendResult(null, .{});}-fn arePatternsSupported(patterns: []RequestPattern) bool {- if (patterns.len == 0) {- return true;- }- if (patterns.len > 1) {- return false;- }-- // While we don't support patterns, yet, both Playwright and Puppeteer send- // a default pattern which happens to be what we support:- // [{"urlPattern":"*","requestStage":"Request"}]- // So, rather than erroring on this case because we don't support patterns,- // we'll allow it, because this pattern is how it works as-is.- const pattern = patterns[0];- if (!std.mem.eql(u8, pattern.urlPattern, "*")) {- return false;- }- if (pattern.resourceType != null) {- return false;- }- if (pattern.requestStage != .Request) {- return false;- }- return true;-}-pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {// The session that enabled Fetch owns its interception events.const session_id = bc.fetch_session_id orelse return;+ const transfer = intercept.transfer;+ if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {+ // Leaves wait_for_interception false: the request proceeds untouched.+ return;+ }+// We keep it around to wait for modifications to the request.// TODO: What to do when receiving replies for a previous frame's requests?- const transfer = intercept.transfer;const intercept_id = try bc.intercept_state.put(transfer.id);errdefer _ = bc.intercept_state.remove(intercept_id);@@ -421,11 +362,16 @@ fn failRequest(cmd: *CDP.Command) !void {pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {const session_id = bc.fetch_session_id orelse return;+ // Auth challenges are only surfaced for requests the patterns select.+ const transfer = intercept.transfer;+ if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {+ return;+ }+// We keep it around to wait for modifications to the request.// NOTE: we assume whomever created the request created it with a lifetime of the Page.// TODO: What to do when receiving replies for a previous frame's requests?- const transfer = intercept.transfer;const intercept_id = try bc.intercept_state.put(transfer.id);errdefer _ = bc.intercept_state.remove(intercept_id);const request = &transfer.req;@@ -464,6 +410,181 @@ fn idFromRequestId(request_id: []const u8) !u32 {return std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;}+// Stored in CDP. Holds the `Fetch.enable` patterns and maps intercept ids to+// *transfer ids* (not *Transfer pointers) of paused transfers waiting for CDP+// continueRequest/fulfillRequest/failRequest/continueWithAuth. Anyone resolving+// an entry must look the transfer up via `Client.findTransfer(id)` — if the+// transfer has been destroyed out-of-band (e.g. frame shutdown), the lookup+// returns null.+pub const InterceptState = struct {+ allocator: Allocator,+ next_id: u32 = 1,+ waiting: std.AutoArrayHashMapUnmanaged(u32, u32),+ patterns: []const Pattern = &.{},++ const Pattern = struct {+ url: []const u8,+ resource_type: ?ResourceType,+ };++ pub fn init(allocator: Allocator) !InterceptState {+ return .{+ .waiting = .empty,+ .allocator = allocator,+ };+ }++ pub fn put(self: *InterceptState, transfer_id: u32) !u32 {+ const intercept_id = self.next_id;+ try self.waiting.put(self.allocator, intercept_id, transfer_id);+ self.next_id +%= 1;+ return intercept_id;+ }++ // Returns the transfer id if the intercept id was present and removed.+ pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {+ const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;+ return entry.value;+ }++ pub fn deinit(self: *InterceptState) void {+ self.clearPatterns();+ self.waiting.deinit(self.allocator);+ }++ pub fn pendingIntercepts(self: *const InterceptState) []u32 {+ return self.waiting.values();+ }++ pub fn setPatterns(self: *InterceptState, patterns: ?[]const RequestPattern) !void {+ self.clearPatterns();++ var has_response_stage = false;+ const source: []const RequestPattern = patterns orelse &.{.{}};++ var count: usize = 0;+ for (source) |pattern| {+ if (pattern.requestStage == .Response) {+ has_response_stage = true;+ continue;+ }+ count += 1;+ }++ if (has_response_stage) {+ log.warn(.not_implemented, "Fetch.enable", .{ .params = "requestStage=Response" });+ }++ const owned = try self.allocator.alloc(Pattern, count);+ var initialized: usize = 0;+ errdefer {+ for (owned[0..initialized]) |pattern| {+ self.allocator.free(pattern.url);+ }+ self.allocator.free(owned);+ }+ for (source) |pattern| {+ if (pattern.requestStage != .Request) {+ continue;+ }+ owned[initialized] = .{+ .url = try self.allocator.dupe(u8, pattern.urlPattern),+ .resource_type = pattern.resourceType,+ };+ initialized += 1;+ }+ self.patterns = owned;+ }++ pub fn clearPatterns(self: *InterceptState) void {+ for (self.patterns) |pattern| {+ self.allocator.free(pattern.url);+ }+ self.allocator.free(self.patterns);+ self.patterns = &.{};+ }++ pub fn matches(self: *const InterceptState, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {+ for (self.patterns) |pattern| {+ if (pattern.resource_type) |rt| {+ if (rt != cdpResourceType(resource_type)) {+ continue;+ }+ }+ if (wildcardMatch(pattern.url, url)) {+ return true;+ }+ }+ return false;+ }++ fn cdpResourceType(resource_type: HttpClient.Request.ResourceType) ResourceType {+ return switch (resource_type) {+ .document => .Document,+ .xhr => .XHR,+ .script => .Script,+ .fetch => .Fetch,+ .stylesheet => .Stylesheet,+ .eventsource => .EventSource,+ .image => .Image,+ };+ }++ // RequestPattern.urlPattern semantics: '*' matches zero or more characters,+ // '?' exactly one, '\' escapes the next character.+ // https://research.swtch.com/glob+ fn wildcardMatch(pattern: []const u8, value: []const u8) bool {+ var value_index: usize = 0;+ var pattern_index: usize = 0;+ // Position just past the most recent '*', and the value position it was+ // matched against; on a mismatch we retry from there with the star+ // consuming one more character.+ var star_index: ?usize = null;+ var star_value_index: usize = 0;++ while (value_index < value.len) {+ if (pattern_index < pattern.len) {+ switch (pattern[pattern_index]) {+ '*' => {+ pattern_index += 1;+ star_index = pattern_index;+ star_value_index = value_index;+ continue;+ },+ '?' => {+ pattern_index += 1;+ value_index += 1;+ continue;+ },+ else => |c| {+ var literal = c;+ var advance: usize = 1;+ if (c == '\\' and pattern_index + 1 < pattern.len) {+ literal = pattern[pattern_index + 1];+ advance = 2;+ }+ if (literal == value[value_index]) {+ pattern_index += advance;+ value_index += 1;+ continue;+ }+ },+ }+ }++ const star = star_index orelse return false;+ pattern_index = star;+ star_value_index += 1;+ value_index = star_value_index;+ }++ while (pattern_index < pattern.len and pattern[pattern_index] == '*') {+ pattern_index += 1;+ }+ return pattern_index == pattern.len;+ }+};+const testing = @import("../testing.zig");test "cdp.Fetch: interception events belong to the enabling session" {var ctx = try testing.context();@@ -515,3 +636,115 @@ test "cdp.Fetch: InterceptState issues a fresh id per pause" {try testing.expectEqual(7, state.remove(second).?);try testing.expectEqual(null, state.remove(second));}++test "cdp.Fetch: wildcardMatch" {+ // exact, case-sensitive+ try testing.expect(InterceptState.wildcardMatch("http://h/a/b", "http://h/a/b"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a/b", "http://h/a/bc"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a/b", "http://H/a/b"));++ // '*' = zero or more+ try testing.expect(InterceptState.wildcardMatch("*", ""));+ try testing.expect(InterceptState.wildcardMatch("*", "http://h/"));+ try testing.expect(InterceptState.wildcardMatch("**a", "a"));+ try testing.expect(InterceptState.wildcardMatch("a*", "a"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("a*", ""));+ try testing.expect(InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/api/get?pageSize=1"));+ try testing.expect(InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/api/post"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/apiary"));+ try testing.expect(InterceptState.wildcardMatch("*api*", "http://h/api/x"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("*api*", "http://h/index.html"));+ // backtracking past a greedy star+ try testing.expect(InterceptState.wildcardMatch("*a*b?c", "xxaxxbYc"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("*a*b?c", "xxaxxbc"));++ // '?' = exactly one+ try testing.expect(InterceptState.wildcardMatch("http://h/?.json", "http://h/a.json"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/?.json", "http://h/ab.json"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/?.json", "http://h/.json"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("a?", "a"));++ // '\' escapes the next character; a trailing '\' is literal+ try testing.expect(InterceptState.wildcardMatch("http://h/a\\?b=1", "http://h/a?b=1"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a\\?b=1", "http://h/aXb=1"));+ try testing.expect(InterceptState.wildcardMatch("http://h/\\*", "http://h/*"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/\\*", "http://h/x"));+ try testing.expect(InterceptState.wildcardMatch("*\\\\", "a\\"));+ try testing.expect(InterceptState.wildcardMatch("abc\\", "abc\\"));+ try testing.expectEqual(false, InterceptState.wildcardMatch("abc\\", "abc"));+}++test "cdp.Fetch: InterceptState patterns" {+ testing.expectLog(&.{.not_implemented});++ var state = try InterceptState.init(testing.allocator);+ defer state.deinit();++ // disabled: nothing is paused+ try testing.expectEqual(false, state.matches("http://h/", .document));++ // omitted patterns = every request+ try state.setPatterns(null);+ try testing.expect(state.matches("http://h/", .document));+ try testing.expect(state.matches("http://h/x.js", .script));++ // an explicit empty list = no request+ try state.setPatterns(&.{});+ try testing.expectEqual(false, state.matches("http://h/", .document));++ try state.setPatterns(&.{+ .{ .urlPattern = "*/api/*", .resourceType = .XHR },+ .{ .urlPattern = "*.css", .requestStage = .Response },+ });+ try testing.expectEqual(1, state.patterns.len);+ try testing.expect(state.matches("http://h/api/x", .xhr));+ try testing.expectEqual(false, state.matches("http://h/api/x", .fetch));+ try testing.expectEqual(false, state.matches("http://h/other", .xhr));+ // Response-stage entries are dropped+ try testing.expectEqual(false, state.matches("http://h/a.css", .stylesheet));++ state.clearPatterns();+ try testing.expectEqual(false, state.matches("http://h/api/x", .xhr));+}++test "cdp.Fetch: enable compiles patterns, disable clears them" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{+ .session_id = "SID-P",+ .target_id = "TID-000000000B".*,+ });++ try ctx.processMessage(.{+ .id = 1,+ .method = "Fetch.enable",+ .params = .{ .patterns = .{+ .{ .urlPattern = "*/json/*", .resourceType = "Fetch" },+ .{ .urlPattern = "http://h/exact" },+ } },+ .sessionId = "SID-P",+ });+ try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-P" });+ try testing.expectEqual(2, bc.intercept_state.patterns.len);+ try testing.expect(bc.intercept_state.matches("http://h/json/a", .fetch));+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/json/a", .xhr));+ try testing.expect(bc.intercept_state.matches("http://h/exact", .xhr));+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));++ // re-enable without patterns = catch-all+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.enable", .sessionId = "SID-P" });+ try ctx.expectSentResult(null, .{ .id = 2, .session_id = "SID-P" });+ try testing.expect(bc.intercept_state.matches("http://h/", .document));++ // explicit empty list = pause nothing; rejected when auth handling is on+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.enable", .params = .{ .patterns = .{} }, .sessionId = "SID-P" });+ try ctx.expectSentResult(null, .{ .id = 3, .session_id = "SID-P" });+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));+ try ctx.processMessage(.{ .id = 4, .method = "Fetch.enable", .params = .{ .patterns = .{}, .handleAuthRequests = true }, .sessionId = "SID-P" });+ try ctx.expectSentError(-32602, "Can't specify empty patterns with handleAuth set", .{ .id = 4 });++ try ctx.processMessage(.{ .id = 5, .method = "Fetch.disable", .sessionId = "SID-P" });+ try testing.expectEqual(0, bc.intercept_state.patterns.len);+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));+}