Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: support request interception patterns

Karl Seguinb7d790561f5f
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index 1efc61efb..62a387e94 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -40,13 +40,15 @@ const Label = @import("../../browser/webapi/element/html/Label.zig");
const Connection = @import("../Connection.zig");
const Driver = @import("../Driver.zig");
const Incrementing = @import("id.zig").Incrementing;
+
+const fetch = @import("domains/fetch.zig");
const network_domain = @import("domains/network.zig");
-const InterceptState = @import("domains/fetch.zig").InterceptState;
const log = lp.log;
const json = std.json;
const posix = std.posix;
const Allocator = std.mem.Allocator;
+const InterceptState = fetch.InterceptState;
pub const URL_BASE = "chrome://newtab/";
@@ -824,8 +826,10 @@ pub const BrowserContext = struct {
}
}
- pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {
+ pub fn fetchEnable(self: *BrowserContext, patterns: ?[]const fetch.RequestPattern, authRequests: bool, session_id: []const u8) !void {
self.fetchDisable(); //in case of multiple calls
+ errdefer self.fetchDisable();
+ try self.intercept_state.setPatterns(patterns);
self.fetch_session_id = session_id;
try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);
if (authRequests) {
@@ -843,6 +847,7 @@ pub const BrowserContext = struct {
pub fn fetchDisable(self: *BrowserContext) void {
self.notification.unregister(.http_request_intercept, self);
self.notification.unregister(.http_request_auth_required, self);
+ self.intercept_state.clearPatterns();
self.fetch_session_id = null;
}
diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zig
index 5ada67fa5..f4c4798d8 100644
--- a/src/server/cdp/domains/fetch.zig
+++ b/src/server/cdp/domains/fetch.zig
@@ -21,6 +21,7 @@ const lp = @import("lightpanda");
const http = @import("../../../network/http.zig");
const Notification = @import("../../../Notification.zig");
+const HttpClient = @import("../../../network/HttpClient.zig");
const id = @import("../id.zig");
const CDP = @import("../CDP.zig");
@@ -50,46 +51,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
}
}
-// Stored in CDP. Maps intercept ids to *transfer ids* (not *Transfer
-// pointers) of paused transfers waiting for CDP continueRequest/
-// fulfillRequest/failRequest/continueWithAuth. Anyone resolving an entry must
-// look the transfer up via `Client.findTransfer(id)` — if the transfer has been
-// destroyed out-of-band (e.g. frame shutdown), the lookup returns null.
-pub const InterceptState = struct {
- allocator: Allocator,
- next_id: u32 = 1,
- waiting: std.AutoArrayHashMapUnmanaged(u32, u32),
-
- pub fn init(allocator: Allocator) !InterceptState {
- return .{
- .waiting = .empty,
- .allocator = allocator,
- };
- }
-
- pub fn put(self: *InterceptState, transfer_id: u32) !u32 {
- const intercept_id = self.next_id;
- try self.waiting.put(self.allocator, intercept_id, transfer_id);
- self.next_id +%= 1;
- return intercept_id;
- }
-
- // Returns the transfer id if the intercept id was present and removed.
- pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {
- const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;
- return entry.value;
- }
-
- pub fn deinit(self: *InterceptState) void {
- self.waiting.deinit(self.allocator);
- }
-
- pub fn pendingIntercepts(self: *const InterceptState) []u32 {
- return self.waiting.values();
- }
-};
-
-const RequestPattern = struct {
+pub const RequestPattern = struct {
// Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed.
// Escape character is backslash. Omitting is equivalent to "*".
urlPattern: []const u8 = "*",
@@ -123,7 +85,8 @@ const RequestStage = enum {
};
const EnableParam = struct {
- patterns: []RequestPattern = &.{},
+ // null (omitted) means every request; an explicit empty list means none.
+ patterns: ?[]RequestPattern = null,
handleAuthRequests: bool = false,
};
const ErrorReason = enum {
@@ -158,51 +121,29 @@ fn disable(cmd: *CDP.Command) !void {
fn enable(cmd: *CDP.Command) !void {
const params = (try cmd.params(EnableParam)) orelse EnableParam{};
- if (!arePatternsSupported(params.patterns)) {
- log.warn(.not_implemented, "Fetch.enable", .{ .params = "pattern" });
- return cmd.sendResult(null, .{});
+ if (params.patterns) |patterns| {
+ if (patterns.len == 0 and params.handleAuthRequests) {
+ return cmd.sendError(-32602, "Can't specify empty patterns with handleAuth set", .{});
+ }
}
-
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
- try bc.fetchEnable(params.handleAuthRequests, try commandSessionId(cmd, bc));
-
+ try bc.fetchEnable(params.patterns, params.handleAuthRequests, try commandSessionId(cmd, bc));
return cmd.sendResult(null, .{});
}
-fn arePatternsSupported(patterns: []RequestPattern) bool {
- if (patterns.len == 0) {
- return true;
- }
- if (patterns.len > 1) {
- return false;
- }
-
- // While we don't support patterns, yet, both Playwright and Puppeteer send
- // a default pattern which happens to be what we support:
- // [{"urlPattern":"*","requestStage":"Request"}]
- // So, rather than erroring on this case because we don't support patterns,
- // we'll allow it, because this pattern is how it works as-is.
- const pattern = patterns[0];
- if (!std.mem.eql(u8, pattern.urlPattern, "*")) {
- return false;
- }
- if (pattern.resourceType != null) {
- return false;
- }
- if (pattern.requestStage != .Request) {
- return false;
- }
- return true;
-}
-
pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {
// The session that enabled Fetch owns its interception events.
const session_id = bc.fetch_session_id orelse return;
+ const transfer = intercept.transfer;
+ if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {
+ // Leaves wait_for_interception false: the request proceeds untouched.
+ return;
+ }
+
// We keep it around to wait for modifications to the request.
// TODO: What to do when receiving replies for a previous frame's requests?
- const transfer = intercept.transfer;
const intercept_id = try bc.intercept_state.put(transfer.id);
errdefer _ = bc.intercept_state.remove(intercept_id);
@@ -421,11 +362,16 @@ fn failRequest(cmd: *CDP.Command) !void {
pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {
const session_id = bc.fetch_session_id orelse return;
+ // Auth challenges are only surfaced for requests the patterns select.
+ const transfer = intercept.transfer;
+ if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {
+ return;
+ }
+
// We keep it around to wait for modifications to the request.
// NOTE: we assume whomever created the request created it with a lifetime of the Page.
// TODO: What to do when receiving replies for a previous frame's requests?
- const transfer = intercept.transfer;
const intercept_id = try bc.intercept_state.put(transfer.id);
errdefer _ = bc.intercept_state.remove(intercept_id);
const request = &transfer.req;
@@ -464,6 +410,181 @@ fn idFromRequestId(request_id: []const u8) !u32 {
return std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;
}
+// Stored in CDP. Holds the `Fetch.enable` patterns and maps intercept ids to
+// *transfer ids* (not *Transfer pointers) of paused transfers waiting for CDP
+// continueRequest/fulfillRequest/failRequest/continueWithAuth. Anyone resolving
+// an entry must look the transfer up via `Client.findTransfer(id)` — if the
+// transfer has been destroyed out-of-band (e.g. frame shutdown), the lookup
+// returns null.
+pub const InterceptState = struct {
+ allocator: Allocator,
+ next_id: u32 = 1,
+ waiting: std.AutoArrayHashMapUnmanaged(u32, u32),
+ patterns: []const Pattern = &.{},
+
+ const Pattern = struct {
+ url: []const u8,
+ resource_type: ?ResourceType,
+ };
+
+ pub fn init(allocator: Allocator) !InterceptState {
+ return .{
+ .waiting = .empty,
+ .allocator = allocator,
+ };
+ }
+
+ pub fn put(self: *InterceptState, transfer_id: u32) !u32 {
+ const intercept_id = self.next_id;
+ try self.waiting.put(self.allocator, intercept_id, transfer_id);
+ self.next_id +%= 1;
+ return intercept_id;
+ }
+
+ // Returns the transfer id if the intercept id was present and removed.
+ pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {
+ const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;
+ return entry.value;
+ }
+
+ pub fn deinit(self: *InterceptState) void {
+ self.clearPatterns();
+ self.waiting.deinit(self.allocator);
+ }
+
+ pub fn pendingIntercepts(self: *const InterceptState) []u32 {
+ return self.waiting.values();
+ }
+
+ pub fn setPatterns(self: *InterceptState, patterns: ?[]const RequestPattern) !void {
+ self.clearPatterns();
+
+ var has_response_stage = false;
+ const source: []const RequestPattern = patterns orelse &.{.{}};
+
+ var count: usize = 0;
+ for (source) |pattern| {
+ if (pattern.requestStage == .Response) {
+ has_response_stage = true;
+ continue;
+ }
+ count += 1;
+ }
+
+ if (has_response_stage) {
+ log.warn(.not_implemented, "Fetch.enable", .{ .params = "requestStage=Response" });
+ }
+
+ const owned = try self.allocator.alloc(Pattern, count);
+ var initialized: usize = 0;
+ errdefer {
+ for (owned[0..initialized]) |pattern| {
+ self.allocator.free(pattern.url);
+ }
+ self.allocator.free(owned);
+ }
+ for (source) |pattern| {
+ if (pattern.requestStage != .Request) {
+ continue;
+ }
+ owned[initialized] = .{
+ .url = try self.allocator.dupe(u8, pattern.urlPattern),
+ .resource_type = pattern.resourceType,
+ };
+ initialized += 1;
+ }
+ self.patterns = owned;
+ }
+
+ pub fn clearPatterns(self: *InterceptState) void {
+ for (self.patterns) |pattern| {
+ self.allocator.free(pattern.url);
+ }
+ self.allocator.free(self.patterns);
+ self.patterns = &.{};
+ }
+
+ pub fn matches(self: *const InterceptState, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {
+ for (self.patterns) |pattern| {
+ if (pattern.resource_type) |rt| {
+ if (rt != cdpResourceType(resource_type)) {
+ continue;
+ }
+ }
+ if (wildcardMatch(pattern.url, url)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ fn cdpResourceType(resource_type: HttpClient.Request.ResourceType) ResourceType {
+ return switch (resource_type) {
+ .document => .Document,
+ .xhr => .XHR,
+ .script => .Script,
+ .fetch => .Fetch,
+ .stylesheet => .Stylesheet,
+ .eventsource => .EventSource,
+ .image => .Image,
+ };
+ }
+
+ // RequestPattern.urlPattern semantics: '*' matches zero or more characters,
+ // '?' exactly one, '\' escapes the next character.
+ // https://research.swtch.com/glob
+ fn wildcardMatch(pattern: []const u8, value: []const u8) bool {
+ var value_index: usize = 0;
+ var pattern_index: usize = 0;
+ // Position just past the most recent '*', and the value position it was
+ // matched against; on a mismatch we retry from there with the star
+ // consuming one more character.
+ var star_index: ?usize = null;
+ var star_value_index: usize = 0;
+
+ while (value_index < value.len) {
+ if (pattern_index < pattern.len) {
+ switch (pattern[pattern_index]) {
+ '*' => {
+ pattern_index += 1;
+ star_index = pattern_index;
+ star_value_index = value_index;
+ continue;
+ },
+ '?' => {
+ pattern_index += 1;
+ value_index += 1;
+ continue;
+ },
+ else => |c| {
+ var literal = c;
+ var advance: usize = 1;
+ if (c == '\\' and pattern_index + 1 < pattern.len) {
+ literal = pattern[pattern_index + 1];
+ advance = 2;
+ }
+ if (literal == value[value_index]) {
+ pattern_index += advance;
+ value_index += 1;
+ continue;
+ }
+ },
+ }
+ }
+
+ const star = star_index orelse return false;
+ pattern_index = star;
+ star_value_index += 1;
+ value_index = star_value_index;
+ }
+
+ while (pattern_index < pattern.len and pattern[pattern_index] == '*') {
+ pattern_index += 1;
+ }
+ return pattern_index == pattern.len;
+ }
+};
+
const testing = @import("../testing.zig");
test "cdp.Fetch: interception events belong to the enabling session" {
var ctx = try testing.context();
@@ -515,3 +636,115 @@ test "cdp.Fetch: InterceptState issues a fresh id per pause" {
try testing.expectEqual(7, state.remove(second).?);
try testing.expectEqual(null, state.remove(second));
}
+
+test "cdp.Fetch: wildcardMatch" {
+ // exact, case-sensitive
+ try testing.expect(InterceptState.wildcardMatch("http://h/a/b", "http://h/a/b"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a/b", "http://h/a/bc"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a/b", "http://H/a/b"));
+
+ // '*' = zero or more
+ try testing.expect(InterceptState.wildcardMatch("*", ""));
+ try testing.expect(InterceptState.wildcardMatch("*", "http://h/"));
+ try testing.expect(InterceptState.wildcardMatch("**a", "a"));
+ try testing.expect(InterceptState.wildcardMatch("a*", "a"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("a*", ""));
+ try testing.expect(InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/api/get?pageSize=1"));
+ try testing.expect(InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/api/post"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("*://*/api/*", "http://127.0.0.1:18086/apiary"));
+ try testing.expect(InterceptState.wildcardMatch("*api*", "http://h/api/x"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("*api*", "http://h/index.html"));
+ // backtracking past a greedy star
+ try testing.expect(InterceptState.wildcardMatch("*a*b?c", "xxaxxbYc"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("*a*b?c", "xxaxxbc"));
+
+ // '?' = exactly one
+ try testing.expect(InterceptState.wildcardMatch("http://h/?.json", "http://h/a.json"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/?.json", "http://h/ab.json"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/?.json", "http://h/.json"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("a?", "a"));
+
+ // '\' escapes the next character; a trailing '\' is literal
+ try testing.expect(InterceptState.wildcardMatch("http://h/a\\?b=1", "http://h/a?b=1"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/a\\?b=1", "http://h/aXb=1"));
+ try testing.expect(InterceptState.wildcardMatch("http://h/\\*", "http://h/*"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("http://h/\\*", "http://h/x"));
+ try testing.expect(InterceptState.wildcardMatch("*\\\\", "a\\"));
+ try testing.expect(InterceptState.wildcardMatch("abc\\", "abc\\"));
+ try testing.expectEqual(false, InterceptState.wildcardMatch("abc\\", "abc"));
+}
+
+test "cdp.Fetch: InterceptState patterns" {
+ testing.expectLog(&.{.not_implemented});
+
+ var state = try InterceptState.init(testing.allocator);
+ defer state.deinit();
+
+ // disabled: nothing is paused
+ try testing.expectEqual(false, state.matches("http://h/", .document));
+
+ // omitted patterns = every request
+ try state.setPatterns(null);
+ try testing.expect(state.matches("http://h/", .document));
+ try testing.expect(state.matches("http://h/x.js", .script));
+
+ // an explicit empty list = no request
+ try state.setPatterns(&.{});
+ try testing.expectEqual(false, state.matches("http://h/", .document));
+
+ try state.setPatterns(&.{
+ .{ .urlPattern = "*/api/*", .resourceType = .XHR },
+ .{ .urlPattern = "*.css", .requestStage = .Response },
+ });
+ try testing.expectEqual(1, state.patterns.len);
+ try testing.expect(state.matches("http://h/api/x", .xhr));
+ try testing.expectEqual(false, state.matches("http://h/api/x", .fetch));
+ try testing.expectEqual(false, state.matches("http://h/other", .xhr));
+ // Response-stage entries are dropped
+ try testing.expectEqual(false, state.matches("http://h/a.css", .stylesheet));
+
+ state.clearPatterns();
+ try testing.expectEqual(false, state.matches("http://h/api/x", .xhr));
+}
+
+test "cdp.Fetch: enable compiles patterns, disable clears them" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{
+ .session_id = "SID-P",
+ .target_id = "TID-000000000B".*,
+ });
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Fetch.enable",
+ .params = .{ .patterns = .{
+ .{ .urlPattern = "*/json/*", .resourceType = "Fetch" },
+ .{ .urlPattern = "http://h/exact" },
+ } },
+ .sessionId = "SID-P",
+ });
+ try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-P" });
+ try testing.expectEqual(2, bc.intercept_state.patterns.len);
+ try testing.expect(bc.intercept_state.matches("http://h/json/a", .fetch));
+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/json/a", .xhr));
+ try testing.expect(bc.intercept_state.matches("http://h/exact", .xhr));
+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+
+ // re-enable without patterns = catch-all
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.enable", .sessionId = "SID-P" });
+ try ctx.expectSentResult(null, .{ .id = 2, .session_id = "SID-P" });
+ try testing.expect(bc.intercept_state.matches("http://h/", .document));
+
+ // explicit empty list = pause nothing; rejected when auth handling is on
+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.enable", .params = .{ .patterns = .{} }, .sessionId = "SID-P" });
+ try ctx.expectSentResult(null, .{ .id = 3, .session_id = "SID-P" });
+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+ try ctx.processMessage(.{ .id = 4, .method = "Fetch.enable", .params = .{ .patterns = .{}, .handleAuthRequests = true }, .sessionId = "SID-P" });
+ try ctx.expectSentError(-32602, "Can't specify empty patterns with handleAuth set", .{ .id = 4 });
+
+ try ctx.processMessage(.{ .id = 5, .method = "Fetch.disable", .sessionId = "SID-P" });
+ try testing.expectEqual(0, bc.intercept_state.patterns.len);
+ try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+}