Skip to content

Lightpanda: the headless browser designed for AI and automation

Fix Playwright CDP session interception routing

Scott Taylorcaaf53dc9f7c
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index 2ed72c51b..603124f58 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -440,15 +440,28 @@ fn dispatchCommand(command: *Command, method: []const u8) !void {
return error.UnknownDomain;
}
-fn isValidSessionId(self: *const CDP, input_session_id: []const u8) bool {
+pub fn resolveSessionId(self: *const CDP, input_session_id: []const u8) ?[]const u8 {
if (self.browser_session_id) |browser_session_id| {
if (std.mem.eql(u8, browser_session_id, input_session_id)) {
- return true;
+ return browser_session_id;
+ }
+ }
+ const browser_context = &(self.browser_context orelse return null);
+ if (browser_context.session_id) |session_id| {
+ if (std.mem.eql(u8, session_id, input_session_id)) {
+ return session_id;
}
}
- const browser_context = &(self.browser_context orelse return false);
- const session_id = browser_context.session_id orelse return false;
- return std.mem.eql(u8, session_id, input_session_id);
+ for (browser_context.attached_sessions.items) |session| {
+ if (std.mem.eql(u8, session.id, input_session_id)) {
+ return session.id;
+ }
+ }
+ return null;
+}
+
+fn isValidSessionId(self: *const CDP, input_session_id: []const u8) bool {
+ return self.resolveSessionId(input_session_id) != null;
}
pub fn createBrowserContext(self: *CDP) ![]const u8 {
@@ -508,6 +521,10 @@ pub const BrowserContext = struct {
id: u32,
};
+ pub const AttachedSession = struct {
+ id: []const u8,
+ parent_id: ?[]const u8,
+ };
id: []const u8,
cdp: *CDP,
@@ -550,6 +567,7 @@ pub const BrowserContext = struct {
// if we get a request with a sessionId that doesn't match the current one
// we should reject it.
session_id: ?[]const u8,
+ attached_sessions: std.ArrayList(AttachedSession) = .empty,
security_origin: []const u8,
page_life_cycle_events: bool,
@@ -572,6 +590,7 @@ pub const BrowserContext = struct {
extra_headers: std.ArrayList(http.Header) = .empty,
intercept_state: InterceptState,
+ fetch_session_id: ?[]const u8 = null,
// When network is enabled, we'll capture the transfer.id -> body
// This is awfully memory intensive, but our underlying http client and
@@ -820,17 +839,26 @@ pub const BrowserContext = struct {
self.notification.unregister(.http_request_served_from_cache, self);
}
- pub fn fetchEnable(self: *BrowserContext, authRequests: bool) !void {
+ pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {
self.fetchDisable(); //in case of multiple calls
+ self.fetch_session_id = session_id;
try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);
if (authRequests) {
try self.notification.register(.http_request_auth_required, self, onHttpRequestAuthRequired);
}
}
+ pub fn fetchDisableForSession(self: *BrowserContext, session_id: []const u8) void {
+ const active_session_id = self.fetch_session_id orelse return;
+ if (std.mem.eql(u8, active_session_id, session_id)) {
+ self.fetchDisable();
+ }
+ }
+
pub fn fetchDisable(self: *BrowserContext) void {
self.notification.unregister(.http_request_intercept, self);
self.notification.unregister(.http_request_auth_required, self);
+ self.fetch_session_id = null;
}
pub fn lifecycleEventsEnable(self: *BrowserContext) !void {
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index c32bf0d03..5d09752b5 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -143,9 +143,16 @@ const ErrorReason = enum {
BlockedByResponse,
};
+fn commandSessionId(cmd: *CDP.Command, bc: *CDP.BrowserContext) ![]const u8 {
+ if (cmd.input.session_id) |session_id| {
+ return cmd.cdp.resolveSessionId(session_id) orelse error.UnknownSessionId;
+ }
+ return bc.session_id orelse error.UnknownSessionId;
+}
+
fn disable(cmd: *CDP.Command) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
- bc.fetchDisable();
+ bc.fetchDisableForSession(try commandSessionId(cmd, bc));
return cmd.sendResult(null, .{});
}
@@ -157,7 +164,7 @@ fn enable(cmd: *CDP.Command) !void {
}
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
- try bc.fetchEnable(params.handleAuthRequests);
+ try bc.fetchEnable(params.handleAuthRequests, try commandSessionId(cmd, bc));
return cmd.sendResult(null, .{});
}
@@ -189,9 +196,8 @@ fn arePatternsSupported(patterns: []RequestPattern) bool {
}
pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {
- // detachTarget could be called, in which case, we still have a frame doing
- // things, but no session.
- const session_id = bc.session_id orelse return;
+ // The session that enabled Fetch owns its interception events.
+ const session_id = bc.fetch_session_id orelse return;
// We keep it around to wait for modifications to the request.
// TODO: What to do when receiving replies for a previous frame's requests?
@@ -413,9 +419,7 @@ fn failRequest(cmd: *CDP.Command) !void {
}
pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {
- // detachTarget could be called, in which case, we still have a frame doing
- // things, but no session.
- const session_id = bc.session_id orelse return;
+ const session_id = bc.fetch_session_id orelse return;
// We keep it around to wait for modifications to the request.
// NOTE: we assume whomever created the request created it with a lifetime of the Page.
@@ -459,3 +463,40 @@ fn idFromRequestId(request_id: []const u8) !u32 {
}
return std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;
}
+
+const testing = @import("../testing.zig");
+
+test "cdp.Fetch: interception events belong to the enabling session" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{
+ .session_id = "SID-PRIMARY",
+ .target_id = "TID-000000000B".*,
+ });
+ try bc.attached_sessions.append(bc.arena, .{
+ .id = "SID-AUX",
+ .parent_id = null,
+ });
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Fetch.enable",
+ .sessionId = "SID-AUX",
+ });
+ try testing.expect(std.mem.eql(u8, "SID-AUX", bc.fetch_session_id.?));
+ try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-AUX" });
+
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Fetch.disable",
+ .sessionId = "SID-PRIMARY",
+ });
+ try testing.expect(std.mem.eql(u8, "SID-AUX", bc.fetch_session_id.?));
+
+ try ctx.processMessage(.{
+ .id = 3,
+ .method = "Fetch.disable",
+ .sessionId = "SID-AUX",
+ });
+ try testing.expectEqual(null, bc.fetch_session_id);
+}
diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig
index 703cd3d5f..ac976b9ce 100644
--- a/src/cdp/domains/target.zig
+++ b/src/cdp/domains/target.zig
@@ -246,9 +246,27 @@ fn attachToTarget(cmd: *CDP.Command) !void {
return error.UnknownTargetId;
}
- try doAttachtoTarget(cmd, target_id);
+ const parent_id = if (cmd.input.session_id) |session_id|
+ cmd.cdp.resolveSessionId(session_id) orelse return error.UnknownSessionId
+ else
+ null;
+ const session_id = try bc.arena.dupe(u8, cmd.cdp.session_id_gen.next());
+ try bc.attached_sessions.append(bc.arena, .{
+ .id = session_id,
+ .parent_id = parent_id,
+ });
+
+ try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{
+ .sessionId = session_id,
+ .targetInfo = TargetInfo{
+ .targetId = target_id,
+ .title = bc.getTitle() orelse "",
+ .url = bc.getURL() orelse "about:blank",
+ .browserContextId = bc.id,
+ },
+ }, .{ .session_id = parent_id });
- return cmd.sendResult(.{ .sessionId = bc.session_id }, .{});
+ return cmd.sendResult(.{ .sessionId = session_id }, .{});
}
fn attachToBrowserTarget(cmd: *CDP.Command) !void {
@@ -290,8 +308,22 @@ fn closeTarget(cmd: *CDP.Command) !void {
try cmd.sendResult(.{ .success = true }, .{ .include_session_id = false });
+ for (bc.attached_sessions.items) |session| {
+ bc.fetchDisableForSession(session.id);
+ try cmd.sendEvent("Inspector.detached", .{
+ .reason = "Render process gone.",
+ }, .{ .session_id = session.id });
+ try cmd.sendEvent("Target.detachedFromTarget", .{
+ .targetId = target_id,
+ .sessionId = session.id,
+ .reason = "Render process gone.",
+ }, .{ .session_id = session.parent_id });
+ }
+ bc.attached_sessions.clearRetainingCapacity();
+
// could be null, created but never attached
if (bc.session_id) |session_id| {
+ bc.fetchDisableForSession(session_id);
// Inspector.detached event
try cmd.sendEvent("Inspector.detached", .{
.reason = "Render process gone.",
@@ -386,8 +418,33 @@ fn sendMessageToTarget(cmd: *CDP.Command) !void {
}
fn detachFromTarget(cmd: *CDP.Command) !void {
+ const Params = struct {
+ sessionId: ?[]const u8 = null,
+ targetId: ?[]const u8 = null,
+ };
+ const params = (try cmd.params(Params)) orelse Params{};
+
if (cmd.browser_context) |bc| {
+ if (params.sessionId) |requested_session_id| {
+ for (bc.attached_sessions.items, 0..) |session, index| {
+ if (!std.mem.eql(u8, session.id, requested_session_id)) continue;
+
+ _ = bc.attached_sessions.orderedRemove(index);
+ bc.fetchDisableForSession(session.id);
+ try cmd.sendEvent("Target.detachedFromTarget", .{
+ .sessionId = session.id,
+ }, .{ .session_id = session.parent_id });
+ return cmd.sendResult(null, .{});
+ }
+
+ const session_id = bc.session_id orelse return error.UnknownSessionId;
+ if (!std.mem.eql(u8, session_id, requested_session_id)) {
+ return error.UnknownSessionId;
+ }
+ }
+
if (bc.session_id) |session_id| {
+ bc.fetchDisableForSession(session_id);
try cmd.sendEvent("Target.detachedFromTarget", .{
.sessionId = session_id,
}, .{});
@@ -418,6 +475,7 @@ fn setAutoAttach(cmd: *CDP.Command) !void {
// detach from all currently attached targets.
if (cmd.browser_context) |bc| {
if (bc.session_id) |session_id| {
+ bc.fetchDisableForSession(session_id);
try cmd.sendEvent("Target.detachedFromTarget", .{
.sessionId = session_id,
}, .{});
@@ -465,7 +523,9 @@ fn setAutoAttach(cmd: *CDP.Command) !void {
fn doAttachtoTarget(cmd: *CDP.Command, target_id: []const u8) !void {
const bc = cmd.browser_context.?;
- const session_id = bc.session_id orelse cmd.cdp.session_id_gen.next();
+ const session_id = bc.session_id orelse blk: {
+ break :blk try bc.arena.dupe(u8, cmd.cdp.session_id_gen.next());
+ };
if (bc.session_id == null) {
// extra_headers should not be kept on a new frame or tab,
@@ -747,12 +807,50 @@ test "cdp.target: attachToTarget" {
{
try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });
- const session_id = bc.session_id.?;
+ const session_id = bc.attached_sessions.items[0].id;
try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
try ctx.expectSentEvent("Target.attachedToTarget", .{ .sessionId = session_id, .targetInfo = .{ .url = "about:blank", .title = "", .attached = true, .type = "page", .canAccessOpener = false, .browserContextId = "BID-9", .targetId = bc.target_id.? } }, .{});
+ try testing.expectEqual(null, bc.session_id);
}
}
+test "cdp.target: auxiliary session is unique and routed through its parent" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{
+ .id = "BID-9",
+ .session_id = "SID-PRIMARY",
+ .target_id = "TID-000000000B".*,
+ });
+
+ try ctx.processMessage(.{ .id = 1, .method = "Target.attachToBrowserTarget" });
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Target.attachToTarget",
+ .sessionId = "BSID-1",
+ .params = .{ .targetId = "TID-000000000B" },
+ });
+
+ const session_id = bc.attached_sessions.items[0].id;
+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));
+ try ctx.expectSentEvent("Target.attachedToTarget", .{
+ .sessionId = session_id,
+ .targetInfo = .{
+ .url = "about:blank",
+ .title = "",
+ .attached = true,
+ .type = "page",
+ .canAccessOpener = false,
+ .browserContextId = "BID-9",
+ .targetId = "TID-000000000B",
+ },
+ }, .{ .index = 2, .session_id = "BSID-1" });
+ try ctx.expectSentResult(.{ .sessionId = session_id }, .{
+ .id = 2,
+ .session_id = "BSID-1",
+ });
+}
+
test "cdp.target: getTargetInfo" {
var ctx = try testing.context();
defer ctx.deinit();
@@ -814,7 +912,7 @@ test "cdp.target: issue#474: attach to just created target" {
try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });
try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- const session_id = bc.session_id.?;
+ const session_id = bc.attached_sessions.items[0].id;
try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
}
}
@@ -823,23 +921,19 @@ test "cdp.target: detachFromTarget" {
var ctx = try testing.context();
defer ctx.deinit();
const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });
- {
- try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
- try testing.expectEqual(true, bc.target_id != null);
- try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });
+ try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
+ try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
+ const session_id = bc.attached_sessions.items[0].id;
- try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- const session_id = bc.session_id.?;
- try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
+ try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });
- try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .targetId = bc.target_id.? } });
- try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
- try testing.expectEqual(null, bc.session_id);
- try ctx.expectSentResult(null, .{ .id = 12 });
+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
+ try testing.expectEqual(0, bc.attached_sessions.items.len);
+ try ctx.expectSentResult(null, .{ .id = 12 });
- try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- try ctx.expectSentResult(.{ .sessionId = bc.session_id.? }, .{ .id = 13 });
- }
+ try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
+ try testing.expect(!std.mem.eql(u8, session_id, bc.attached_sessions.items[0].id));
+ try ctx.expectSentResult(.{ .sessionId = bc.attached_sessions.items[0].id }, .{ .id = 13 });
}
test "cdp.target: detachFromTarget without session" {
@@ -858,19 +952,13 @@ test "cdp.target: setAutoAttach false sends detachedFromTarget" {
var ctx = try testing.context();
defer ctx.deinit();
const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });
- {
- try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
- try testing.expectEqual(true, bc.target_id != null);
- try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });
+ try ctx.processMessage(.{ .id = 10, .method = "Target.setAutoAttach", .params = .{ .autoAttach = true, .waitForDebuggerOnStart = false } });
+ try ctx.processMessage(.{ .id = 11, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
+ const session_id = bc.session_id.?;
- try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- const session_id = bc.session_id.?;
- try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
+ try ctx.processMessage(.{ .id = 12, .method = "Target.setAutoAttach", .params = .{ .autoAttach = false, .waitForDebuggerOnStart = false } });
- // setAutoAttach false should fire detachedFromTarget event
- try ctx.processMessage(.{ .id = 12, .method = "Target.setAutoAttach", .params = .{ .autoAttach = false, .waitForDebuggerOnStart = false } });
- try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
- try testing.expectEqual(null, bc.session_id);
- try ctx.expectSentResult(null, .{ .id = 12 });
- }
+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
+ try testing.expectEqual(null, bc.session_id);
+ try ctx.expectSentResult(null, .{ .id = 12 });
}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 32155136b..dcca1bddd 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1453,6 +1453,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
try transfer.handleRedirect(location.value);
+ transfer.restoreInterceptHeaders();
if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
@@ -1464,6 +1465,23 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (!transfer.req.internal) lp.metrics.http_redirects.incr();
+ if (self.serve_mode) {
+ var wait_for_interception = false;
+ transfer.req.notification.dispatch(.http_request_intercept, &.{
+ .transfer = transfer,
+ .wait_for_interception = &wait_for_interception,
+ });
+ if (wait_for_interception) {
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.reset();
+ transfer.state = .created;
+ self.intercepted += 1;
+ transfer.park(.intercept_request);
+ return false;
+ }
+ }
+
const conn = transfer._conn.?;
try self.handles.remove(conn);
@@ -1913,6 +1931,9 @@ pub const Transfer = struct {
_tries: u8 = 0,
_redirect_count: u8 = 0,
+ // Fetch.continueRequest header overrides apply to one network hop.
+ _intercept_original_headers: ?[]const RequestHeader = null,
+
// Linked into client.pending_queue while .queued; reused to link the
// retired transfer into client.graveyard (deinit unlinks it from the
// pending queue first, so the node is always free by then).
@@ -2510,6 +2531,13 @@ pub const Transfer = struct {
}
}
+ fn restoreInterceptHeaders(self: *Transfer) void {
+ const headers = self._intercept_original_headers orelse return;
+ self.req_headers.clearRetainingCapacity();
+ self.req_headers.appendSliceAssumeCapacity(headers);
+ self._intercept_original_headers = null;
+ }
+
pub fn reset(self: *Transfer) void {
// Note: do NOT reset _auth_challenge or _redirect_count here. They
// span retries — _auth_challenge tells makeRequest whether to use
@@ -2709,6 +2737,8 @@ pub const Transfer = struct {
// CDP Fetch.continueRequest: the intercepting client supplies the
// complete header set, replacing whatever the request carried.
pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {
+ lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });
+ self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);
self.req_headers.clearRetainingCapacity();
try self.seedHeaders();
for (headers) |hdr| {
@@ -3140,6 +3170,29 @@ test "HttpClient: isFetchInterceptionMethod rejects unrelated methods" {
try testing.expect(!isFetchInterceptionMethod("Fetch.continueRequest "));
}
+test "HttpClient: Fetch header overrides restore after one hop" {
+ const original = [_]Transfer.RequestHeader{
+ .{ .name = "User-Agent", .value = "original" },
+ .{ .name = "X-Original", .value = "yes" },
+ };
+ var overridden = [_]Transfer.RequestHeader{
+ .{ .name = "User-Agent", .value = "override" },
+ .{ .name = "X-Override", .value = "yes" },
+ };
+
+ var transfer: Transfer = undefined;
+ transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };
+ transfer._intercept_original_headers = &original;
+ transfer.restoreInterceptHeaders();
+
+ try testing.expectEqual(2, transfer.req_headers.items.len);
+ try testing.expectString("User-Agent", transfer.req_headers.items[0].name);
+ try testing.expectString("original", transfer.req_headers.items[0].value);
+ try testing.expectString("X-Original", transfer.req_headers.items[1].name);
+ try testing.expectString("yes", transfer.req_headers.items[1].value);
+ try testing.expectEqual(null, transfer._intercept_original_headers);
+}
+
test "HttpClient: allowDuringSyncWait allows ping/close/disconnect" {
const test_arena = try testing.test_app.arena_pool.acquire(.tiny, "HttpClient test");
defer test_arena.release();