Lightpanda: the headless browser designed for AI and automation
Fix Playwright CDP session interception routing
Scott Taylorcaaf53dc9f7c
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex 2ed72c51b..603124f58 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -440,15 +440,28 @@ fn dispatchCommand(command: *Command, method: []const u8) !void {return error.UnknownDomain;}-fn isValidSessionId(self: *const CDP, input_session_id: []const u8) bool {+pub fn resolveSessionId(self: *const CDP, input_session_id: []const u8) ?[]const u8 {if (self.browser_session_id) |browser_session_id| {if (std.mem.eql(u8, browser_session_id, input_session_id)) {- return true;+ return browser_session_id;+ }+ }+ const browser_context = &(self.browser_context orelse return null);+ if (browser_context.session_id) |session_id| {+ if (std.mem.eql(u8, session_id, input_session_id)) {+ return session_id;}}- const browser_context = &(self.browser_context orelse return false);- const session_id = browser_context.session_id orelse return false;- return std.mem.eql(u8, session_id, input_session_id);+ for (browser_context.attached_sessions.items) |session| {+ if (std.mem.eql(u8, session.id, input_session_id)) {+ return session.id;+ }+ }+ return null;+}++fn isValidSessionId(self: *const CDP, input_session_id: []const u8) bool {+ return self.resolveSessionId(input_session_id) != null;}pub fn createBrowserContext(self: *CDP) ![]const u8 {@@ -508,6 +521,10 @@ pub const BrowserContext = struct {id: u32,};+ pub const AttachedSession = struct {+ id: []const u8,+ parent_id: ?[]const u8,+ };id: []const u8,cdp: *CDP,@@ -550,6 +567,7 @@ pub const BrowserContext = struct {// if we get a request with a sessionId that doesn't match the current one// we should reject it.session_id: ?[]const u8,+ attached_sessions: std.ArrayList(AttachedSession) = .empty,security_origin: []const u8,page_life_cycle_events: bool,@@ -572,6 +590,7 @@ pub const BrowserContext = struct {extra_headers: std.ArrayList(http.Header) = .empty,intercept_state: InterceptState,+ fetch_session_id: ?[]const u8 = null,// When network is enabled, we'll capture the transfer.id -> body// This is awfully memory intensive, but our underlying http client and@@ -820,17 +839,26 @@ pub const BrowserContext = struct {self.notification.unregister(.http_request_served_from_cache, self);}- pub fn fetchEnable(self: *BrowserContext, authRequests: bool) !void {+ pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {self.fetchDisable(); //in case of multiple calls+ self.fetch_session_id = session_id;try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);if (authRequests) {try self.notification.register(.http_request_auth_required, self, onHttpRequestAuthRequired);}}+ pub fn fetchDisableForSession(self: *BrowserContext, session_id: []const u8) void {+ const active_session_id = self.fetch_session_id orelse return;+ if (std.mem.eql(u8, active_session_id, session_id)) {+ self.fetchDisable();+ }+ }+pub fn fetchDisable(self: *BrowserContext) void {self.notification.unregister(.http_request_intercept, self);self.notification.unregister(.http_request_auth_required, self);+ self.fetch_session_id = null;}pub fn lifecycleEventsEnable(self: *BrowserContext) !void {diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex c32bf0d03..5d09752b5 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -143,9 +143,16 @@ const ErrorReason = enum {BlockedByResponse,};+fn commandSessionId(cmd: *CDP.Command, bc: *CDP.BrowserContext) ![]const u8 {+ if (cmd.input.session_id) |session_id| {+ return cmd.cdp.resolveSessionId(session_id) orelse error.UnknownSessionId;+ }+ return bc.session_id orelse error.UnknownSessionId;+}+fn disable(cmd: *CDP.Command) !void {const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;- bc.fetchDisable();+ bc.fetchDisableForSession(try commandSessionId(cmd, bc));return cmd.sendResult(null, .{});}@@ -157,7 +164,7 @@ fn enable(cmd: *CDP.Command) !void {}const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;- try bc.fetchEnable(params.handleAuthRequests);+ try bc.fetchEnable(params.handleAuthRequests, try commandSessionId(cmd, bc));return cmd.sendResult(null, .{});}@@ -189,9 +196,8 @@ fn arePatternsSupported(patterns: []RequestPattern) bool {}pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {- // detachTarget could be called, in which case, we still have a frame doing- // things, but no session.- const session_id = bc.session_id orelse return;+ // The session that enabled Fetch owns its interception events.+ const session_id = bc.fetch_session_id orelse return;// We keep it around to wait for modifications to the request.// TODO: What to do when receiving replies for a previous frame's requests?@@ -413,9 +419,7 @@ fn failRequest(cmd: *CDP.Command) !void {}pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {- // detachTarget could be called, in which case, we still have a frame doing- // things, but no session.- const session_id = bc.session_id orelse return;+ const session_id = bc.fetch_session_id orelse return;// We keep it around to wait for modifications to the request.// NOTE: we assume whomever created the request created it with a lifetime of the Page.@@ -459,3 +463,40 @@ fn idFromRequestId(request_id: []const u8) !u32 {}return std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;}++const testing = @import("../testing.zig");++test "cdp.Fetch: interception events belong to the enabling session" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{+ .session_id = "SID-PRIMARY",+ .target_id = "TID-000000000B".*,+ });+ try bc.attached_sessions.append(bc.arena, .{+ .id = "SID-AUX",+ .parent_id = null,+ });++ try ctx.processMessage(.{+ .id = 1,+ .method = "Fetch.enable",+ .sessionId = "SID-AUX",+ });+ try testing.expect(std.mem.eql(u8, "SID-AUX", bc.fetch_session_id.?));+ try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-AUX" });++ try ctx.processMessage(.{+ .id = 2,+ .method = "Fetch.disable",+ .sessionId = "SID-PRIMARY",+ });+ try testing.expect(std.mem.eql(u8, "SID-AUX", bc.fetch_session_id.?));++ try ctx.processMessage(.{+ .id = 3,+ .method = "Fetch.disable",+ .sessionId = "SID-AUX",+ });+ try testing.expectEqual(null, bc.fetch_session_id);+}diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zigindex 703cd3d5f..ac976b9ce 100644--- a/src/cdp/domains/target.zig+++ b/src/cdp/domains/target.zig@@ -246,9 +246,27 @@ fn attachToTarget(cmd: *CDP.Command) !void {return error.UnknownTargetId;}- try doAttachtoTarget(cmd, target_id);+ const parent_id = if (cmd.input.session_id) |session_id|+ cmd.cdp.resolveSessionId(session_id) orelse return error.UnknownSessionId+ else+ null;+ const session_id = try bc.arena.dupe(u8, cmd.cdp.session_id_gen.next());+ try bc.attached_sessions.append(bc.arena, .{+ .id = session_id,+ .parent_id = parent_id,+ });++ try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{+ .sessionId = session_id,+ .targetInfo = TargetInfo{+ .targetId = target_id,+ .title = bc.getTitle() orelse "",+ .url = bc.getURL() orelse "about:blank",+ .browserContextId = bc.id,+ },+ }, .{ .session_id = parent_id });- return cmd.sendResult(.{ .sessionId = bc.session_id }, .{});+ return cmd.sendResult(.{ .sessionId = session_id }, .{});}fn attachToBrowserTarget(cmd: *CDP.Command) !void {@@ -290,8 +308,22 @@ fn closeTarget(cmd: *CDP.Command) !void {try cmd.sendResult(.{ .success = true }, .{ .include_session_id = false });+ for (bc.attached_sessions.items) |session| {+ bc.fetchDisableForSession(session.id);+ try cmd.sendEvent("Inspector.detached", .{+ .reason = "Render process gone.",+ }, .{ .session_id = session.id });+ try cmd.sendEvent("Target.detachedFromTarget", .{+ .targetId = target_id,+ .sessionId = session.id,+ .reason = "Render process gone.",+ }, .{ .session_id = session.parent_id });+ }+ bc.attached_sessions.clearRetainingCapacity();+// could be null, created but never attachedif (bc.session_id) |session_id| {+ bc.fetchDisableForSession(session_id);// Inspector.detached eventtry cmd.sendEvent("Inspector.detached", .{.reason = "Render process gone.",@@ -386,8 +418,33 @@ fn sendMessageToTarget(cmd: *CDP.Command) !void {}fn detachFromTarget(cmd: *CDP.Command) !void {+ const Params = struct {+ sessionId: ?[]const u8 = null,+ targetId: ?[]const u8 = null,+ };+ const params = (try cmd.params(Params)) orelse Params{};+if (cmd.browser_context) |bc| {+ if (params.sessionId) |requested_session_id| {+ for (bc.attached_sessions.items, 0..) |session, index| {+ if (!std.mem.eql(u8, session.id, requested_session_id)) continue;++ _ = bc.attached_sessions.orderedRemove(index);+ bc.fetchDisableForSession(session.id);+ try cmd.sendEvent("Target.detachedFromTarget", .{+ .sessionId = session.id,+ }, .{ .session_id = session.parent_id });+ return cmd.sendResult(null, .{});+ }++ const session_id = bc.session_id orelse return error.UnknownSessionId;+ if (!std.mem.eql(u8, session_id, requested_session_id)) {+ return error.UnknownSessionId;+ }+ }+if (bc.session_id) |session_id| {+ bc.fetchDisableForSession(session_id);try cmd.sendEvent("Target.detachedFromTarget", .{.sessionId = session_id,}, .{});@@ -418,6 +475,7 @@ fn setAutoAttach(cmd: *CDP.Command) !void {// detach from all currently attached targets.if (cmd.browser_context) |bc| {if (bc.session_id) |session_id| {+ bc.fetchDisableForSession(session_id);try cmd.sendEvent("Target.detachedFromTarget", .{.sessionId = session_id,}, .{});@@ -465,7 +523,9 @@ fn setAutoAttach(cmd: *CDP.Command) !void {fn doAttachtoTarget(cmd: *CDP.Command, target_id: []const u8) !void {const bc = cmd.browser_context.?;- const session_id = bc.session_id orelse cmd.cdp.session_id_gen.next();+ const session_id = bc.session_id orelse blk: {+ break :blk try bc.arena.dupe(u8, cmd.cdp.session_id_gen.next());+ };if (bc.session_id == null) {// extra_headers should not be kept on a new frame or tab,@@ -747,12 +807,50 @@ test "cdp.target: attachToTarget" {{try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });- const session_id = bc.session_id.?;+ const session_id = bc.attached_sessions.items[0].id;try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });try ctx.expectSentEvent("Target.attachedToTarget", .{ .sessionId = session_id, .targetInfo = .{ .url = "about:blank", .title = "", .attached = true, .type = "page", .canAccessOpener = false, .browserContextId = "BID-9", .targetId = bc.target_id.? } }, .{});+ try testing.expectEqual(null, bc.session_id);}}+test "cdp.target: auxiliary session is unique and routed through its parent" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{+ .id = "BID-9",+ .session_id = "SID-PRIMARY",+ .target_id = "TID-000000000B".*,+ });++ try ctx.processMessage(.{ .id = 1, .method = "Target.attachToBrowserTarget" });+ try ctx.processMessage(.{+ .id = 2,+ .method = "Target.attachToTarget",+ .sessionId = "BSID-1",+ .params = .{ .targetId = "TID-000000000B" },+ });++ const session_id = bc.attached_sessions.items[0].id;+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));+ try ctx.expectSentEvent("Target.attachedToTarget", .{+ .sessionId = session_id,+ .targetInfo = .{+ .url = "about:blank",+ .title = "",+ .attached = true,+ .type = "page",+ .canAccessOpener = false,+ .browserContextId = "BID-9",+ .targetId = "TID-000000000B",+ },+ }, .{ .index = 2, .session_id = "BSID-1" });+ try ctx.expectSentResult(.{ .sessionId = session_id }, .{+ .id = 2,+ .session_id = "BSID-1",+ });+}+test "cdp.target: getTargetInfo" {var ctx = try testing.context();defer ctx.deinit();@@ -814,7 +912,7 @@ test "cdp.target: issue#474: attach to just created target" {try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- const session_id = bc.session_id.?;+ const session_id = bc.attached_sessions.items[0].id;try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });}}@@ -823,23 +921,19 @@ test "cdp.target: detachFromTarget" {var ctx = try testing.context();defer ctx.deinit();const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });- {- try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });- try testing.expectEqual(true, bc.target_id != null);- try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });+ try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });+ try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });+ const session_id = bc.attached_sessions.items[0].id;- try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- const session_id = bc.session_id.?;- try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });+ try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });- try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .targetId = bc.target_id.? } });- try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});- try testing.expectEqual(null, bc.session_id);- try ctx.expectSentResult(null, .{ .id = 12 });+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});+ try testing.expectEqual(0, bc.attached_sessions.items.len);+ try ctx.expectSentResult(null, .{ .id = 12 });- try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- try ctx.expectSentResult(.{ .sessionId = bc.session_id.? }, .{ .id = 13 });- }+ try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });+ try testing.expect(!std.mem.eql(u8, session_id, bc.attached_sessions.items[0].id));+ try ctx.expectSentResult(.{ .sessionId = bc.attached_sessions.items[0].id }, .{ .id = 13 });}test "cdp.target: detachFromTarget without session" {@@ -858,19 +952,13 @@ test "cdp.target: setAutoAttach false sends detachedFromTarget" {var ctx = try testing.context();defer ctx.deinit();const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });- {- try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });- try testing.expectEqual(true, bc.target_id != null);- try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });+ try ctx.processMessage(.{ .id = 10, .method = "Target.setAutoAttach", .params = .{ .autoAttach = true, .waitForDebuggerOnStart = false } });+ try ctx.processMessage(.{ .id = 11, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });+ const session_id = bc.session_id.?;- try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- const session_id = bc.session_id.?;- try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });+ try ctx.processMessage(.{ .id = 12, .method = "Target.setAutoAttach", .params = .{ .autoAttach = false, .waitForDebuggerOnStart = false } });- // setAutoAttach false should fire detachedFromTarget event- try ctx.processMessage(.{ .id = 12, .method = "Target.setAutoAttach", .params = .{ .autoAttach = false, .waitForDebuggerOnStart = false } });- try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});- try testing.expectEqual(null, bc.session_id);- try ctx.expectSentResult(null, .{ .id = 12 });- }+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});+ try testing.expectEqual(null, bc.session_id);+ try ctx.expectSentResult(null, .{ .id = 12 });}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 32155136b..dcca1bddd 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1453,6 +1453,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {try transfer.handleRedirect(location.value);+ transfer.restoreInterceptHeaders();if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {log.warn(.http, "blocked url", .{ .url = transfer.req.url });@@ -1464,6 +1465,23 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (!transfer.req.internal) lp.metrics.http_redirects.incr();+ if (self.serve_mode) {+ var wait_for_interception = false;+ transfer.req.notification.dispatch(.http_request_intercept, &.{+ .transfer = transfer,+ .wait_for_interception = &wait_for_interception,+ });+ if (wait_for_interception) {+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.reset();+ transfer.state = .created;+ self.intercepted += 1;+ transfer.park(.intercept_request);+ return false;+ }+ }+const conn = transfer._conn.?;try self.handles.remove(conn);@@ -1913,6 +1931,9 @@ pub const Transfer = struct {_tries: u8 = 0,_redirect_count: u8 = 0,+ // Fetch.continueRequest header overrides apply to one network hop.+ _intercept_original_headers: ?[]const RequestHeader = null,+// Linked into client.pending_queue while .queued; reused to link the// retired transfer into client.graveyard (deinit unlinks it from the// pending queue first, so the node is always free by then).@@ -2510,6 +2531,13 @@ pub const Transfer = struct {}}+ fn restoreInterceptHeaders(self: *Transfer) void {+ const headers = self._intercept_original_headers orelse return;+ self.req_headers.clearRetainingCapacity();+ self.req_headers.appendSliceAssumeCapacity(headers);+ self._intercept_original_headers = null;+ }+pub fn reset(self: *Transfer) void {// Note: do NOT reset _auth_challenge or _redirect_count here. They// span retries — _auth_challenge tells makeRequest whether to use@@ -2709,6 +2737,8 @@ pub const Transfer = struct {// CDP Fetch.continueRequest: the intercepting client supplies the// complete header set, replacing whatever the request carried.pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {+ lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });+ self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);self.req_headers.clearRetainingCapacity();try self.seedHeaders();for (headers) |hdr| {@@ -3140,6 +3170,29 @@ test "HttpClient: isFetchInterceptionMethod rejects unrelated methods" {try testing.expect(!isFetchInterceptionMethod("Fetch.continueRequest "));}+test "HttpClient: Fetch header overrides restore after one hop" {+ const original = [_]Transfer.RequestHeader{+ .{ .name = "User-Agent", .value = "original" },+ .{ .name = "X-Original", .value = "yes" },+ };+ var overridden = [_]Transfer.RequestHeader{+ .{ .name = "User-Agent", .value = "override" },+ .{ .name = "X-Override", .value = "yes" },+ };++ var transfer: Transfer = undefined;+ transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };+ transfer._intercept_original_headers = &original;+ transfer.restoreInterceptHeaders();++ try testing.expectEqual(2, transfer.req_headers.items.len);+ try testing.expectString("User-Agent", transfer.req_headers.items[0].name);+ try testing.expectString("original", transfer.req_headers.items[0].value);+ try testing.expectString("X-Original", transfer.req_headers.items[1].name);+ try testing.expectString("yes", transfer.req_headers.items[1].value);+ try testing.expectEqual(null, transfer._intercept_original_headers);+}+test "HttpClient: allowDuringSyncWait allows ping/close/disconnect" {const test_arena = try testing.test_app.arena_pool.acquire(.tiny, "HttpClient test");defer test_arena.release();