Lightpanda: the headless browser designed for AI and automation
split PR, fix non-playwright drivers
Karl Seguinf9071d1cf859
diff --git a/.github/scripts/playwright-cdp-redirect.mjs b/.github/scripts/playwright-cdp-redirect.mjsdeleted file mode 100644index 7e3e76edb..000000000--- a/.github/scripts/playwright-cdp-redirect.mjs+++ /dev/null@@ -1,140 +0,0 @@-import assert from "node:assert/strict";-import { spawn } from "node:child_process";-import { createServer as createHttpServer } from "node:http";-import { createServer as createTcpServer } from "node:net";-import { once } from "node:events";--import { chromium } from "playwright-core";--const binary = process.argv[2];-if (!binary) throw new Error("usage: node playwright-cdp-redirect.mjs <lightpanda-binary>");--function listen(server) {- return new Promise((resolve, reject) => {- server.once("error", reject);- server.listen(0, "127.0.0.1", () => {- server.off("error", reject);- resolve(server.address().port);- });- });-}--async function reservePort() {- const server = createTcpServer();- const port = await listen(server);- await new Promise((resolve) => server.close(resolve));- return port;-}--function closeServer(server) {- return new Promise((resolve) => server.close(resolve));-}--async function waitForLightpanda(child, port) {- await new Promise((resolve, reject) => {- const timeout = setTimeout(() => reject(new Error("Lightpanda startup timeout")), 5_000);- const onData = (chunk) => {- const line = chunk.toString();- if (!line.includes("server running") || !line.includes(`127.0.0.1:${port}`)) return;- clearTimeout(timeout);- child.off("error", onError);- child.off("exit", onExit);- resolve();- };- const onError = (error) => {- clearTimeout(timeout);- reject(error);- };- const onExit = (code) => {- clearTimeout(timeout);- reject(new Error(`Lightpanda exited during startup (${code})`));- };- child.stdout.on("data", onData);- child.stderr.on("data", onData);- child.once("error", onError);- child.once("exit", onExit);- });-}--const destinationRequests = [];-const destination = createHttpServer((request, response) => {- destinationRequests.push(request.headers);- response.end("ok");-});-const destinationPort = await listen(destination);-const destinationUrl = `http://127.0.0.1:${destinationPort}/destination`;--const initialRequests = [];-const initial = createHttpServer((request, response) => {- initialRequests.push(request.headers);- response.writeHead(302, { location: destinationUrl });- response.end();-});-const initialPort = await listen(initial);-const initialUrl = `http://127.0.0.1:${initialPort}/start`;--const cdpPort = await reservePort();-const child = spawn(binary, [- "serve",- "--host",- "127.0.0.1",- "--port",- String(cdpPort),- "--log-level",- "info",-], {- env: { ...process.env, LIGHTPANDA_DISABLE_TELEMETRY: "true" },-});--let browser;-try {- await waitForLightpanda(child, cdpPort);- browser = await chromium.connectOverCDP(`ws://127.0.0.1:${cdpPort}`);- const context = await browser.newContext();- const page = await context.newPage();- const session = await context.newCDPSession(page);- const pausedUrls = [];- const continuationErrors = [];-- session.on("Fetch.requestPaused", async (event) => {- if (event.request.url !== initialUrl && event.request.url !== destinationUrl) return;- pausedUrls.push(event.request.url);- const params = { requestId: event.requestId };- if (event.request.url === initialUrl) {- params.headers = Object.entries({- ...event.request.headers,- "x-single-hop-probe": "initial",- }).map(([name, value]) => ({ name, value: String(value) }));- }- try {- await session.send("Fetch.continueRequest", params);- } catch (error) {- continuationErrors.push(error);- }- });-- await session.send("Fetch.enable", {- patterns: [{ urlPattern: "*", requestStage: "Request" }],- });- const response = await page.goto(initialUrl, {- waitUntil: "domcontentloaded",- timeout: 6_000,- });-- assert.equal(response.status(), 200);- assert.deepEqual(pausedUrls, [initialUrl, destinationUrl]);- assert.equal(continuationErrors.length, 0);- assert.equal(initialRequests.length, 1);- assert.equal(initialRequests[0]["x-single-hop-probe"], "initial");- assert.equal(destinationRequests.length, 1);- assert.equal(destinationRequests[0]["x-single-hop-probe"], undefined);-- console.log("Playwright auxiliary CDP redirect interception passed");-} finally {- await browser?.close().catch(() => {});- if (child.exitCode === null && child.signalCode === null) {- child.kill("SIGTERM");- await Promise.race([once(child, "exit"), new Promise((resolve) => setTimeout(resolve, 2_000))]);- }- await Promise.all([closeServer(initial), closeServer(destination)]);-}diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex 99f68a23d..246e760d4 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -189,29 +189,6 @@ jobs:kill `cat LPD.pid`while kill -0 `cat LPD.pid` 2>/dev/null; do sleep 1; done- playwright-cdp-session:- name: playwright-cdp-session- needs:- - zig-build-release-- runs-on: ubuntu-latest- timeout-minutes: 5-- steps:- - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3-- - run: npm install --no-save --no-package-lock [email protected]-- - name: download lightpanda release- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1- with:- name: lightpanda-build-release-- - run: chmod a+x ./lightpanda-- - name: run auxiliary CDP redirect interception- run: node .github/scripts/playwright-cdp-redirect.mjs ./lightpanda-wba-test:name: wba-testneeds: zig-build-releasediff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zigindex ac976b9ce..e764e9c7f 100644--- a/src/cdp/domains/target.zig+++ b/src/cdp/domains/target.zig@@ -246,6 +246,16 @@ fn attachToTarget(cmd: *CDP.Command) !void {return error.UnknownTargetId;}+ // The first attach becomes the page's primary session, which all+ // Page/Network/... events are emitted to. Later attaches — e.g.+ // Playwright's browserContext.newCDPSession(page), sent through its+ // browser session while the page session is live — get a distinct+ // auxiliary session so they don't collide with the primary one.+ if (bc.session_id == null) {+ try doAttachtoTarget(cmd, target_id);+ return cmd.sendResult(.{ .sessionId = bc.session_id }, .{});+ }+const parent_id = if (cmd.input.session_id) |session_id|cmd.cdp.resolveSessionId(session_id) orelse return error.UnknownSessionIdelse@@ -806,11 +816,12 @@ test "cdp.target: attachToTarget" {}{+ // the first attach becomes the primary sessiontry ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });- const session_id = bc.attached_sessions.items[0].id;+ const session_id = bc.session_id.?;try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });try ctx.expectSentEvent("Target.attachedToTarget", .{ .sessionId = session_id, .targetInfo = .{ .url = "about:blank", .title = "", .attached = true, .type = "page", .canAccessOpener = false, .browserContextId = "BID-9", .targetId = bc.target_id.? } }, .{});- try testing.expectEqual(null, bc.session_id);+ try testing.expectEqual(0, bc.attached_sessions.items.len);}}@@ -912,7 +923,7 @@ test "cdp.target: issue#474: attach to just created target" {try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- const session_id = bc.attached_sessions.items[0].id;+ const session_id = bc.session_id.?;try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });}}@@ -923,17 +934,37 @@ test "cdp.target: detachFromTarget" {const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- const session_id = bc.attached_sessions.items[0].id;+ const session_id = bc.session_id.?;try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});- try testing.expectEqual(0, bc.attached_sessions.items.len);+ try testing.expectEqual(null, bc.session_id);try ctx.expectSentResult(null, .{ .id = 12 });try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });- try testing.expect(!std.mem.eql(u8, session_id, bc.attached_sessions.items[0].id));- try ctx.expectSentResult(.{ .sessionId = bc.attached_sessions.items[0].id }, .{ .id = 13 });+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));+ try ctx.expectSentResult(.{ .sessionId = bc.session_id.? }, .{ .id = 13 });+}++test "cdp.target: detachFromTarget auxiliary session" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{+ .id = "BID-9",+ .session_id = "SID-PRIMARY",+ .target_id = "TID-000000000B".*,+ });++ try ctx.processMessage(.{ .id = 10, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });+ const session_id = bc.attached_sessions.items[0].id;+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));++ try ctx.processMessage(.{ .id = 11, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});+ try testing.expectEqual(0, bc.attached_sessions.items.len);+ try testing.expectEqual(true, bc.session_id != null);+ try ctx.expectSentResult(null, .{ .id = 11 });}test "cdp.target: detachFromTarget without session" {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex dcca1bddd..32155136b 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1453,7 +1453,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {try transfer.handleRedirect(location.value);- transfer.restoreInterceptHeaders();if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {log.warn(.http, "blocked url", .{ .url = transfer.req.url });@@ -1465,23 +1464,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (!transfer.req.internal) lp.metrics.http_redirects.incr();- if (self.serve_mode) {- var wait_for_interception = false;- transfer.req.notification.dispatch(.http_request_intercept, &.{- .transfer = transfer,- .wait_for_interception = &wait_for_interception,- });- if (wait_for_interception) {- self.removeConn(msg.conn);- transfer._conn = null;- transfer.reset();- transfer.state = .created;- self.intercepted += 1;- transfer.park(.intercept_request);- return false;- }- }-const conn = transfer._conn.?;try self.handles.remove(conn);@@ -1931,9 +1913,6 @@ pub const Transfer = struct {_tries: u8 = 0,_redirect_count: u8 = 0,- // Fetch.continueRequest header overrides apply to one network hop.- _intercept_original_headers: ?[]const RequestHeader = null,-// Linked into client.pending_queue while .queued; reused to link the// retired transfer into client.graveyard (deinit unlinks it from the// pending queue first, so the node is always free by then).@@ -2531,13 +2510,6 @@ pub const Transfer = struct {}}- fn restoreInterceptHeaders(self: *Transfer) void {- const headers = self._intercept_original_headers orelse return;- self.req_headers.clearRetainingCapacity();- self.req_headers.appendSliceAssumeCapacity(headers);- self._intercept_original_headers = null;- }-pub fn reset(self: *Transfer) void {// Note: do NOT reset _auth_challenge or _redirect_count here. They// span retries — _auth_challenge tells makeRequest whether to use@@ -2737,8 +2709,6 @@ pub const Transfer = struct {// CDP Fetch.continueRequest: the intercepting client supplies the// complete header set, replacing whatever the request carried.pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {- lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });- self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);self.req_headers.clearRetainingCapacity();try self.seedHeaders();for (headers) |hdr| {@@ -3170,29 +3140,6 @@ test "HttpClient: isFetchInterceptionMethod rejects unrelated methods" {try testing.expect(!isFetchInterceptionMethod("Fetch.continueRequest "));}-test "HttpClient: Fetch header overrides restore after one hop" {- const original = [_]Transfer.RequestHeader{- .{ .name = "User-Agent", .value = "original" },- .{ .name = "X-Original", .value = "yes" },- };- var overridden = [_]Transfer.RequestHeader{- .{ .name = "User-Agent", .value = "override" },- .{ .name = "X-Override", .value = "yes" },- };-- var transfer: Transfer = undefined;- transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };- transfer._intercept_original_headers = &original;- transfer.restoreInterceptHeaders();-- try testing.expectEqual(2, transfer.req_headers.items.len);- try testing.expectString("User-Agent", transfer.req_headers.items[0].name);- try testing.expectString("original", transfer.req_headers.items[0].value);- try testing.expectString("X-Original", transfer.req_headers.items[1].name);- try testing.expectString("yes", transfer.req_headers.items[1].value);- try testing.expectEqual(null, transfer._intercept_original_headers);-}-test "HttpClient: allowDuringSyncWait allows ping/close/disconnect" {const test_arena = try testing.test_app.arena_pool.acquire(.tiny, "HttpClient test");defer test_arena.release();