Skip to content

Lightpanda: the headless browser designed for AI and automation

split PR, fix non-playwright drivers

Karl Seguinf9071d1cf859
diff --git a/.github/scripts/playwright-cdp-redirect.mjs b/.github/scripts/playwright-cdp-redirect.mjs
deleted file mode 100644
index 7e3e76edb..000000000
--- a/.github/scripts/playwright-cdp-redirect.mjs
+++ /dev/null
@@ -1,140 +0,0 @@
-import assert from "node:assert/strict";
-import { spawn } from "node:child_process";
-import { createServer as createHttpServer } from "node:http";
-import { createServer as createTcpServer } from "node:net";
-import { once } from "node:events";
-
-import { chromium } from "playwright-core";
-
-const binary = process.argv[2];
-if (!binary) throw new Error("usage: node playwright-cdp-redirect.mjs <lightpanda-binary>");
-
-function listen(server) {
- return new Promise((resolve, reject) => {
- server.once("error", reject);
- server.listen(0, "127.0.0.1", () => {
- server.off("error", reject);
- resolve(server.address().port);
- });
- });
-}
-
-async function reservePort() {
- const server = createTcpServer();
- const port = await listen(server);
- await new Promise((resolve) => server.close(resolve));
- return port;
-}
-
-function closeServer(server) {
- return new Promise((resolve) => server.close(resolve));
-}
-
-async function waitForLightpanda(child, port) {
- await new Promise((resolve, reject) => {
- const timeout = setTimeout(() => reject(new Error("Lightpanda startup timeout")), 5_000);
- const onData = (chunk) => {
- const line = chunk.toString();
- if (!line.includes("server running") || !line.includes(`127.0.0.1:${port}`)) return;
- clearTimeout(timeout);
- child.off("error", onError);
- child.off("exit", onExit);
- resolve();
- };
- const onError = (error) => {
- clearTimeout(timeout);
- reject(error);
- };
- const onExit = (code) => {
- clearTimeout(timeout);
- reject(new Error(`Lightpanda exited during startup (${code})`));
- };
- child.stdout.on("data", onData);
- child.stderr.on("data", onData);
- child.once("error", onError);
- child.once("exit", onExit);
- });
-}
-
-const destinationRequests = [];
-const destination = createHttpServer((request, response) => {
- destinationRequests.push(request.headers);
- response.end("ok");
-});
-const destinationPort = await listen(destination);
-const destinationUrl = `http://127.0.0.1:${destinationPort}/destination`;
-
-const initialRequests = [];
-const initial = createHttpServer((request, response) => {
- initialRequests.push(request.headers);
- response.writeHead(302, { location: destinationUrl });
- response.end();
-});
-const initialPort = await listen(initial);
-const initialUrl = `http://127.0.0.1:${initialPort}/start`;
-
-const cdpPort = await reservePort();
-const child = spawn(binary, [
- "serve",
- "--host",
- "127.0.0.1",
- "--port",
- String(cdpPort),
- "--log-level",
- "info",
-], {
- env: { ...process.env, LIGHTPANDA_DISABLE_TELEMETRY: "true" },
-});
-
-let browser;
-try {
- await waitForLightpanda(child, cdpPort);
- browser = await chromium.connectOverCDP(`ws://127.0.0.1:${cdpPort}`);
- const context = await browser.newContext();
- const page = await context.newPage();
- const session = await context.newCDPSession(page);
- const pausedUrls = [];
- const continuationErrors = [];
-
- session.on("Fetch.requestPaused", async (event) => {
- if (event.request.url !== initialUrl && event.request.url !== destinationUrl) return;
- pausedUrls.push(event.request.url);
- const params = { requestId: event.requestId };
- if (event.request.url === initialUrl) {
- params.headers = Object.entries({
- ...event.request.headers,
- "x-single-hop-probe": "initial",
- }).map(([name, value]) => ({ name, value: String(value) }));
- }
- try {
- await session.send("Fetch.continueRequest", params);
- } catch (error) {
- continuationErrors.push(error);
- }
- });
-
- await session.send("Fetch.enable", {
- patterns: [{ urlPattern: "*", requestStage: "Request" }],
- });
- const response = await page.goto(initialUrl, {
- waitUntil: "domcontentloaded",
- timeout: 6_000,
- });
-
- assert.equal(response.status(), 200);
- assert.deepEqual(pausedUrls, [initialUrl, destinationUrl]);
- assert.equal(continuationErrors.length, 0);
- assert.equal(initialRequests.length, 1);
- assert.equal(initialRequests[0]["x-single-hop-probe"], "initial");
- assert.equal(destinationRequests.length, 1);
- assert.equal(destinationRequests[0]["x-single-hop-probe"], undefined);
-
- console.log("Playwright auxiliary CDP redirect interception passed");
-} finally {
- await browser?.close().catch(() => {});
- if (child.exitCode === null && child.signalCode === null) {
- child.kill("SIGTERM");
- await Promise.race([once(child, "exit"), new Promise((resolve) => setTimeout(resolve, 2_000))]);
- }
- await Promise.all([closeServer(initial), closeServer(destination)]);
-}
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 99f68a23d..246e760d4 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -189,29 +189,6 @@ jobs:
kill `cat LPD.pid`
while kill -0 `cat LPD.pid` 2>/dev/null; do sleep 1; done
- playwright-cdp-session:
- name: playwright-cdp-session
- needs:
- - zig-build-release
-
- runs-on: ubuntu-latest
- timeout-minutes: 5
-
- steps:
- - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-
- - run: npm install --no-save --no-package-lock [email protected]
-
- - name: download lightpanda release
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: lightpanda-build-release
-
- - run: chmod a+x ./lightpanda
-
- - name: run auxiliary CDP redirect interception
- run: node .github/scripts/playwright-cdp-redirect.mjs ./lightpanda
-
wba-test:
name: wba-test
needs: zig-build-release
diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig
index ac976b9ce..e764e9c7f 100644
--- a/src/cdp/domains/target.zig
+++ b/src/cdp/domains/target.zig
@@ -246,6 +246,16 @@ fn attachToTarget(cmd: *CDP.Command) !void {
return error.UnknownTargetId;
}
+ // The first attach becomes the page's primary session, which all
+ // Page/Network/... events are emitted to. Later attaches — e.g.
+ // Playwright's browserContext.newCDPSession(page), sent through its
+ // browser session while the page session is live — get a distinct
+ // auxiliary session so they don't collide with the primary one.
+ if (bc.session_id == null) {
+ try doAttachtoTarget(cmd, target_id);
+ return cmd.sendResult(.{ .sessionId = bc.session_id }, .{});
+ }
+
const parent_id = if (cmd.input.session_id) |session_id|
cmd.cdp.resolveSessionId(session_id) orelse return error.UnknownSessionId
else
@@ -806,11 +816,12 @@ test "cdp.target: attachToTarget" {
}
{
+ // the first attach becomes the primary session
try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });
- const session_id = bc.attached_sessions.items[0].id;
+ const session_id = bc.session_id.?;
try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
try ctx.expectSentEvent("Target.attachedToTarget", .{ .sessionId = session_id, .targetInfo = .{ .url = "about:blank", .title = "", .attached = true, .type = "page", .canAccessOpener = false, .browserContextId = "BID-9", .targetId = bc.target_id.? } }, .{});
- try testing.expectEqual(null, bc.session_id);
+ try testing.expectEqual(0, bc.attached_sessions.items.len);
}
}
@@ -912,7 +923,7 @@ test "cdp.target: issue#474: attach to just created target" {
try ctx.expectSentResult(.{ .targetId = bc.target_id.? }, .{ .id = 10 });
try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- const session_id = bc.attached_sessions.items[0].id;
+ const session_id = bc.session_id.?;
try ctx.expectSentResult(.{ .sessionId = session_id }, .{ .id = 11 });
}
}
@@ -923,17 +934,37 @@ test "cdp.target: detachFromTarget" {
const bc = try ctx.loadBrowserContext(.{ .id = "BID-9" });
try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
try ctx.processMessage(.{ .id = 11, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- const session_id = bc.attached_sessions.items[0].id;
+ const session_id = bc.session_id.?;
try ctx.processMessage(.{ .id = 12, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });
try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
- try testing.expectEqual(0, bc.attached_sessions.items.len);
+ try testing.expectEqual(null, bc.session_id);
try ctx.expectSentResult(null, .{ .id = 12 });
try ctx.processMessage(.{ .id = 13, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
- try testing.expect(!std.mem.eql(u8, session_id, bc.attached_sessions.items[0].id));
- try ctx.expectSentResult(.{ .sessionId = bc.attached_sessions.items[0].id }, .{ .id = 13 });
+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));
+ try ctx.expectSentResult(.{ .sessionId = bc.session_id.? }, .{ .id = 13 });
+}
+
+test "cdp.target: detachFromTarget auxiliary session" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{
+ .id = "BID-9",
+ .session_id = "SID-PRIMARY",
+ .target_id = "TID-000000000B".*,
+ });
+
+ try ctx.processMessage(.{ .id = 10, .method = "Target.attachToTarget", .params = .{ .targetId = "TID-000000000B" } });
+ const session_id = bc.attached_sessions.items[0].id;
+ try testing.expect(!std.mem.eql(u8, session_id, bc.session_id.?));
+
+ try ctx.processMessage(.{ .id = 11, .method = "Target.detachFromTarget", .params = .{ .sessionId = session_id } });
+ try ctx.expectSentEvent("Target.detachedFromTarget", .{ .sessionId = session_id }, .{});
+ try testing.expectEqual(0, bc.attached_sessions.items.len);
+ try testing.expectEqual(true, bc.session_id != null);
+ try ctx.expectSentResult(null, .{ .id = 11 });
}
test "cdp.target: detachFromTarget without session" {
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index dcca1bddd..32155136b 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1453,7 +1453,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
try transfer.handleRedirect(location.value);
- transfer.restoreInterceptHeaders();
if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
@@ -1465,23 +1464,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (!transfer.req.internal) lp.metrics.http_redirects.incr();
- if (self.serve_mode) {
- var wait_for_interception = false;
- transfer.req.notification.dispatch(.http_request_intercept, &.{
- .transfer = transfer,
- .wait_for_interception = &wait_for_interception,
- });
- if (wait_for_interception) {
- self.removeConn(msg.conn);
- transfer._conn = null;
- transfer.reset();
- transfer.state = .created;
- self.intercepted += 1;
- transfer.park(.intercept_request);
- return false;
- }
- }
-
const conn = transfer._conn.?;
try self.handles.remove(conn);
@@ -1931,9 +1913,6 @@ pub const Transfer = struct {
_tries: u8 = 0,
_redirect_count: u8 = 0,
- // Fetch.continueRequest header overrides apply to one network hop.
- _intercept_original_headers: ?[]const RequestHeader = null,
-
// Linked into client.pending_queue while .queued; reused to link the
// retired transfer into client.graveyard (deinit unlinks it from the
// pending queue first, so the node is always free by then).
@@ -2531,13 +2510,6 @@ pub const Transfer = struct {
}
}
- fn restoreInterceptHeaders(self: *Transfer) void {
- const headers = self._intercept_original_headers orelse return;
- self.req_headers.clearRetainingCapacity();
- self.req_headers.appendSliceAssumeCapacity(headers);
- self._intercept_original_headers = null;
- }
-
pub fn reset(self: *Transfer) void {
// Note: do NOT reset _auth_challenge or _redirect_count here. They
// span retries — _auth_challenge tells makeRequest whether to use
@@ -2737,8 +2709,6 @@ pub const Transfer = struct {
// CDP Fetch.continueRequest: the intercepting client supplies the
// complete header set, replacing whatever the request carried.
pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {
- lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });
- self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);
self.req_headers.clearRetainingCapacity();
try self.seedHeaders();
for (headers) |hdr| {
@@ -3170,29 +3140,6 @@ test "HttpClient: isFetchInterceptionMethod rejects unrelated methods" {
try testing.expect(!isFetchInterceptionMethod("Fetch.continueRequest "));
}
-test "HttpClient: Fetch header overrides restore after one hop" {
- const original = [_]Transfer.RequestHeader{
- .{ .name = "User-Agent", .value = "original" },
- .{ .name = "X-Original", .value = "yes" },
- };
- var overridden = [_]Transfer.RequestHeader{
- .{ .name = "User-Agent", .value = "override" },
- .{ .name = "X-Override", .value = "yes" },
- };
-
- var transfer: Transfer = undefined;
- transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };
- transfer._intercept_original_headers = &original;
- transfer.restoreInterceptHeaders();
-
- try testing.expectEqual(2, transfer.req_headers.items.len);
- try testing.expectString("User-Agent", transfer.req_headers.items[0].name);
- try testing.expectString("original", transfer.req_headers.items[0].value);
- try testing.expectString("X-Original", transfer.req_headers.items[1].name);
- try testing.expectString("yes", transfer.req_headers.items[1].value);
- try testing.expectEqual(null, transfer._intercept_original_headers);
-}
-
test "HttpClient: allowDuringSyncWait allows ping/close/disconnect" {
const test_arena = try testing.test_app.arena_pool.acquire(.tiny, "HttpClient test");
defer test_arena.release();