Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: remove `additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes`
Martin Adámek577135d03cd6
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 3e8232966..de696db05 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -43,3 +43,7 @@ Previously, the crawling context extended a `Record` type, allowing to access an## `additionalBlockedStatusCodes` parameter is removed`additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes` method is removed. Use the `blockedStatusCodes` crawler option instead.++## Remove `experimentalContainers` option++This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex c6ddda820..983d0f382 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -454,7 +454,6 @@ export abstract class BrowserCrawler<};const useIncognitoPages = this.launchContext?.useIncognitoPages;- const experimentalContainers = this.launchContext?.experimentalContainers;if (this.proxyConfiguration) {const { session } = crawlingContext;@@ -481,7 +480,7 @@ export abstract class BrowserCrawler<const page = (await this.browserPool.newPage(newPageOptions)) as CommonPage;tryCancel();- this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages || experimentalContainers);+ this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);// DO NOT MOVE THIS LINE ABOVE!// `enhanceCrawlingContextWithPageInfo` gives us a valid session.diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 7e6f10e87..cdff657de 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -49,13 +49,6 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO*/useIncognitoPages?: boolean;- /**- * @experimental- * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.- * Works best with Firefox. Unstable on Chromium.- */- experimentalContainers?: boolean;-/*** Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.@@ -116,7 +109,6 @@ export abstract class BrowserLauncher<useChrome: ow.optional.boolean,useIncognitoPages: ow.optional.boolean,browserPerProxy: ow.optional.boolean,- experimentalContainers: ow.optional.boolean,ignoreProxyCertificate: ow.optional.boolean,userDataDir: ow.optional.string,launchOptions: ow.optional.object,diff --git a/packages/browser-pool/copy-definitions.mjs b/packages/browser-pool/copy-definitions.mjsdeleted file mode 100644index 797e62a13..000000000--- a/packages/browser-pool/copy-definitions.mjs+++ /dev/null@@ -1,16 +0,0 @@-import { copyFileSync, mkdirSync, readdirSync } from 'node:fs';-import { join } from 'node:path';--const copyFolderSync = (from, to) => {- mkdirSync(to);-- for (const file of readdirSync(from, { withFileTypes: true })) {- if (file.isDirectory()) {- copyFolderSync(join(from, file.name), join(to, file.name));- } else if (file.isFile()) {- copyFileSync(join(from, file.name), join(to, file.name));- }- }-};--copyFolderSync('tab-as-a-container', 'dist/tab-as-a-container');diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex cc2dda049..902fa5fc8 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -24,7 +24,7 @@"url": "https://github.com/apify/crawlee/issues"},"scripts": {- "build": "yarn clean && yarn compile && node copy-definitions.mjs && yarn copy",+ "build": "yarn clean && yarn compile && yarn copy","clean": "rimraf ./dist","compile": "tsc -p tsconfig.build.json","copy": "tsx ../../scripts/copy.ts"diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 5c91c1163..becad13df 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -66,12 +66,6 @@ export interface BrowserPluginOptions<LibraryOptions> {* @default false*/useIncognitoPages?: boolean;- /**- * @experimental- * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.- * Works best with Firefox. Unstable on Chromium.- */- experimentalContainers?: boolean;/*** Path to a User Data Directory, which stores browser session data like cookies and local storage.*/@@ -117,19 +111,11 @@ export abstract class BrowserPlugin<NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,> {name = this.constructor.name;-library: Library;-launchOptions: LibraryOptions;-proxyUrl?: string;-userDataDir?: string;-useIncognitoPages: boolean;-- experimentalContainers: boolean;-browserPerProxy?: boolean;ignoreProxyCertificate?: boolean;@@ -140,7 +126,6 @@ export abstract class BrowserPlugin<proxyUrl,userDataDir,useIncognitoPages = false,- experimentalContainers = false,browserPerProxy = false,ignoreProxyCertificate = false,} = options;@@ -150,7 +135,6 @@ export abstract class BrowserPlugin<this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);this.userDataDir = userDataDir;this.useIncognitoPages = useIncognitoPages;- this.experimentalContainers = experimentalContainers;this.browserPerProxy = browserPerProxy;this.ignoreProxyCertificate = ignoreProxyCertificate;}@@ -170,7 +154,6 @@ export abstract class BrowserPlugin<proxyUrl = this.proxyUrl,useIncognitoPages = this.useIncognitoPages,userDataDir = this.userDataDir,- experimentalContainers = this.experimentalContainers,browserPerProxy = this.browserPerProxy,ignoreProxyCertificate = this.ignoreProxyCertificate,proxyTier,@@ -182,7 +165,6 @@ export abstract class BrowserPlugin<browserPlugin: this,proxyUrl,useIncognitoPages,- experimentalContainers,userDataDir,browserPerProxy,ignoreProxyCertificate,diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.tsindex 500b13606..432a176a2 100644--- a/packages/browser-pool/src/browser-pool.ts+++ b/packages/browser-pool/src/browser-pool.ts@@ -569,10 +569,7 @@ export class BrowserPool<await browserController['isActivePromise'];tryCancel();- const finalPageOptions =- browserController.launchContext.useIncognitoPages || browserController.launchContext.experimentalContainers- ? pageOptions- : undefined;+ const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;if (finalPageOptions) {Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.tsindex 70d2a0168..9ae4f3343 100644--- a/packages/browser-pool/src/launch-context.ts+++ b/packages/browser-pool/src/launch-context.ts@@ -46,12 +46,6 @@ export interface LaunchContextOptions<* If set to `true` each page uses its own context that is destroyed once the page is closed or crashes.*/useIncognitoPages?: boolean;- /**- * @experimental- * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.- * Works best with Firefox. Unstable on Chromium.- */- experimentalContainers?: boolean;/*** Path to a User Data Directory, which stores browser session data like cookies and local storage.*/@@ -77,7 +71,6 @@ export class LaunchContext<launchOptions: LibraryOptions;useIncognitoPages: boolean;browserPerProxy?: boolean;- experimentalContainers: boolean;userDataDir: string;proxyTier?: number;ignoreProxyCertificate?: boolean;@@ -96,7 +89,6 @@ export class LaunchContext<proxyUrl,useIncognitoPages,browserPerProxy,- experimentalContainers,userDataDir = '',proxyTier,ignoreProxyCertificate,@@ -107,7 +99,6 @@ export class LaunchContext<this.launchOptions = launchOptions;this.browserPerProxy = browserPerProxy ?? false;this.useIncognitoPages = useIncognitoPages ?? false;- this.experimentalContainers = experimentalContainers ?? false;this.userDataDir = userDataDir;this.proxyTier = proxyTier;this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;diff --git a/packages/browser-pool/src/playwright/load-firefox-addon.ts b/packages/browser-pool/src/playwright/load-firefox-addon.tsdeleted file mode 100644index a11960248..000000000--- a/packages/browser-pool/src/playwright/load-firefox-addon.ts+++ /dev/null@@ -1,104 +0,0 @@-import { Buffer } from 'node:buffer';-import net from 'node:net';--export const loadFirefoxAddon = async (port: number, host: string, addonPath: string) => {- return new Promise<boolean>((resolve) => {- const socket = net.connect({- port,- host,- });-- let success = false;-- socket.once('error', () => {});- socket.once('close', () => {- resolve(success);- });-- const send = (data: Record<string, string>) => {- const raw = Buffer.from(JSON.stringify(data));-- socket.write(`${raw.length}`);- socket.write(':');- socket.write(raw);- };-- send({- to: 'root',- type: 'getRoot',- });-- const onMessage = (message: any) => {- if (message.addonsActor) {- send({- to: message.addonsActor,- type: 'installTemporaryAddon',- addonPath,- });- }-- if (message.addon) {- success = true;- socket.end();- }-- if (message.error) {- socket.end();- }- };-- const buffers: Buffer[] = [];- let remainingBytes = 0;-- socket.on('data', (data) => {- while (true) {- if (remainingBytes === 0) {- const index = data.indexOf(':');-- buffers.push(data);-- if (index === -1) {- return;- }-- const buffer = Buffer.concat(buffers);- const bufferIndex = buffer.indexOf(':');-- buffers.length = 0;- remainingBytes = Number(buffer.subarray(0, bufferIndex).toString());-- if (!Number.isFinite(remainingBytes)) {- throw new Error('Invalid state');- }-- data = buffer.subarray(bufferIndex + 1);- }-- if (data.length < remainingBytes) {- remainingBytes -= data.length;- buffers.push(data);- break;- }-- buffers.push(data.subarray(0, remainingBytes));-- const buffer = Buffer.concat(buffers);- buffers.length = 0;-- const json = JSON.parse(buffer.toString());- queueMicrotask(() => {- onMessage(json);- });-- const remainder = data.subarray(remainingBytes);- remainingBytes = 0;-- if (remainder.length === 0) {- break;- }-- data = remainder;- }- });- });-};diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.tsindex baab51572..6032f2b7a 100644--- a/packages/browser-pool/src/playwright/playwright-controller.ts+++ b/packages/browser-pool/src/playwright/playwright-controller.ts@@ -36,14 +36,8 @@ export class PlaywrightController extends BrowserController<}protected async _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page> {- if (- contextOptions !== undefined &&- !this.launchContext.useIncognitoPages &&- !this.launchContext.experimentalContainers- ) {- throw new Error(- 'A new page can be created with provided context only when using incognito pages or experimental containers.',- );+ if (contextOptions !== undefined && !this.launchContext.useIncognitoPages) {+ throw new Error('A new page can be created with provided context only when using incognito pages.');}let close = async () => {};@@ -83,50 +77,6 @@ export class PlaywrightController extends BrowserController<await close();});- if (this.launchContext.experimentalContainers) {- await page.goto('data:text/plain,tabid');- await page.waitForNavigation();- const { tabid, proxyip }: { tabid: number; proxyip: string } = JSON.parse(- decodeURIComponent(page.url().slice('about:blank#'.length)),- );-- if (contextOptions?.proxy) {- const url = new URL(contextOptions.proxy.server);- url.username = contextOptions.proxy.username ?? '';- url.password = contextOptions.proxy.password ?? '';-- (this.browserPlugin as PlaywrightPlugin)._containerProxyServer!.ipToProxy.set(proxyip, url.href);- }-- if (this.browserPlugin.library.name() === 'firefox') {- // Playwright does not support creating new CDP sessions with Firefox- } else {- const session = await page.context().newCDPSession(page);- await session.send('Network.enable');-- session.on('Network.responseReceived', (responseReceived) => {- const logOnly = ['Document', 'XHR', 'Fetch', 'EventSource', 'WebSocket', 'Other'];- if (!logOnly.includes(responseReceived.type)) {- return;- }-- const { response } = responseReceived;- if (response.fromDiskCache || response.fromPrefetchCache || response.fromServiceWorker) {- return;- }-- const { remoteIPAddress } = response;- if (remoteIPAddress && remoteIPAddress !== proxyip) {- console.warn(- `Request to ${response.url} was through ${remoteIPAddress} instead of ${proxyip}`,- );- }- });- }-- tabIds.set(page, tabid);- }-tryCancel();return page;@@ -148,46 +98,11 @@ export class PlaywrightController extends BrowserController<protected async _getCookies(page: Page): Promise<Cookie[]> {const context = page.context();- const cookies = await context.cookies();-- if (this.launchContext.experimentalContainers) {- const tabId = tabIds.get(page);-- if (tabId === undefined) {- throw new Error('Failed to find tabId for page');- }-- const key = keyFromTabId(tabId);-- return cookies- .filter((cookie) => cookie.name.startsWith(key))- .map((cookie) => ({- ...cookie,- name: cookie.name.slice(key.length),- }));- }-- return cookies;+ return context.cookies();}protected async _setCookies(page: Page, cookies: Cookie[]): Promise<void> {const context = page.context();-- if (this.launchContext.experimentalContainers) {- const tabId = tabIds.get(page);-- if (tabId === undefined) {- throw new Error('Failed to find tabId for page');- }-- const key = keyFromTabId(tabId);-- cookies = cookies.map((cookie) => ({- ...cookie,- name: `${key}${cookie.name}`,- }));- }-return context.addCookies(cookies);}}diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex 73333d5dd..b79ab48c8 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -1,37 +1,17 @@import fs from 'node:fs';-import net from 'node:net';-import os from 'node:os';-import path from 'node:path';import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';import { anonymizeProxySugar } from '../anonymize-proxy.js';-import { createProxyServerForContainers } from '../container-proxy-server.js';+import type { createProxyServerForContainers } from '../container-proxy-server.js';import type { LaunchContext } from '../launch-context.js';import { log } from '../logger.js';import { getLocalProxyAddress } from '../proxy-server.js';import type { SafeParameters } from '../utils.js';-import { loadFirefoxAddon } from './load-firefox-addon.js';import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';import { PlaywrightController } from './playwright-controller.js';-const getFreePort = async () => {- return new Promise<number>((resolve, reject) => {- const server = net- .createServer()- .once('error', reject)- .listen(() => {- resolve((server.address() as net.AddressInfo).port);- server.close();- });- });-};--// import.meta.dirname = browser-pool/dist/playwright-// taacPath = browser-pool/dist/tab-as-a-container-const taacPath = path.join(import.meta.dirname, '..', 'tab-as-a-container');-export class PlaywrightPlugin extends BrowserPlugin<BrowserType,SafeParameters<BrowserType['launch']>[0],@@ -41,10 +21,7 @@ export class PlaywrightPlugin extends BrowserPlugin<_containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;protected async _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser> {- const { launchOptions, useIncognitoPages, proxyUrl } = launchContext;-- let { userDataDir } = launchContext;-+ const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;let browser: PlaywrightBrowser;// Required for the `proxy` context option to work.@@ -80,44 +57,6 @@ export class PlaywrightPlugin extends BrowserPlugin<});}} else {- const experimentalContainers = launchContext.experimentalContainers && this.library.name() !== 'webkit';- let firefoxPort: number | undefined;-- if (experimentalContainers) {- launchOptions!.args = [...(launchOptions!.args ?? [])];-- // Use native headless mode so we can load an extension- if (launchOptions!.headless && this.library.name() === 'chromium') {- launchOptions!.args.push('--headless=chrome');- }-- if (this.library.name() === 'chromium') {- launchOptions!.args.push(- `--disable-extensions-except=${taacPath}`,- `--load-extension=${taacPath}`,- );- } else if (this.library.name() === 'firefox') {- firefoxPort = await getFreePort();-- launchOptions!.args.push(`--start-debugger-server=${firefoxPort}`);-- const prefs = {- 'devtools.debugger.remote-enabled': true,- 'devtools.debugger.prompt-connection': false,- };-- const prefsRaw = Object.entries(prefs)- .map(([name, value]) => `user_pref(${JSON.stringify(name)}, ${JSON.stringify(value)});`)- .join('\n');-- if (userDataDir === '') {- userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'apify-playwright-firefox-taac-'));- }-- fs.writeFileSync(path.join(userDataDir, 'user.js'), prefsRaw);- }- }-const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {@@ -133,34 +72,6 @@ export class PlaywrightPlugin extends BrowserPlugin<}});- if (experimentalContainers) {- if (this.library.name() === 'firefox') {- const loaded = await loadFirefoxAddon(firefoxPort!, '127.0.0.1', taacPath);-- if (!loaded) {- await browserContext.close();- throw new Error('Failed to load Firefox experimental containers addon');- }- }-- // Wait for the extension to load.- const checker = await browserContext.newPage();- await checker.goto('data:text/plain,tabid');- await checker.waitForNavigation();- await checker.close();-- this._containerProxyServer = await createProxyServerForContainers();-- const page = await browserContext.newPage();- await page.goto(`data:text/plain,proxy#{"port":${this._containerProxyServer.port}}`);- await page.waitForNavigation();- await page.close();-- browserContext.on('close', async () => {- await this._containerProxyServer!.close(true);- });- }-if (anonymizedProxyUrl) {browserContext.on('close', async () => {await close();diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex 50f1b4814..ac9dcf7fa 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -43,9 +43,7 @@ export class PuppeteerController extends BrowserController<protected async _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page> {if (contextOptions !== undefined) {if (!this.launchContext.useIncognitoPages) {- throw new Error(- 'A new page can be created with provided context only when using incognito pages or experimental containers.',- );+ throw new Error('A new page can be created with provided context only when using incognito pages.');}let close = async () => {};diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex 8e345f74a..1bdb3e986 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -40,6 +40,7 @@ export class PuppeteerPlugin extends BrowserPlugin<} catch {// ignore}+const {launchOptions,userDataDir,diff --git a/packages/browser-pool/tab-as-a-container/background.js b/packages/browser-pool/tab-as-a-container/background.jsdeleted file mode 100644index f315fcdf1..000000000--- a/packages/browser-pool/tab-as-a-container/background.js+++ /dev/null@@ -1,433 +0,0 @@-'use strict';--/* eslint-disable no-undef */--const isFirefox = navigator.userAgent.includes('Firefox');--const webRequestPermissions = {- blockingRequest: isFirefox ? ['blocking', 'requestHeaders'] : ['blocking', 'requestHeaders', 'extraHeaders'],- blockingResponse: isFirefox ? ['blocking', 'responseHeaders'] : ['blocking', 'responseHeaders', 'extraHeaders'],-};--chrome.privacy.network.networkPredictionEnabled.set({ value: false });--const translator = new Map();-const counter = new Map();--const getOpenerId = (id) => {- if (typeof id !== 'number' || !Number.isFinite(id)) {- throw new Error('Expected `id` to be a number');- }-- if (translator.has(id)) {- const opener = translator.get(id);-- if (translator.has(opener)) {- throw new Error('Opener is not the most ascendent');- }-- // console.log(`getopener ${id} -> ${opener}`);- return opener;- }-- return id;-};--const keyFromTabId = (tabId) => `.${tabId}.`;--const getCookieURL = (cookie) => {- const protocol = cookie.secure ? 'https:' : 'http:';- const fixedDomain = cookie.domain[0] === '.' ? cookie.domain.slice(1) : cookie.domain;- const url = `${protocol}//${fixedDomain}${cookie.path}`;-- return url;-};--// Rewrite cookies that were programmatically set to tabId instead of openerId.-// This is required because we cannot reliably get openerId inside Playwright.-chrome.cookies.onChanged.addListener(async (changeInfo) => {- if (!changeInfo.removed) {- const { cookie } = changeInfo;-- if (cookie.name[0] !== '.') {- return;- }-- const dotIndex = cookie.name.indexOf('.', 1);- if (dotIndex === -1) {- return;- }-- const tabId = Number(cookie.name.slice(1, dotIndex));-- if (!Number.isFinite(tabId)) {- return;- }-- const realCookieName = cookie.name.slice(dotIndex + 1);- const opener = getOpenerId(tabId);-- if (tabId !== opener) {- console.log(`${realCookieName} -> ${keyFromTabId(opener)}`);-- await chrome.cookies.remove({- name: cookie.name,- url: getCookieURL(cookie),- storeId: cookie.storeId,- });-- delete cookie.hostOnly;- delete cookie.session;-- await chrome.cookies.set({- ...cookie,- name: `${keyFromTabId(opener)}${realCookieName}`,- url: getCookieURL(cookie),- });- }- }-});--chrome.webRequest.onBeforeSendHeaders.addListener(- (details) => {- for (const header of details.requestHeaders) {- if (header.name.toLowerCase() === 'cookie') {- const id = keyFromTabId(getOpenerId(details.tabId));-- const fixedCookies = header.value- .split('; ')- .filter((x) => x.startsWith(id))- .map((x) => x.slice(id.length))- .join('; ');- header.value = fixedCookies;- }-- // Sometimes Chrome makes a request on a ghost tab.- // We don't want these in order to prevent cluttering cookies.- // Yes, `webNavigation.onCommitted` is emitted and `webNavigation.onCreatedNavigationTarget` is not.- if (header.name.toLowerCase() === 'purpose' && header.value === 'prefetch' && !counter.has(details.tabId)) {- console.log(details);- return {- cancel: true,- };- }-- // This one is for Firefox- if (header.name.toLowerCase() === 'x-moz' && header.value === 'prefetch' && !counter.has(details.tabId)) {- console.log(details);- return {- cancel: true,- };- }-- if (['beacon', 'csp_report', 'ping', 'speculative'].includes(details.type)) {- console.log(details);- return {- cancel: true,- };- }-- if (details.tabId === -1) {- console.log(details);- }- }-- return {- requestHeaders: details.requestHeaders.filter(- (header) => header.name.toLowerCase() !== 'cookie' || header.value !== '',- ),- };- },- { urls: ['<all_urls>'] },- webRequestPermissions.blockingRequest,-);--// Firefox Bug: doesn't catch https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-uri-chrome.webRequest.onHeadersReceived.addListener(- (details) => {- for (const header of details.responseHeaders) {- if (header.name.toLowerCase() === 'set-cookie') {- const parts = header.value.split('\n');-- // `details.tabId` === -1 when Chrome is making internal requests, such downloading a service worker.-- const openerId = getOpenerId(details.tabId);-- header.value = parts- .map((part) => {- const equalsIndex = part.indexOf('=');- if (equalsIndex === -1) {- return `${keyFromTabId(openerId)}=${part.trimStart()}`;- }- return keyFromTabId(openerId) + part.trimStart();- })- .join('\n');- }- }-- return {- responseHeaders: details.responseHeaders,- };- },- { urls: ['<all_urls>'] },- webRequestPermissions.blockingResponse,-);--chrome.tabs.onRemoved.addListener(async (tabId) => {- const opener = getOpenerId(tabId);- translator.delete(tabId);-- if (counter.has(opener)) {- counter.set(opener, counter.get(opener) - 1);-- if (counter.get(opener) < 1) {- counter.delete(opener);- } else {- return;- }- }-- const id = keyFromTabId(opener);-- chrome.cookies.getAll({}, async (cookies) => {- await Promise.allSettled(- cookies- .filter((cookie) => cookie.name.startsWith(id))- .map((cookie) => {- return chrome.cookies.remove({- name: cookie.name,- url: getCookieURL(cookie),- storeId: cookie.storeId,- });- }),- );- });-});--// Proxy per tab-const getProxyConfiguration = (scheme, host, port) => {- return {- mode: 'fixed_servers',- rules: {- proxyForHttp: {- scheme,- host,- port,- },- proxyForHttps: {- scheme,- host,- port,- },- },- };-};--const localhostIpCache = new Map();-const localHostIp = [127, 0, 0, 1];-const getNextLocalhostIp = (openerId) => {- if (localhostIpCache.has(openerId)) {- return localhostIpCache.get(openerId);- }-- const result = localHostIp.join('.');-- localhostIpCache.set(openerId, result);-- if (localHostIp[3] === 254) {- if (localHostIp[2] === 255) {- if (localHostIp[1] === 255) {- localHostIp[1] = 0;- } else {- localHostIp[1]++;- }-- localHostIp[2] = 0;- } else {- localHostIp[2]++;- }-- localHostIp[3] = 1;- } else {- localHostIp[3]++;- }-- // [127.0.0.1 - 127.255.255.254] = 1 * 255 * 255 * 254 = 16 516 350- while (localhostIpCache.length >= 1 * 255 * 255 * 254) {- localhostIpCache.delete(localhostIpCache.keys().next().value);- }-- return result;-};--let proxyPort;--// Clear extension's proxy settings on reload-if (isFirefox) {- browser.proxy.settings.clear({});-} else {- chrome.proxy.settings.clear({});-}--// Proxy per tab-if (isFirefox) {- // On Firefox, we could use the `dns` permission to enforce DoH- // but then the extension would not be compatible with Chrome.- // Therefore users need to manually set the DNS settings.-- browser.proxy.onRequest.addListener(- (details) => {- const openerId = getOpenerId(details.tabId);-- if (typeof proxyPort === 'number') {- return {- type: 'http',- host: getNextLocalhostIp(openerId),- port: proxyPort,- };- }- return {- type: 'direct',- };- },- { urls: ['<all_urls>'] },- );-} else {- // The connection is not yet created with `onBeforeSendHeaders`, but is with `onSendHeaders`.- chrome.webRequest.onBeforeSendHeaders.addListener(- (details) => {- const openerId = getOpenerId(details.tabId);-- if (typeof proxyPort === 'number') {- chrome.proxy.settings.set({- value: getProxyConfiguration('http', getNextLocalhostIp(openerId), proxyPort),- scope: 'regular',- });- } else {- chrome.proxy.settings.clear({});- }- },- { urls: ['<all_urls>'] },- webRequestPermissions.blockingRequest,- );-}--// External communication. Note: the JSON keys are lowercased by the browser.-const routes = Object.assign(Object.create(null), {- async tabid(details) {- return { tabid: details.tabId, proxyip: getNextLocalhostIp(details.tabId) };- },- async proxy(details, body) {- proxyPort = body.port;-- return '';- },-});--const onCompleted = async (details) => {- const textPlain = 'data:text/plain,';-- if (details.frameId === 0 && details.url.startsWith(textPlain)) {- try {- const url = new URL(details.url);- const route = url.pathname.slice('text/plain,'.length);-- if (route in routes) {- const hash = url.hash.slice(1);-- let body = {};-- if (hash !== '') {- try {- body = JSON.parse(decodeURIComponent(hash));- } catch {- // Empty on purpose.- }- }-- // Different protocols are required, otherwise `onCompleted` won't be emitted.- const result = await routes[route](details, body);- if (result !== undefined) {- await chrome.tabs.update(details.tabId, {- url: `about:blank#${encodeURIComponent(JSON.stringify(result))}`,- });- }- }- } catch {- // Invalid URL, ignore.- }- }-};--chrome.webNavigation.onCompleted.addListener(onCompleted);--// Load content scripts.-void (async () => {- const contentResponse = await fetch(chrome.runtime.getURL('content.js'));- const contentText = await contentResponse.text();-- // `tabs.onCreated` doesn't work here when manually creating new tabs,- // because the opener is the current tab active.- //- // This events only fires when the page opens something.- chrome.webNavigation.onCreatedNavigationTarget.addListener((details) => {- translator.set(details.tabId, getOpenerId(details.sourceTabId));-- const opener = getOpenerId(details.tabId);-- if (counter.has(opener)) {- counter.set(opener, counter.get(opener) + 1);- } else {- counter.set(opener, 2); // the current one + opener = 2- }- });-- chrome.webNavigation.onCommitted.addListener(async (details) => {- if (details.url.startsWith('chrome')) {- return;- }-- const executeCodeInPageContext = `- const script = document.createElement('script');- script.textContent = code;-- const destination = document.head ?? document.documentElement;-- if (document instanceof HTMLDocument) {- destination.append(script);- script.remove();- }- `;-- // Race condition: website scripts may run first- await chrome.tabs.executeScript(details.tabId, {- code: `'use strict';- (() => {- if (window.totallyRandomString) {- return;- }-- window.totallyRandomString = true;-- const code = "'use strict'; const tabId = '${getOpenerId(- details.tabId,- )}'; (() => {\\n" + ${JSON.stringify(contentText)} + "\\n})();\\n";- ${executeCodeInPageContext}- })();- `,- matchAboutBlank: true,- allFrames: true,- runAt: 'document_start',- });- });-- chrome.tabs.query({}, async (tabs) => {- for (const tab of tabs) {- await onCompleted({- frameId: 0,- url: tab.url,- tabId: tab.id,- });- }- });-})();diff --git a/packages/browser-pool/tab-as-a-container/content.js b/packages/browser-pool/tab-as-a-container/content.jsdeleted file mode 100644index efbbff7c0..000000000--- a/packages/browser-pool/tab-as-a-container/content.js+++ /dev/null@@ -1,611 +0,0 @@-// When in doubt, refer to https://github.com/nodejs/node/blob/main/doc/contributing/primordials.md--/* eslint-disable no-undef */-/* eslint-disable no-cond-assign */-/* eslint-disable prefer-rest-params */-/* eslint-disable no-shadow */--// TODO: https://developer.mozilla.org/en-US/docs/Web/API/Cookie_Store_API-// TODO: custom error messages for Firefox (for now it uses Chrome's)--// The only way to detect this "container" is to benchmark document.cookie or compare localStorage performance with sessionStorage (it's the same).--const isFirefox = navigator.userAgent.includes('Firefox');-const tabPrefix = `.${tabId}.`;--const {- String,- Array,- Set,- TypeError,- WeakMap,- Object,- Number,- Function,- Proxy,- IDBFactory,- IDBDatabase,- BroadcastChannel,- Storage,- // We don't have to implement StorageEvent because this implementation does not use localStorage at all.-} = globalThis;--const ObjectDefineProperty = Object.defineProperty;-const ObjectDefineProperties = Object.defineProperties;-const ObjectGetOwnPropertyDescriptors = Object.getOwnPropertyDescriptors;-const ObjectGetPrototypeOf = Object.getPrototypeOf;-const ObjectGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;-const ObjectCreate = Object.create;-const ObjectEntries = Object.entries;-const ReflectGet = Reflect.get;-const ReflectSet = Reflect.set;-const ObjectKeys = Object.keys;-const NumberIsFinite = Number.isFinite;--const clonePrototype = (from) => {- const target = ObjectCreate(null);- const prototype = ObjectGetOwnPropertyDescriptors(from.prototype);-- const entries = ObjectEntries(prototype);-- for (let i = 0; i < entries.length; i++) {- const entry = entries[i];-- const { 0: name, 1: descriptor } = entry;- target[name] = ObjectCreate(null);-- if ('get' in descriptor) {- target[name].get = descriptor.get;- }-- if ('set' in descriptor) {- target[name].set = descriptor.set;- }-- if ('value' in descriptor) {- target[name] = descriptor.value;- }- }-- return target;-};--const StringSplitSafe = (string, separator) => {- const result = [];- const separatorLength = separator.length;-- if (separatorLength === 0) {- throw new Error('Separator must not be empty');- }-- let startFrom = 0;- let index;- while ((index = StringPrototype.indexOf.call(string, separator, startFrom)) !== -1) {- ArrayPrototype.push.call(result, StringPrototype.slice.call(string, startFrom, index));-- startFrom = index + separatorLength;- }-- const lastChunk = StringPrototype.slice.call(string, startFrom);-- ArrayPrototype.push.call(result, lastChunk);-- return result;-};--const fixStack = (error) => {- const lines = StringSplitSafe(error.stack, '\n');-- if (isFirefox) {- ArrayPrototype.splice.call(lines, 0, 1);- } else {- ArrayPrototype.splice.call(lines, 1, 1);- }-- error.stack = ArrayPrototype.join.call(lines, '\n');-- return error;-};--const SetPrototype = clonePrototype(Set);-const WeakMapPrototype = clonePrototype(WeakMap);-const ArrayPrototype = clonePrototype(Array);-const StringPrototype = clonePrototype(String);-const IDBFactoryPrototype = clonePrototype(IDBFactory);-const IDBDatabasePrototype = clonePrototype(IDBDatabase);-const StoragePrototype = clonePrototype(Storage);--const privates = new WeakMap();--let invocable = false;--const FakeStorage = class Storage {- constructor() {- if (invocable) {- throw fixStack(new TypeError('Illegal constructor'));- }-- WeakMapPrototype.set.call(privates, this, arguments[0]);- }-- get length() {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- const { storage, prefix } = priv;- const length = StoragePrototype.length.get.call(storage);-- let fakeLength = 0;- for (let i = 0; i < length; i++) {- const storageKey = StoragePrototype.key.call(storage, i);- if (StringPrototype.startsWith.call(storageKey, prefix)) {- fakeLength++;- }- }-- return fakeLength;- }-- clear() {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- const { storage, prefix } = priv;- const length = StoragePrototype.length.get.call(storage);- const keys = [];-- for (let i = 0; i < length; i++) {- ArrayPrototype.push.call(keys, StoragePrototype.key.call(storage, i));- }-- for (let i = 0; i < length; i++) {- const storageKey = keys[i];- if (StringPrototype.startsWith.call(storageKey, prefix)) {- StoragePrototype.removeItem.call(storage, storageKey);- }- }- }-- key(index) {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- if (arguments.length === 0) {- throw fixStack(- new TypeError(`Failed to execute 'key' on 'Storage': 1 argument required, but only 0 present.`),- );- }-- index = NumberIsFinite(index) ? index : 0;-- const { storage, prefix } = priv;- const length = StoragePrototype.length.get.call(storage);-- let fakeLength = 0;- for (let i = 0; i < length; i++) {- const storageKey = StoragePrototype.key.call(storage, i);-- if (StringPrototype.startsWith.call(storageKey, prefix)) {- if (fakeLength === index) {- return StringPrototype.slice.call(storageKey, prefix.length);- }-- fakeLength++;- }- }-- return null;- }-- getItem(key) {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- if (arguments.length === 0) {- throw fixStack(- new TypeError(`Failed to execute 'getItem' on 'Storage': 1 argument required, but only 0 present.`),- );- }-- return StoragePrototype.getItem.call(priv.storage, priv.prefix + key);- }-- removeItem(key) {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- if (arguments.length === 0) {- throw fixStack(- new TypeError(`Failed to execute 'removeItem' on 'Storage': 1 argument required, but only 0 present.`),- );- }-- StoragePrototype.removeItem.call(priv.storage, priv.prefix + key);- }-- setItem(key, value) {- const priv = WeakMapPrototype.get.call(privates, this);- if (!priv) {- throw fixStack(new TypeError('Illegal invocation'));- }-- if (arguments.length === 0 || arguments.length === 1) {- throw fixStack(- new TypeError(- `Failed to execute 'setItem' on 'Storage': 2 arguments required, but only ${arguments.length} present.`,- ),- );- }-- StoragePrototype.setItem.call(priv.storage, priv.prefix + key, value);- }-};--const FakeStoragePrototype = clonePrototype(FakeStorage);--const createStorage = ({ storage, prefix }) => {- invocable = false;- const fake = new FakeStorage({ storage, prefix });- invocable = true;-- const proxy = new Proxy(fake, {- __proto__: null,- // Default:- // apply: (target, thisArg, args) => {},- // construct(target, args) => {},- // setPrototypeOf: (target, proto) => {},- // getPrototypeOf: (target) => {},- defineProperty: (target, key, descriptor) => {- if ('set' in descriptor || 'get' in descriptor) {- throw fixStack(- new TypeError(`Failed to set a named property on 'Storage': Accessor properties are not allowed.`),- );- }-- FakeStoragePrototype.setItem.call(target, key, descriptor.value);- },- deleteProperty: (target, key) => {- if (typeof key === 'symbol') {- delete target[key];- } else {- FakeStoragePrototype.removeItem.call(target, key);- }-- return true;- },- get: (target, key) => {- if (typeof key === 'symbol') {- return target[key];- }-- if (key in target) {- return ReflectGet(target, key);- }-- return FakeStoragePrototype.getItem.call(target, key) ?? undefined;- },- set: (target, key, value) => {- if (typeof key === 'symbol') {- ObjectDefineProperty(target, key, {- __proto__: null,- value,- configurable: true,- writable: true,- enumerable: false,- });-- return true;- }-- if (key in target) {- return ReflectSet(target, key, value);- }-- return FakeStoragePrototype.setItem.call(target, key, value) ?? true;- },- has: (target, key) => {- if (key in target) {- return true;- }-- return FakeStoragePrototype.getItem.call(target, key) !== null;- },- isExtensible: () => {- return true;- },- preventExtensions: () => {- throw fixStack(new TypeError(`Cannot prevent extensions`));- },- getOwnPropertyDescriptor: (target, key) => {- if (key in target) {- return ObjectGetOwnPropertyDescriptor(ObjectGetPrototypeOf(target), key);- }-- const value = FakeStoragePrototype.getItem.call(target, key);-- if (value !== null) {- return {- value,- writable: true,- enumerable: true,- configurable: true,- };- }- },- ownKeys: (target) => {- const keys = [];-- const { storage, prefix } = WeakMapPrototype.get.call(privates, target);- const length = StoragePrototype.length.get.call(storage);-- for (let i = 0; i < length; i++) {- const storageKey = StoragePrototype.key.call(storage, i);-- if (StringPrototype.startsWith.call(storageKey, prefix)) {- ArrayPrototype.push.call(keys, StringPrototype.slice.call(storageKey, prefix.length));- }- }-- ArrayPrototype.push.apply(keys, ObjectKeys(target));-- const set = new Set();-- for (let i = 0; i < keys.length; i++) {- SetPrototype.add.call(set, keys[i]);- }-- return ArrayPrototype.slice.call(set);- },- });-- privates.set(proxy, privates.get(fake));-- return proxy;-};--const toHide = new WeakMap();-for (const Type of [Function, Object, Array]) {- const create = (fallback) =>- function () {- if (this instanceof FakeStorage) {- return '[object Storage]';- }-- if (WeakMapPrototype.has.call(toHide, this)) {- return `function ${WeakMapPrototype.get.call(toHide, this)}() { [native code] }`;- }-- return fallback.call(this);- };-- const toString = create(Type.prototype.toString);- const toLocaleString = create(Type.prototype.toLocaleString);-- WeakMapPrototype.set.call(toHide, toString, 'toString');- WeakMapPrototype.set.call(toHide, toLocaleString, 'toLocaleString');-- Object.defineProperty(Type.prototype, 'toString', {- __proto__: null,- value: toString,- });- Object.defineProperty(Type.prototype, 'toLocaleString', {- __proto__: null,- value: toLocaleString,- });-}--// https://stackoverflow.com/q/30481516-try {- // We use sessionStorage as the underlying storage for localStorage.- // This way we do not have to worry about clean up.- const { sessionStorage } = globalThis;-- const fakeLocalStorage = createStorage({ storage: sessionStorage, prefix: 'l.' });- const fakeSessionStorage = createStorage({ storage: sessionStorage, prefix: 's.' });-- const getLocalStorage = function localStorage() {- return fakeLocalStorage;- };- const getSessionStorage = function sessionStorage() {- return fakeSessionStorage;- };-- WeakMapPrototype.set.call(toHide, FakeStorage, 'Storage');- WeakMapPrototype.set.call(toHide, FakeStoragePrototype.key, 'key');- WeakMapPrototype.set.call(toHide, FakeStoragePrototype.getItem, 'getItem');- WeakMapPrototype.set.call(toHide, FakeStoragePrototype.setItem, 'setItem');- WeakMapPrototype.set.call(toHide, FakeStoragePrototype.removeItem, 'removeItem');- WeakMapPrototype.set.call(toHide, FakeStoragePrototype.clear, 'clear');- WeakMapPrototype.set.call(toHide, getLocalStorage, 'get localStorage');- WeakMapPrototype.set.call(toHide, getSessionStorage, 'get sessionStorage');-- ObjectDefineProperties(window, {- __proto__: null,- Storage: {- __proto__: null,- value: FakeStorage,- configurable: true,- enumerable: false,- writable: true,- },- localStorage: {- __proto__: null,- configurable: true,- enumerable: true,- get: getLocalStorage,- set: undefined,- },- sessionStorage: {- __proto__: null,- configurable: true,- enumerable: true,- get: getSessionStorage,- set: undefined,- },- });-} catch (error) {- console.error(error);-}--{- const { Document } = globalThis;-- const realGetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').get;- const realSetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').set;-- const getCookie = function cookie() {- try {- const cookies = StringSplitSafe(realGetCookie.call(this), '; ');- const filtered = ArrayPrototype.filter.call(cookies, (cookie) =>- StringPrototype.startsWith.call(cookie, tabPrefix),- );- const mapped = ArrayPrototype.map.call(filtered, (cookie) => {- const result = StringPrototype.slice.call(cookie, tabPrefix.length);-- if (result[0] === '=') {- return StringPrototype.slice.call(result, 1);- }-- return result;- });-- return ArrayPrototype.join.call(mapped, '; ');- } catch (error) {- throw fixStack(error);- }- };-- const setCookie = function cookie(cookieString) {- cookieString = StringPrototype.trimStart.call(String(cookieString));-- const delimiterIndex = StringPrototype.indexOf.call(cookieString, ';');- const equalsIndex = StringPrototype.indexOf.call(cookieString, '=');- if (equalsIndex === -1 || (delimiterIndex !== -1 && equalsIndex > delimiterIndex)) {- cookieString = `=${cookieString}`;- }-- try {- realSetCookie.call(this, tabPrefix + cookieString);- } catch (error) {- throw fixStack(error);- }- };-- WeakMapPrototype.set.call(toHide, getCookie, 'get cookie');- WeakMapPrototype.set.call(toHide, setCookie, 'set cookie');-- ObjectDefineProperty(Document.prototype, 'cookie', {- __proto__: null,- configurable: true,- enumerable: true,- get: getCookie,- set: setCookie,- });-}--{- const openDatabase = function open(name) {- try {- return IDBFactoryPrototype.open.call(this, tabPrefix + name);- } catch (error) {- throw fixStack(error);- }- };-- const deleteDatabase = function deleteDatabase(name) {- try {- return IDBFactoryPrototype.deleteDatabase.call(this, tabPrefix + name);- } catch (error) {- throw fixStack(error);- }- };-- const databaseName = function name() {- try {- return StringPrototype.slice.call(IDBDatabasePrototype.name.get.call(this), tabPrefix.length);- } catch (error) {- throw fixStack(error);- }- };-- WeakMapPrototype.set.call(toHide, openDatabase, 'open');- WeakMapPrototype.set.call(toHide, deleteDatabase, 'deleteDatabase');- WeakMapPrototype.set.call(toHide, databaseName, 'get name');-- ObjectDefineProperties(IDBFactory.prototype, {- __proto__: null,- open: {- __proto__: null,- writable: true,- configurable: true,- enumerable: true,- value: openDatabase,- },- deleteDatabase: {- __proto__: null,- writable: true,- configurable: true,- enumerable: true,- value: deleteDatabase,- },- name: {- __proto__: null,- configurable: true,- enumerable: true,- get: databaseName,- set: undefined,- },- });-}--{- ObjectDefineProperty(window, 'BroadcastChannel', {- __proto__: null,- configurable: true,- enumerable: false,- writable: true,- value: new Proxy(BroadcastChannel, {- __proto__: null,- construct: (Target, name) => {- return new Target(tabPrefix + name);- },- }),- });-- WeakMapPrototype.set.call(toHide, window.BroadcastChannel, 'BroadcastChannel');-- const getBroadcastChannelName = ObjectGetOwnPropertyDescriptor(BroadcastChannel.prototype, 'name').get;- const broadcastChannelName = function name() {- try {- const realName = getBroadcastChannelName.call(this);-- if (StringPrototype.startsWith.call(realName, tabPrefix)) {- return StringPrototype.slice.call(realName, tabPrefix.length);- }-- return realName;- } catch (error) {- throw fixStack(error);- }- };-- WeakMapPrototype.set.call(toHide, broadcastChannelName, 'get name');-- ObjectDefineProperty(BroadcastChannel.prototype, 'name', {- __proto__: null,- configurable: true,- enumerable: true,- get: broadcastChannelName,- set: undefined,- });-}diff --git a/packages/browser-pool/tab-as-a-container/manifest.json b/packages/browser-pool/tab-as-a-container/manifest.jsondeleted file mode 100644index cc77a982f..000000000--- a/packages/browser-pool/tab-as-a-container/manifest.json+++ /dev/null@@ -1,21 +0,0 @@-{- "manifest_version": 2,- "name": "Tab as a Container",- "version": "1.0.0",- "background": {- "scripts": ["background.js"],- "persistent": true- },- "permissions": [- "webRequest",- "webRequestBlocking",- "webNavigation",- "tabs",- "cookies",- "privacy",- "proxy",- "<all_urls>"- ],- "web_accessible_resources": ["content.js"],- "incognito": "not_allowed"-}diff --git a/packages/playwright-crawler/src/internals/playwright-launcher.ts b/packages/playwright-crawler/src/internals/playwright-launcher.tsindex e9920a76a..a8bf13c86 100644--- a/packages/playwright-crawler/src/internals/playwright-launcher.ts+++ b/packages/playwright-crawler/src/internals/playwright-launcher.ts@@ -58,13 +58,6 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti*/useIncognitoPages?: boolean;- /**- * @experimental- * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.- * Works best with Firefox. Unstable on Chromium.- */- experimentalContainers?: boolean;-/*** Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 1eb746dfb..a79fcd5ed 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1144,9 +1144,7 @@ describe('CheerioCrawler', () => {]);expect(cookie2).toBe('Foo=bar1; other=cookie1; coo=kie; foo=bar3; baz=123; Other=cookie2');expect(warningSpy).toBeCalledTimes(3);- expect(warningSpy).toBeCalledWith(- `Found cookies with similar name during cookie merging: 'foo' and 'Foo'`,- );+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'foo' and 'Foo'`);expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Other' and 'other'`,);@@ -1159,12 +1157,8 @@ describe('CheerioCrawler', () => {]);expect(cookie3).toBe('foo=bar2; Other=cookie2; Coo=kie; baz=123; Foo=bar3; coo=kee');expect(warningSpy).toBeCalledTimes(2);- expect(warningSpy).toBeCalledWith(- `Found cookies with similar name during cookie merging: 'Foo' and 'foo'`,- );- expect(warningSpy).toBeCalledWith(- `Found cookies with similar name during cookie merging: 'coo' and 'Coo'`,- );+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Foo' and 'foo'`);+ expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'coo' and 'Coo'`);});test('mergeCookies() throws a contextual error for malformed cookie fragments', () => {diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.jsondeleted file mode 100644index 0be68bf20..000000000--- a/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-playwright-chromium-experimental-containers",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore b/test/e2e/playwright-chromium-experimental-containers/actor/.gitignoredeleted file mode 100644index ced7cbfc5..000000000--- a/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore+++ /dev/null@@ -1,7 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storagediff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfiledeleted file mode 100644index 491acc4dd..000000000--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile+++ /dev/null@@ -1,23 +0,0 @@-FROM node:24 AS builder--COPY /packages ./packages-COPY /package*.json ./-RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \- && npm update--FROM apify/actor-node-playwright-chrome:24-1.58.2-beta--RUN rm -r node_modules-COPY --from=builder /node_modules ./node_modules-COPY --from=builder /packages ./packages-COPY --from=builder /package*.json ./-COPY /.actor ./.actor-COPY /main.js ./--RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --versiondiff --git a/test/e2e/playwright-chromium-experimental-containers/actor/main.js b/test/e2e/playwright-chromium-experimental-containers/actor/main.jsdeleted file mode 100644index 887cbb744..000000000--- a/test/e2e/playwright-chromium-experimental-containers/actor/main.js+++ /dev/null@@ -1,33 +0,0 @@-import { Actor } from 'apify';-import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';--// fails after update to playwright 1.29.0, looks like issue the chromium extension, maybe the manifest_version 2 vs 3?-process.exit(404);--const mainOptions = {- exit: Actor.isAtHome(),- storage:- process.env.STORAGE_IMPLEMENTATION === 'LOCAL'- ? new (await import('@apify/storage-local')).ApifyStorageLocal()- : undefined,-};--await Actor.main(async () => {- const crawler = new PlaywrightCrawler({- proxyConfiguration: await Actor.createProxyConfiguration(),- launchContext: {- experimentalContainers: true,- },- preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = 'networkidle';- },- ],- async requestHandler({ page }) {- const content = await page.content();- await Dataset.pushData({ ip: content.match(/"clientIp":\s*"(.*)"/)?.[1] });- },- });-- await crawler.run(['https://api.apify.com/v2/browser-info?1', 'https://api.apify.com/v2/browser-info?2']);-}, mainOptions);diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/package.json b/test/e2e/playwright-chromium-experimental-containers/actor/package.jsondeleted file mode 100644index 043160ba0..000000000--- a/test/e2e/playwright-chromium-experimental-containers/actor/package.json+++ /dev/null@@ -1,29 +0,0 @@-{- "name": "test-playwright-chromium-experimental-containers",- "version": "0.0.1",- "description": "Playwright Test - Chromium - Experimental containers",- "dependencies": {- "apify": "next",- "@apify/storage-local": "^3.0.0",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser": "file:./packages/browser-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/core": "file:./packages/core",- "@crawlee/memory-storage": "file:./packages/memory-storage",- "@crawlee/playwright": "file:./packages/playwright-crawler",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils",- "playwright": "1.58.2"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/utils": "file:./packages/utils"- }- },- "scripts": {- "start": "node main.js"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/playwright-chromium-experimental-containers/test.mjs b/test/e2e/playwright-chromium-experimental-containers/test.mjsdeleted file mode 100644index d42359b86..000000000--- a/test/e2e/playwright-chromium-experimental-containers/test.mjs+++ /dev/null@@ -1,18 +0,0 @@-import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs';--await skipTest('on hold');--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { datasetItems } = await runActor(testActorDirname, 16384);--await expect(datasetItems.length > 0, 'Has dataset items');--const ips = new Set();--for (const { ip } of datasetItems) {- await expect(!ips.has(ip), 'Unique proxy ip');-- ips.add(ip);-}diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.jsondeleted file mode 100644index d1bf754a5..000000000--- a/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-playwright-firefox-experimental-containers",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/.gitignore b/test/e2e/playwright-firefox-experimental-containers/actor/.gitignoredeleted file mode 100644index ced7cbfc5..000000000--- a/test/e2e/playwright-firefox-experimental-containers/actor/.gitignore+++ /dev/null@@ -1,7 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storagediff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfiledeleted file mode 100644index 891810913..000000000--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile+++ /dev/null@@ -1,23 +0,0 @@-FROM node:24 AS builder--COPY /packages ./packages-COPY /package*.json ./-RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \- && npm update--FROM apify/actor-node-playwright-firefox:24-1.58.2-beta--RUN rm -r node_modules-COPY --from=builder /node_modules ./node_modules-COPY --from=builder /packages ./packages-COPY --from=builder /package*.json ./-COPY /.actor ./.actor-COPY /main.js ./--RUN echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --versiondiff --git a/test/e2e/playwright-firefox-experimental-containers/actor/main.js b/test/e2e/playwright-firefox-experimental-containers/actor/main.jsdeleted file mode 100644index a07251a80..000000000--- a/test/e2e/playwright-firefox-experimental-containers/actor/main.js+++ /dev/null@@ -1,35 +0,0 @@-import { Actor } from 'apify';-import playwright from 'playwright';-import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';--// timeouts nowadays, hard to say why-process.exit(404);--const mainOptions = {- exit: Actor.isAtHome(),- storage:- process.env.STORAGE_IMPLEMENTATION === 'LOCAL'- ? new (await import('@apify/storage-local')).ApifyStorageLocal()- : undefined,-};--await Actor.main(async () => {- const crawler = new PlaywrightCrawler({- proxyConfiguration: await Actor.createProxyConfiguration(),- launchContext: {- launcher: playwright.firefox,- experimentalContainers: true,- },- preNavigationHooks: [- (_ctx, goToOptions) => {- goToOptions.waitUntil = 'networkidle';- },- ],- async requestHandler({ page }) {- const content = await page.content();- await Dataset.pushData({ ip: content.match(/"clientIp":\s*"(.*)"/)?.[1] });- },- });-- await crawler.run(['https://api.apify.com/v2/browser-info?1', 'https://api.apify.com/v2/browser-info?2']);-}, mainOptions);diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/package.json b/test/e2e/playwright-firefox-experimental-containers/actor/package.jsondeleted file mode 100644index e364bff81..000000000--- a/test/e2e/playwright-firefox-experimental-containers/actor/package.json+++ /dev/null@@ -1,29 +0,0 @@-{- "name": "test-playwright-firefox-experimental-containers",- "version": "0.0.1",- "description": "Playwright Test - Firefox - Experimental containers",- "dependencies": {- "apify": "next",- "@apify/storage-local": "^3.0.0",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser": "file:./packages/browser-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/core": "file:./packages/core",- "@crawlee/memory-storage": "file:./packages/memory-storage",- "@crawlee/playwright": "file:./packages/playwright-crawler",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils",- "playwright": "1.58.2"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/utils": "file:./packages/utils"- }- },- "scripts": {- "start": "node main.js"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/playwright-firefox-experimental-containers/test.mjs b/test/e2e/playwright-firefox-experimental-containers/test.mjsdeleted file mode 100644index d42359b86..000000000--- a/test/e2e/playwright-firefox-experimental-containers/test.mjs+++ /dev/null@@ -1,18 +0,0 @@-import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs';--await skipTest('on hold');--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { datasetItems } = await runActor(testActorDirname, 16384);--await expect(datasetItems.length > 0, 'Has dataset items');--const ips = new Set();--for (const { ip } of datasetItems) {- await expect(!ips.has(ip), 'Unique proxy ip');-- ips.add(ip);-}