Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: tab as a container + proxy per page (#1417)
Szymon Marczak71a755de4ef0
diff --git a/.eslintrc.json b/.eslintrc.jsonindex 563a03e54..935bfdd96 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -12,36 +12,44 @@"project": "./tsconfig.eslint.json","ecmaVersion": 2020},- "plugins": [- "@typescript-eslint"- ],"ignorePatterns": ["node_modules","dist","coverage",- "**/*.js","**/*.d.ts"],+ "overrides": [+ {+ "plugins": [+ "@typescript-eslint"+ ],+ "files": [+ "*.ts"+ ],+ "rules": {+ "@typescript-eslint/array-type": "error",+ "@typescript-eslint/ban-ts-comment": 0,+ "@typescript-eslint/consistent-type-imports": ["error", {+ "disallowTypeAnnotations": false+ }],+ "@typescript-eslint/consistent-type-definitions": ["error", "interface"],+ "@typescript-eslint/member-delimiter-style": ["error", {+ "multiline": { "delimiter": "semi", "requireLast": true },+ "singleline": { "delimiter": "semi", "requireLast": false }+ }],+ "@typescript-eslint/no-empty-interface": "off",+ "@typescript-eslint/no-empty-function": "off",+ "@typescript-eslint/no-explicit-any": "off",+ "@typescript-eslint/no-floating-promises": "error",+ "@typescript-eslint/no-unused-vars": "off"+ }+ }+ ],"rules": {"import/no-extraneous-dependencies": "off","max-classes-per-file": 0,"no-console": "error","no-underscore-dangle": 0,- "no-void": 0,- "@typescript-eslint/array-type": "error",- "@typescript-eslint/ban-ts-comment": 0,- "@typescript-eslint/consistent-type-imports": ["error", {- "disallowTypeAnnotations": false- }],- "@typescript-eslint/consistent-type-definitions": ["error", "interface"],- "@typescript-eslint/member-delimiter-style": ["error", {- "multiline": { "delimiter": "semi", "requireLast": true },- "singleline": { "delimiter": "semi", "requireLast": false }- }],- "@typescript-eslint/no-empty-interface": "off",- "@typescript-eslint/no-empty-function": "off",- "@typescript-eslint/no-explicit-any": "off",- "@typescript-eslint/no-floating-promises": "error",- "@typescript-eslint/no-unused-vars": "off"+ "no-void": 0}}diff --git a/package.json b/package.jsonindex 2a6a71f98..1b2723e98 100644--- a/package.json+++ b/package.json@@ -47,8 +47,8 @@"release": "npm run build && lerna version patch && lerna publish from-package --contents dist --force-publish","publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish","release:next": "npm run build && npm run publish:next",- "lint": "eslint packages/*/src packages/*/test test",- "lint:fix": "eslint packages/*/src packages/*/test test --fix"+ "lint": "eslint packages/browser-pool/tab-as-a-container packages/*/src packages/*/test test",+ "lint:fix": "eslint packages/browser-pool/tab-as-a-container packages/*/src packages/*/test test --fix"},"devDependencies": {"@apify/eslint-config-ts": "^0.2.3",diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex cccd02155..209e63bde 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -429,8 +429,9 @@ export abstract class BrowserCrawler<};const useIncognitoPages = this.launchContext?.useIncognitoPages;+ const experimentalContainers = this.launchContext?.experimentalContainers;- if (this.proxyConfiguration && useIncognitoPages) {+ if (this.proxyConfiguration && (useIncognitoPages || experimentalContainers)) {const { session } = crawlingContext;const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);@@ -451,7 +452,7 @@ export abstract class BrowserCrawler<const page = await this.browserPool.newPage(newPageOptions) as CommonPage;tryCancel();- this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);+ this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages || experimentalContainers);// DO NOT MOVE THIS LINE ABOVE!// `enhanceCrawlingContextWithPageInfo` gives us a valid session.@@ -489,7 +490,7 @@ export abstract class BrowserCrawler<}}- protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, useIncognitoPages?: boolean): void {+ protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, createNewSession?: boolean): void {crawlingContext.page = page;// This switch is because the crawlingContexts are created on per request basis.@@ -499,7 +500,7 @@ export abstract class BrowserCrawler<const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page as any) as Context['browserController'];crawlingContext.browserController = browserControllerInstance;- if (!useIncognitoPages) {+ if (!createNewSession) {crawlingContext.session = browserControllerInstance.launchContext.session as Session;}diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 7367e6415..4025ba6e2 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -37,6 +37,12 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO*/useIncognitoPages?: boolean;+ /**+ * @experimental+ * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.+ */+ experimentalContainers?: boolean;+/*** Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.@@ -78,6 +84,7 @@ export abstract class BrowserLauncher<proxyUrl: ow.optional.string.url,useChrome: ow.optional.boolean,useIncognitoPages: ow.optional.boolean,+ experimentalContainers: ow.optional.boolean,userDataDir: ow.optional.string,launchOptions: ow.optional.object,userAgent: ow.optional.string,diff --git a/packages/browser-pool/copy-definitions.mjs b/packages/browser-pool/copy-definitions.mjsindex 33aabb1a3..d1bf51c7d 100644--- a/packages/browser-pool/copy-definitions.mjs+++ b/packages/browser-pool/copy-definitions.mjs@@ -1,4 +1,4 @@-import { copyFileSync } from 'node:fs';+import { copyFileSync, mkdirSync, readdirSync } from 'node:fs';import { join } from 'node:path';const files = [@@ -11,3 +11,17 @@ for (const file of files) {join('dist', file),);}++const copyFolderSync = (from, to) => {+ mkdirSync(to);++ for (const file of readdirSync(from, { withFileTypes: true })) {+ if (file.isDirectory()) {+ copyFolderSync(join(from, file.name), join(to, file.name));+ } else if (file.isFile()) {+ copyFileSync(join(from, file.name), join(to, file.name));+ }+ }+};++copyFolderSync('tab-as-a-container', 'dist/tab-as-a-container');diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 0b48aea5a..01c7e48bb 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -60,6 +60,11 @@ export interface BrowserPluginOptions<LibraryOptions> {* @default false*/useIncognitoPages?: boolean;+ /**+ * @experimental+ * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.+ */+ experimentalContainers?: boolean;/*** Path to a User Data Directory, which stores browser session data like cookies and local storage.*/@@ -99,12 +104,15 @@ export abstract class BrowserPlugin<useIncognitoPages: boolean;+ experimentalContainers: boolean;+constructor(library: Library, options: BrowserPluginOptions<LibraryOptions> = {}) {const {launchOptions = {} as LibraryOptions,proxyUrl,userDataDir,useIncognitoPages = false,+ experimentalContainers = false,} = options;this.library = library;@@ -112,6 +120,7 @@ export abstract class BrowserPlugin<this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);this.userDataDir = userDataDir;this.useIncognitoPages = useIncognitoPages;+ this.experimentalContainers = experimentalContainers;}/**@@ -129,6 +138,7 @@ export abstract class BrowserPlugin<proxyUrl = this.proxyUrl,useIncognitoPages = this.useIncognitoPages,userDataDir = this.userDataDir,+ experimentalContainers = this.experimentalContainers,} = options;return new LaunchContext({@@ -137,6 +147,7 @@ export abstract class BrowserPlugin<browserPlugin: this,proxyUrl,useIncognitoPages,+ experimentalContainers,userDataDir,});}diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.tsindex ab4c1a25c..0d600580c 100644--- a/packages/browser-pool/src/browser-pool.ts+++ b/packages/browser-pool/src/browser-pool.ts@@ -498,7 +498,9 @@ export class BrowserPool<await browserController['isActivePromise'];tryCancel();- const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;+ const finalPageOptions = (browserController.launchContext.useIncognitoPages || browserController.launchContext.experimentalContainers)+ ? pageOptions+ : undefined;if (finalPageOptions) {Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));diff --git a/packages/browser-pool/src/container-proxy-server.ts b/packages/browser-pool/src/container-proxy-server.tsnew file mode 100644index 000000000..434a06b5c--- /dev/null+++ b/packages/browser-pool/src/container-proxy-server.ts@@ -0,0 +1,45 @@+import { Server as ProxyChainServer } from 'proxy-chain';++/**+ * Creates a proxy server designed to handle requests from "container" instances.+ * Each container instance is assigned to a different (but still localhost) IP address+ * in order to work around authorization and to enable upstream.+ * @internal+ */+export async function createProxyServerForContainers() {+ const ipToProxy = new Map<string, string>();++ const proxyServer = new ProxyChainServer({+ prepareRequestFunction({ request }) {+ const prefix4to6 = '::ffff:';+ const localAddress = request.socket.localAddress!.startsWith(prefix4to6)+ ? request.socket.localAddress!.slice(prefix4to6.length)+ : request.socket.localAddress!;++ const upstreamProxyUrl = ipToProxy.get(localAddress);++ if (upstreamProxyUrl === undefined) {+ // eslint-disable-next-line no-console+ console.warn(`Request without proxy ${localAddress} ${request.headers.host}`);+ }++ return {+ upstreamProxyUrl,+ requestAuthentication: false,+ };+ },+ port: 0,+ });++ await proxyServer.listen();++ proxyServer.server.unref();++ return {+ port: proxyServer.port,+ ipToProxy,+ async close(closeConnections: boolean) {+ return proxyServer.close(closeConnections);+ },+ };+};diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.tsindex e526fc978..174884dc4 100644--- a/packages/browser-pool/src/launch-context.ts+++ b/packages/browser-pool/src/launch-context.ts@@ -37,6 +37,11 @@ export interface LaunchContextOptions<* If set to `true` each page uses its own context that is destroyed once the page is closed or crashes.*/useIncognitoPages?: boolean;+ /**+ * @experimental+ * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.+ */+ experimentalContainers?: boolean;/*** Path to a User Data Directory, which stores browser session data like cookies and local storage.*/@@ -55,6 +60,7 @@ export class LaunchContext<browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;launchOptions: LibraryOptions;useIncognitoPages: boolean;+ experimentalContainers: boolean;userDataDir: string;private _proxyUrl?: string;@@ -70,6 +76,7 @@ export class LaunchContext<launchOptions,proxyUrl,useIncognitoPages,+ experimentalContainers,userDataDir = '',} = options;@@ -77,6 +84,7 @@ export class LaunchContext<this.browserPlugin = browserPlugin;this.launchOptions = launchOptions;this.useIncognitoPages = useIncognitoPages ?? false;+ this.experimentalContainers = experimentalContainers ?? false;this.userDataDir = userDataDir;this._proxyUrl = proxyUrl;diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.tsindex da0bc74bd..4e8ce44f7 100644--- a/packages/browser-pool/src/playwright/playwright-controller.ts+++ b/packages/browser-pool/src/playwright/playwright-controller.ts@@ -3,6 +3,10 @@ import { tryCancel } from '@apify/timeout';import type { Cookie } from '@crawlee/types';import { BrowserController } from '../abstract-classes/browser-controller';import { anonymizeProxySugar } from '../anonymize-proxy';+import type { PlaywrightPlugin } from './playwright-plugin';++const tabIds = new WeakMap<Page, number>();+const keyFromTabId = (tabId: string | number) => `.${tabId}.`;export class PlaywrightController extends BrowserController<BrowserType, Parameters<BrowserType['launch']>[0], Browser> {normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown> {@@ -25,13 +29,14 @@ export class PlaywrightController extends BrowserController<BrowserType, Paramet}protected async _newPage(contextOptions?: Parameters<Browser['newPage']>[0]): Promise<Page> {- if (contextOptions !== undefined && !this.launchContext.useIncognitoPages) {- throw new Error('A new page can be created with provided context only when using incognito pages.');+ if (contextOptions !== undefined && !this.launchContext.useIncognitoPages && !this.launchContext.experimentalContainers) {+ throw new Error('A new page can be created with provided context only when using incognito pages or experimental containers.');}// eslint-disable-next-line @typescript-eslint/no-empty-functionlet close = async () => {};- if (contextOptions?.proxy) {++ if (this.launchContext.useIncognitoPages && contextOptions?.proxy) {const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server,contextOptions.proxy.username,@@ -50,7 +55,6 @@ export class PlaywrightController extends BrowserController<BrowserType, Paramettry {const page = await this.browser.newPage(contextOptions);- tryCancel();page.once('close', async () => {this.activePages--;@@ -58,6 +62,45 @@ export class PlaywrightController extends BrowserController<BrowserType, Parametawait close();});+ if (this.launchContext.experimentalContainers) {+ await page.goto('data:text/plain,tabid');+ await page.waitForNavigation();+ const { tabid, proxyip }: { tabid: number; proxyip: string } = JSON.parse(decodeURIComponent(page.url().slice('about:blank#'.length)));++ if (contextOptions?.proxy) {+ const url = new URL(contextOptions.proxy.server);+ url.username = contextOptions.proxy.username ?? '';+ url.password = contextOptions.proxy.password ?? '';++ (this.browserPlugin as PlaywrightPlugin)._containerProxyServer!.ipToProxy.set(proxyip, url.href);+ }++ const session = await page.context().newCDPSession(page);+ await session.send('Network.enable');++ session.on('Network.responseReceived', (responseRecevied) => {+ const logOnly = ['Document', 'XHR', 'Fetch', 'EventSource', 'WebSocket', 'Other'];+ if (!logOnly.includes(responseRecevied.type)) {+ return;+ }++ const { response } = responseRecevied;+ if (response.fromDiskCache || response.fromPrefetchCache || response.fromServiceWorker) {+ return;+ }++ const { remoteIPAddress } = response;+ if (remoteIPAddress && remoteIPAddress !== proxyip) {+ // eslint-disable-next-line no-console+ console.warn(`Request to ${response.url} was through ${remoteIPAddress} instead of ${proxyip}`);+ }+ });++ tabIds.set(page, tabid);+ }++ tryCancel();+return page;} catch (error) {await close();@@ -75,13 +118,48 @@ export class PlaywrightController extends BrowserController<BrowserType, Parametawait this.browser.close(); // Playwright does not have the browser child process attached to normal browser server}- protected _getCookies(page: Page): Promise<Cookie[]> {+ protected async _getCookies(page: Page): Promise<Cookie[]> {const context = page.context();- return context.cookies();+ const cookies = await context.cookies();++ if (this.launchContext.experimentalContainers) {+ const tabId = tabIds.get(page);++ if (tabId === undefined) {+ throw new Error('Failed to find tabId for page');+ }++ const key = keyFromTabId(tabId);++ return cookies+ .filter((cookie) => cookie.name.startsWith(key))+ .map((cookie) => ({+ ...cookie,+ name: cookie.name.slice(key.length),+ }));+ }++ return cookies;}- protected _setCookies(page: Page, cookies: Cookie[]): Promise<void> {+ protected async _setCookies(page: Page, cookies: Cookie[]): Promise<void> {const context = page.context();++ if (this.launchContext.experimentalContainers) {+ const tabId = tabIds.get(page);++ if (tabId === undefined) {+ throw new Error('Failed to find tabId for page');+ }++ const key = keyFromTabId(tabId);++ cookies = cookies.map((cookie) => ({+ ...cookie,+ name: `${key}${cookie.name}`,+ }));+ }+return context.addCookies(cookies);}}diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex 1fb3968e7..c7b2045cd 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -1,3 +1,4 @@+import path from 'path';import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser';import { PlaywrightController } from './playwright-controller';@@ -7,9 +8,15 @@ import type { LaunchContext } from '../launch-context';import { log } from '../logger';import { getLocalProxyAddress } from '../proxy-server';import { anonymizeProxySugar } from '../anonymize-proxy';+import { createProxyServerForContainers } from '../container-proxy-server';++// __dirname = browser-pool/dist/playwright+// taacPath = browser-pool/dist/tab-as-a-container+const taacPath = path.join(__dirname, '..', 'tab-as-a-container');export class PlaywrightPlugin extends BrowserPlugin<BrowserType, Parameters<BrowserType['launch']>[0], PlaywrightBrowser> {private _browserVersion?: string;+ _containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;protected async _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser> {const {@@ -45,8 +52,43 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, Parameters<Brow});}} else {+ if (launchContext.experimentalContainers) {+ launchOptions!.args = [+ ...(launchOptions!.args ?? []),+ ];++ // Use native headless mode so we can load an extension+ if (launchOptions!.headless) {+ launchOptions!.args.push('--headless=chrome');+ }++ launchOptions!.args.push(`--disable-extensions-except=${taacPath}`, `--load-extension=${taacPath}`);+ }+const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions);+ if (launchContext.experimentalContainers) {+ // Wait for the extension to load.+ let [backgroundPage] = browserContext.backgroundPages();+ if (!backgroundPage) {+ backgroundPage = await browserContext.waitForEvent('backgroundpage');+ }++ this._containerProxyServer = await createProxyServerForContainers();++ // @ts-expect-error loading is defined inside background script+ await backgroundPage.evaluate(() => loading);++ const page = await browserContext.newPage();+ await page.goto(`data:text/plain,proxy#{"port":${this._containerProxyServer.port}}`);+ await page.waitForNavigation();+ await page.close();++ browserContext.on('close', async () => {+ await this._containerProxyServer!.close(true);+ });+ }+if (anonymizedProxyUrl) {browserContext.on('close', async () => {await close();diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex 21328e796..105aa5980 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -28,7 +28,7 @@ export class PuppeteerController extends BrowserController<typeof Puppeteer> {protected async _newPage(contextOptions?: Puppeteer.ContextOptions): Promise<Puppeteer.Page> {if (contextOptions !== undefined) {if (!this.launchContext.useIncognitoPages) {- throw new Error('A new page can be created with provided context only when using incognito pages.');+ throw new Error('A new page can be created with provided context only when using incognito pages or experimental containers.');}// eslint-disable-next-line @typescript-eslint/no-empty-functiondiff --git a/packages/browser-pool/tab-as-a-container/background.js b/packages/browser-pool/tab-as-a-container/background.jsnew file mode 100644index 000000000..594c2fc7a--- /dev/null+++ b/packages/browser-pool/tab-as-a-container/background.js@@ -0,0 +1,387 @@+'use strict';++/* eslint-disable no-unused-vars */+/* eslint-disable no-undef */+/* eslint-disable no-empty */++let resolveLoading;+const loading = new Promise((resolve) => {+ resolveLoading = resolve;+});++const isFirefox = navigator.userAgent.includes('Firefox');++const webRequestPermissions = {+ blockingRequest: isFirefox ? ['blocking', 'requestHeaders'] : ['blocking', 'requestHeaders', 'extraHeaders'],+ blockingResponse: isFirefox ? ['blocking', 'responseHeaders'] : ['blocking', 'responseHeaders', 'extraHeaders'],+};++chrome.privacy.network.networkPredictionEnabled.set({ value: false });++const translator = new Map();+const counter = new Map();++const getOpenerId = (id) => {+ if (typeof id !== 'number') {+ throw new Error('Expected `id` to be a number');+ }++ if (translator.has(id)) {+ const opener = translator.get(id);++ if (translator.has(opener)) {+ throw new Error('Opener is not the most ascendent');+ }++ // console.log(`getopener ${id} -> ${opener}`);+ return opener;+ }++ return id;+};++const keyFromTabId = (tabId) => `.${tabId}.`;++const getCookieURL = (cookie) => {+ const protocol = cookie.secure ? 'https:' : 'http:';+ const fixedDomain = cookie.domain[0] === '.' ? cookie.domain.slice(1) : cookie.domain;+ const url = `${protocol}//${fixedDomain}${cookie.path}`;++ return url;+};++// Rewrite cookies that were programatically set to tabId instead of openerId.+// This is requried because we cannot reliably get openerId inside Playwright.+chrome.cookies.onChanged.addListener(async (changeInfo) => {+ if (!changeInfo.removed) {+ const { cookie } = changeInfo;++ if (cookie.name[0] !== '.') {+ return;+ }++ const dotIndex = cookie.name.slice(1).indexOf('.');+ if (dotIndex === -1) {+ return;+ }++ const tabId = Number(cookie.name.slice(0, dotIndex));+ const realCookieName = cookie.name.slice(dotIndex + 1);+ const opener = getOpenerId(tabId);++ if (tabId !== opener) {+ await chrome.cookies.remove({+ name: cookie.name,+ url: getCookieURL(cookie),+ storeId: cookie.storeId,+ });++ delete cookie.hostOnly;+ delete cookie.session;++ await chrome.cookies.set({+ ...cookie,+ name: `${keyFromTabId(opener)}${realCookieName}`,+ url: getCookieURL(cookie),+ });+ }+ }+});++chrome.webRequest.onBeforeSendHeaders.addListener(+ (details) => {+ for (const header of details.requestHeaders) {+ if (header.name.toLowerCase() === 'cookie') {+ const id = keyFromTabId(getOpenerId(details.tabId));++ const fixedCookies = header.value.split('; ').filter((x) => x.startsWith(id)).map((x) => x.slice(id.length)).join('; ');+ header.value = fixedCookies;+ }++ // Sometimes Chrome makes a request on a ghost tab.+ // We don't want these in order to prevent cluttering cookies.+ // Yes, `webNavigation.onComitted` is emitted and `webNavigation.onCreatedNavigationTarget` is not.+ if (header.name.toLowerCase() === 'purpose' && header.value === 'prefetch' && !(counter.has(details.tabId))) {+ // eslint-disable-next-line no-console+ console.log(details);+ return {+ cancel: true,+ };+ }+ }++ return {+ requestHeaders: details.requestHeaders.filter((header) => header.name !== 'cookie' || header.value !== ''),+ };+ },+ { urls: ['<all_urls>'] },+ webRequestPermissions.blockingRequest,+);++// Firefox Bug: doesn't catch https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-uri+chrome.webRequest.onHeadersReceived.addListener(+ (details) => {+ for (const header of details.responseHeaders) {+ if (header.name.toLowerCase() === 'set-cookie') {+ const parts = header.value.split('\n');++ // `details.tabId` === -1 when Chrome is making internal requests, such downloading a service worker.++ const openerId = getOpenerId(details.tabId);++ header.value = parts.map((part) => {+ const equalsIndex = part.indexOf('=');+ if (equalsIndex === -1) {+ return `${keyFromTabId(openerId)}=${part.trimStart()}`;+ }+ return keyFromTabId(openerId) + part.trimStart();+ }).join('\n');+ }+ }++ return {+ responseHeaders: details.responseHeaders,+ };+ },+ { urls: ['<all_urls>'] },+ webRequestPermissions.blockingResponse,+);++chrome.tabs.onRemoved.addListener(async (tabId) => {+ const opener = getOpenerId(tabId);+ translator.delete(tabId);++ if (counter.has(opener)) {+ counter.set(opener, counter.get(opener) - 1);++ if (counter.get(opener) < 1) {+ counter.delete(opener);+ } else {+ return;+ }+ }++ const id = keyFromTabId(opener);++ chrome.cookies.getAll({}, async (cookies) => {+ await Promise.allSettled(cookies.filter((cookie) => cookie.name.startsWith(id)).map((cookie) => {+ return chrome.cookies.remove({+ name: cookie.name,+ url: getCookieURL(cookie),+ storeId: cookie.storeId,+ });+ }));+ });+});++// Proxy per tab+const getProxyConfiguration = (scheme, host, port) => {+ return {+ mode: 'fixed_servers',+ rules: {+ proxyForHttp: {+ scheme,+ host,+ port,+ },+ proxyForHttps: {+ scheme,+ host,+ port,+ },+ },+ };+};++const localhostIpCache = new Map();+const localHostIp = [127, 0, 0, 1];+const getNextLocalhostIp = (openerId) => {+ if (localhostIpCache.has(openerId)) {+ return localhostIpCache.get(openerId);+ }++ const result = localHostIp.join('.');++ localhostIpCache.set(openerId, result);++ if (localHostIp[3] === 254) {+ if (localHostIp[2] === 255) {+ if (localHostIp[1] === 255) {+ localHostIp[1] = 0;+ } else {+ localHostIp[1]++;+ }++ localHostIp[2] = 0;+ } else {+ localHostIp[2]++;+ }++ localHostIp[3] = 1;+ } else {+ localHostIp[3]++;+ }++ // [127.0.0.1 - 127.255.255.254] = 1 * 255 * 255 * 254 = 16 516 350+ while (localhostIpCache.length >= (1 * 255 * 255 * 254)) {+ localhostIpCache.delete(localhostIpCache.keys().next().value);+ }++ return result;+};++let proxyPort;++// Clear extension's proxy settings on reload+if (isFirefox) {+ browser.proxy.settings.clear({});+} else {+ chrome.proxy.settings.clear({});+}++// Proxy per tab+if (isFirefox) {+ // On Firefox, we could use the `dns` permission to enforce DoH+ // but then the extension would not be compatible with Chrome.+ // Therefore users need to manually set the DNS settings.++ browser.proxy.onRequest.addListener(+ (details) => {+ const openerId = getOpenerId(details.tabId);++ if (typeof proxyPort === 'number') {+ return {+ type: 'http',+ host: getNextLocalhostIp(openerId),+ port: proxyPort,+ };+ }+ return {+ type: 'direct',+ };+ },+ { urls: ['<all_urls>'] },+ );+} else {+ // The connection is not yet created with `onBeforeSendHeaders`, but is with `onSendHeaders`.+ chrome.webRequest.onBeforeSendHeaders.addListener(+ (details) => {+ const openerId = getOpenerId(details.tabId);++ if (typeof proxyPort === 'number') {+ chrome.proxy.settings.set({+ value: getProxyConfiguration('http', getNextLocalhostIp(openerId), proxyPort),+ scope: 'regular',+ });+ } else {+ chrome.proxy.settings.clear({});+ }+ },+ { urls: ['<all_urls>'] },+ webRequestPermissions.blockingRequest,+ );+}++// External communication. Note: the JSON keys are lowercased by the browser.+const routes = Object.assign(Object.create(null), {+ async tabid(details) {+ return { tabid: details.tabId, proxyip: getNextLocalhostIp(details.tabId) };+ },+ async proxy(details, body) {+ proxyPort = body.port;++ return '';+ },+});++chrome.webNavigation.onCompleted.addListener(async (details) => {+ const textPlain = 'data:text/plain,';++ if (details.frameId === 0 && details.url.startsWith(textPlain)) {+ try {+ const url = new URL(details.url);+ const route = url.pathname.slice('text/plain,'.length);++ if (route in routes) {+ const hash = url.hash.slice(1);++ let body = {};++ if (hash !== '') {+ try {+ body = JSON.parse(decodeURIComponent(hash));+ } catch {}+ }++ // Different protocols are required, otherwise `onCompleted` won't be emitted.+ const result = await routes[route](details, body);+ if (result !== undefined) {+ await chrome.tabs.update(details.tabId, { url: `about:blank#${encodeURIComponent(JSON.stringify(result))}` });+ }+ }+ } catch {+ // Invalid URL, ignore.+ }+ }+});++// Load content scripts.+(async () => {+ const contentResponse = await fetch(chrome.runtime.getURL('content.js'));+ const contentText = await contentResponse.text();++ // `tabs.onCreated` doesn't work here when manually creating new tabs,+ // because the opener is the current tab active.+ //+ // This events only fires when the page opens something.+ chrome.webNavigation.onCreatedNavigationTarget.addListener((details) => {+ translator.set(details.tabId, getOpenerId(details.sourceTabId));++ const opener = getOpenerId(details.tabId);++ if (counter.has(opener)) {+ counter.set(opener, counter.get(opener) + 1);+ } else {+ counter.set(opener, 2); // the current one + opener = 2+ }+ });++ chrome.webNavigation.onCommitted.addListener(async (details) => {+ if (details.url.startsWith('chrome')) {+ return;+ }++ const executeCodeInPageContext = `+ const script = document.createElement('script');+ script.textContent = code;++ const destination = document.head ?? document.documentElement;++ if (document instanceof HTMLDocument) {+ destination.append(script);+ script.remove();+ }+ `;++ // Race condition: website scripts may run first+ await chrome.tabs.executeScript(details.tabId, {+ code: `'use strict';+ (() => {+ if (window.totallyRandomString) {+ return;+ }++ window.totallyRandomString = true;++ const code = "'use strict'; const tabId = '${getOpenerId(details.tabId)}'; (() => {\\n" + ${JSON.stringify(contentText)} + "\\n})();\\n";+ ${executeCodeInPageContext}+ })();+ `,+ matchAboutBlank: true,+ allFrames: true,+ runAt: 'document_start',+ });+ });++ resolveLoading();+})();diff --git a/packages/browser-pool/tab-as-a-container/content.js b/packages/browser-pool/tab-as-a-container/content.jsnew file mode 100644index 000000000..4635bf27d--- /dev/null+++ b/packages/browser-pool/tab-as-a-container/content.js@@ -0,0 +1,593 @@+// When in doubt, refer to https://github.com/nodejs/node/blob/main/doc/contributing/primordials.md++/* eslint-disable no-undef */+/* eslint-disable no-cond-assign */+/* eslint-disable prefer-rest-params */+/* eslint-disable no-shadow */++// TODO: https://developer.mozilla.org/en-US/docs/Web/API/Cookie_Store_API+// TODO: custom error messages for Firefox (for now it uses Chrome's)++// The only way to detect this "container" is to benchmark document.cookie or compare localStorage performance with sessionStorage (it's the same).++const isFirefox = navigator.userAgent.includes('Firefox');+const tabPrefix = `.${tabId}.`;++const {+ String,+ Array,+ Set,+ TypeError,+ WeakMap,+ Object,+ Number,+ Function,+ Proxy,+ IDBFactory,+ IDBDatabase,+ BroadcastChannel,+ Storage,+ // We don't have to implement StorageEvent because this implementation does not use localStorage at all.+} = globalThis;++const ObjectDefineProperty = Object.defineProperty;+const ObjectDefineProperties = Object.defineProperties;+const ObjectGetOwnPropertyDescriptors = Object.getOwnPropertyDescriptors;+const ObjectGetPrototypeOf = Object.getPrototypeOf;+const ObjectGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;+const ObjectCreate = Object.create;+const ObjectEntries = Object.entries;+const ReflectGet = Reflect.get;+const ReflectSet = Reflect.set;+const ObjectKeys = Object.keys;+const NumberIsFinite = Number.isFinite;++const clonePrototype = (from) => {+ const target = ObjectCreate(null);+ const prototype = ObjectGetOwnPropertyDescriptors(from.prototype);++ const entries = ObjectEntries(prototype);++ for (let i = 0; i < entries.length; i++) {+ const entry = entries[i];++ const { 0: name, 1: descriptor } = entry;+ target[name] = ObjectCreate(null);++ if ('get' in descriptor) {+ target[name].get = descriptor.get;+ }++ if ('set' in descriptor) {+ target[name].set = descriptor.set;+ }++ if ('value' in descriptor) {+ target[name] = descriptor.value;+ }+ }++ return target;+};++const StringSplitSafe = (string, separator) => {+ const result = [];+ const separatorLength = separator.length;++ if (separatorLength === 0) {+ throw new Error('Separator must not be empty');+ }++ let startFrom = 0;+ let index;+ while ((index = StringPrototype.indexOf.call(string, separator, startFrom)) !== -1) {+ ArrayPrototype.push.call(result, StringPrototype.slice.call(string, startFrom, index));++ startFrom = index + separatorLength;+ }++ const lastChunk = StringPrototype.slice.call(string, startFrom);++ ArrayPrototype.push.call(result, lastChunk);++ return result;+};++const fixStack = (error) => {+ const lines = StringSplitSafe(error.stack, '\n');++ if (isFirefox) {+ ArrayPrototype.splice.call(lines, 0, 1);+ } else {+ ArrayPrototype.splice.call(lines, 1, 1);+ }++ error.stack = ArrayPrototype.join.call(lines, '\n');++ return error;+};++const SetPrototype = clonePrototype(Set);+const WeakMapPrototype = clonePrototype(WeakMap);+const ArrayPrototype = clonePrototype(Array);+const StringPrototype = clonePrototype(String);+const IDBFactoryPrototype = clonePrototype(IDBFactory);+const IDBDatabasePrototype = clonePrototype(IDBDatabase);+const StoragePrototype = clonePrototype(Storage);++const privates = new WeakMap();++let invocable = false;++const FakeStorage = class Storage {+ constructor() {+ if (invocable) {+ throw fixStack(new TypeError('Illegal constructor'));+ }++ WeakMapPrototype.set.call(privates, this, arguments[0]);+ }++ get length() {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ const { storage, prefix } = priv;+ const length = StoragePrototype.length.get.call(storage);++ let fakeLength = 0;+ for (let i = 0; i < length; i++) {+ const storageKey = StoragePrototype.key.call(storage, i);+ if (StringPrototype.startsWith.call(storageKey, prefix)) {+ fakeLength++;+ }+ }++ return fakeLength;+ }++ clear() {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ const { storage, prefix } = priv;+ const length = StoragePrototype.length.get.call(storage);+ const keys = [];++ for (let i = 0; i < length; i++) {+ ArrayPrototype.push.call(keys, StoragePrototype.key.call(storage, i));+ }++ for (let i = 0; i < length; i++) {+ const storageKey = keys[i];+ if (StringPrototype.startsWith.call(storageKey, prefix)) {+ StoragePrototype.removeItem.call(storage, storageKey);+ }+ }+ }++ key(index) {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ if (arguments.length === 0) {+ throw fixStack(new TypeError(`Failed to execute 'key' on 'Storage': 1 argument required, but only 0 present.`));+ }++ index = NumberIsFinite(index) ? index : 0;++ const { storage, prefix } = priv;+ const length = StoragePrototype.length.get.call(storage);++ let fakeLength = 0;+ for (let i = 0; i < length; i++) {+ const storageKey = StoragePrototype.key.call(storage, i);++ if (StringPrototype.startsWith.call(storageKey, prefix)) {+ if (fakeLength === index) {+ return StringPrototype.slice.call(storageKey, prefix.length);+ }++ fakeLength++;+ }+ }++ return null;+ }++ getItem(key) {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ if (arguments.length === 0) {+ throw fixStack(new TypeError(`Failed to execute 'getItem' on 'Storage': 1 argument required, but only 0 present.`));+ }++ return StoragePrototype.getItem.call(priv.storage, priv.prefix + key);+ }++ removeItem(key) {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ if (arguments.length === 0) {+ throw fixStack(new TypeError(`Failed to execute 'removeItem' on 'Storage': 1 argument required, but only 0 present.`));+ }++ StoragePrototype.removeItem.call(priv.storage, priv.prefix + key);+ }++ setItem(key, value) {+ const priv = WeakMapPrototype.get.call(privates, this);+ if (!priv) {+ throw fixStack(new TypeError('Illegal invocation'));+ }++ if (arguments.length === 0 || arguments.length === 1) {+ throw fixStack(new TypeError(`Failed to execute 'setItem' on 'Storage': 2 arguments required, but only ${arguments.length} present.`));+ }++ StoragePrototype.setItem.call(priv.storage, priv.prefix + key, value);+ }+};++const FakeStoragePrototype = clonePrototype(FakeStorage);++const createStorage = ({ storage, prefix }) => {+ invocable = false;+ const fake = new FakeStorage({ storage, prefix });+ invocable = true;++ const proxy = new Proxy(fake, {+ __proto__: null,+ // Default:+ // apply: (target, thisArg, args) => {},+ // construct(target, args) => {},+ // setPrototypeOf: (target, proto) => {},+ // getPrototypeOf: (target) => {},+ defineProperty: (target, key, descriptor) => {+ if ('set' in descriptor || 'get' in descriptor) {+ throw fixStack(new TypeError(`Failed to set a named property on 'Storage': Accessor properties are not allowed.`));+ }++ FakeStoragePrototype.setItem.call(target, key, descriptor.value);+ },+ deleteProperty: (target, key) => {+ if (typeof key === 'symbol') {+ delete target[key];+ } else {+ FakeStoragePrototype.removeItem.call(target, key);+ }++ return true;+ },+ get: (target, key) => {+ if (typeof key === 'symbol') {+ return target[key];+ }++ if (key in target) {+ return ReflectGet(target, key);+ }++ return FakeStoragePrototype.getItem.call(target, key) ?? undefined;+ },+ set: (target, key, value) => {+ if (typeof key === 'symbol') {+ ObjectDefineProperty(target, key, {+ __proto__: null,+ value,+ configurable: true,+ writable: true,+ enumerable: false,+ });++ return true;+ }++ if (key in target) {+ return ReflectSet(target, key, value);+ }++ return FakeStoragePrototype.setItem.call(target, key, value) ?? true;+ },+ has: (target, key) => {+ if (key in target) {+ return true;+ }++ return FakeStoragePrototype.getItem.call(target, key) !== null;+ },+ isExtensible: () => {+ return true;+ },+ preventExtensions: () => {+ throw fixStack(new TypeError(`Cannot prevent extensions`));+ },+ getOwnPropertyDescriptor: (target, key) => {+ if (key in target) {+ return ObjectGetOwnPropertyDescriptor(ObjectGetPrototypeOf(target), key);+ }++ const value = FakeStoragePrototype.getItem.call(target, key);++ if (value !== null) {+ return {+ value,+ writable: true,+ enumerable: true,+ configurable: true,+ };+ }+ },+ ownKeys: (target) => {+ const keys = [];++ const { storage, prefix } = WeakMapPrototype.get.call(privates, target);+ const length = StoragePrototype.length.get.call(storage);++ for (let i = 0; i < length; i++) {+ const storageKey = StoragePrototype.key.call(storage, i);++ if (StringPrototype.startsWith.call(storageKey, prefix)) {+ ArrayPrototype.push.call(keys, StringPrototype.slice.call(storageKey, prefix.length));+ }+ }++ ArrayPrototype.push.apply(keys, ObjectKeys(target));++ const set = new Set();++ for (let i = 0; i < keys.length; i++) {+ SetPrototype.add.call(set, keys[i]);+ }++ return ArrayPrototype.slice.call(set);+ },+ });++ privates.set(proxy, privates.get(fake));++ return proxy;+};++const toHide = new WeakMap();+for (const Type of [Function, Object, Array]) {+ const create = (fallback) => function () {+ if (this instanceof FakeStorage) {+ return '[object Storage]';+ }++ if (WeakMapPrototype.has.call(toHide, this)) {+ return `function ${WeakMapPrototype.get.call(toHide, this)}() { [native code] }`;+ }++ return fallback.call(this);+ };++ const toString = create(Type.prototype.toString);+ const toLocaleString = create(Type.prototype.toLocaleString);++ WeakMapPrototype.set.call(toHide, toString, 'toString');+ WeakMapPrototype.set.call(toHide, toLocaleString, 'toLocaleString');++ Object.defineProperty(Type.prototype, 'toString', {+ __proto__: null,+ value: toString,+ });+ Object.defineProperty(Type.prototype, 'toLocaleString', {+ __proto__: null,+ value: toLocaleString,+ });+}++// https://stackoverflow.com/q/30481516+try {+ // We use sessionStorage as the underlying storage for localStorage.+ // This way we do not have to worry about clean up.+ const { sessionStorage } = globalThis;++ const fakeLocalStorage = createStorage({ storage: sessionStorage, prefix: 'l.' });+ const fakeSessionStorage = createStorage({ storage: sessionStorage, prefix: 's.' });++ const getLocalStorage = function localStorage() { return fakeLocalStorage; };+ const getSessionStorage = function sessionStorage() { return fakeSessionStorage; };++ WeakMapPrototype.set.call(toHide, FakeStorage, 'Storage');+ WeakMapPrototype.set.call(toHide, FakeStoragePrototype.key, 'key');+ WeakMapPrototype.set.call(toHide, FakeStoragePrototype.getItem, 'getItem');+ WeakMapPrototype.set.call(toHide, FakeStoragePrototype.setItem, 'setItem');+ WeakMapPrototype.set.call(toHide, FakeStoragePrototype.removeItem, 'removeItem');+ WeakMapPrototype.set.call(toHide, FakeStoragePrototype.clear, 'clear');+ WeakMapPrototype.set.call(toHide, getLocalStorage, 'get localStorage');+ WeakMapPrototype.set.call(toHide, getSessionStorage, 'get sessionStorage');++ ObjectDefineProperties(window, {+ __proto__: null,+ Storage: {+ __proto__: null,+ value: FakeStorage,+ configurable: true,+ enumerable: false,+ writable: true,+ },+ localStorage: {+ __proto__: null,+ configurable: true,+ enumerable: true,+ get: getLocalStorage,+ set: undefined,+ },+ sessionStorage: {+ __proto__: null,+ configurable: true,+ enumerable: true,+ get: getSessionStorage,+ set: undefined,+ },+ });+} catch (error) {+ // eslint-disable-next-line no-console+ console.error(error);+}++{+ const { Document } = globalThis;++ const realGetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').get;+ const realSetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').set;++ const getCookie = function cookie() {+ try {+ const cookies = StringSplitSafe(realGetCookie.call(this), '; ');+ const filtered = ArrayPrototype.filter.call(cookies, (cookie) => StringPrototype.startsWith.call(cookie, tabPrefix));+ const mapped = ArrayPrototype.map.call(filtered, (cookie) => {+ const result = StringPrototype.slice.call(cookie, tabPrefix.length);++ if (result[0] === '=') {+ return StringPrototype.slice.call(result, 1);+ }++ return result;+ });++ return ArrayPrototype.join.call(mapped, '; ');+ } catch (error) {+ throw fixStack(error);+ }+ };++ const setCookie = function cookie(cookieString) {+ cookieString = StringPrototype.trimStart.call(String(cookieString));++ const delimiterIndex = StringPrototype.indexOf.call(cookieString, ';');+ const equalsIndex = StringPrototype.indexOf.call(cookieString, '=');+ if ((equalsIndex === -1) || ((delimiterIndex !== -1) && (equalsIndex > delimiterIndex))) {+ cookieString = `=${cookieString}`;+ }++ try {+ realSetCookie.call(this, tabPrefix + cookieString);+ } catch (error) {+ throw fixStack(error);+ }+ };++ WeakMapPrototype.set.call(toHide, getCookie, 'get cookie');+ WeakMapPrototype.set.call(toHide, setCookie, 'set cookie');++ ObjectDefineProperty(Document.prototype, 'cookie', {+ __proto__: null,+ configurable: true,+ enumerable: true,+ get: getCookie,+ set: setCookie,+ });+}++{+ const openDatabase = function open(name) {+ try {+ return IDBFactoryPrototype.open.call(this, tabPrefix + name);+ } catch (error) {+ throw fixStack(error);+ }+ };++ const deleteDatabase = function deleteDatabase(name) {+ try {+ return IDBFactoryPrototype.deleteDatabase.call(this, tabPrefix + name);+ } catch (error) {+ throw fixStack(error);+ }+ };++ const databaseName = function name() {+ try {+ return StringPrototype.slice.call(IDBDatabasePrototype.name.get.call(this), tabPrefix.length);+ } catch (error) {+ throw fixStack(error);+ }+ };++ WeakMapPrototype.set.call(toHide, openDatabase, 'open');+ WeakMapPrototype.set.call(toHide, deleteDatabase, 'deleteDatabase');+ WeakMapPrototype.set.call(toHide, databaseName, 'get name');++ ObjectDefineProperties(IDBFactory.prototype, {+ __proto__: null,+ open: {+ __proto__: null,+ writable: true,+ configurable: true,+ enumerable: true,+ value: openDatabase,+ },+ deleteDatabase: {+ __proto__: null,+ writable: true,+ configurable: true,+ enumerable: true,+ value: deleteDatabase,+ },+ name: {+ __proto__: null,+ configurable: true,+ enumerable: true,+ get: databaseName,+ set: undefined,+ },+ });+}++{+ ObjectDefineProperty(window, 'BroadcastChannel', {+ __proto__: null,+ configurable: true,+ enumerable: false,+ writable: true,+ value: new Proxy(BroadcastChannel, {+ __proto__: null,+ construct: (Target, name) => {+ return new Target(tabPrefix + name);+ },+ }),+ });++ WeakMapPrototype.set.call(toHide, window.BroadcastChannel, 'BroadcastChannel');++ const getBroadcastChannelName = ObjectGetOwnPropertyDescriptor(BroadcastChannel.prototype, 'name').get;+ const broadcastChannelName = function name() {+ try {+ const realName = getBroadcastChannelName.call(this);++ if (StringPrototype.startsWith.call(realName, tabPrefix)) {+ return StringPrototype.slice.call(realName, tabPrefix.length);+ }++ return realName;+ } catch (error) {+ throw fixStack(error);+ }+ };++ WeakMapPrototype.set.call(toHide, broadcastChannelName, 'get name');++ ObjectDefineProperty(BroadcastChannel.prototype, 'name', {+ __proto__: null,+ configurable: true,+ enumerable: true,+ get: broadcastChannelName,+ set: undefined,+ });+}diff --git a/packages/browser-pool/tab-as-a-container/manifest.json b/packages/browser-pool/tab-as-a-container/manifest.jsonnew file mode 100644index 000000000..b7a38b8ec--- /dev/null+++ b/packages/browser-pool/tab-as-a-container/manifest.json@@ -0,0 +1,25 @@+{+ "manifest_version": 2,+ "name": "Tab as a Container",+ "version": "1.0.0",+ "background": {+ "scripts": [+ "background.js"+ ],+ "persistent": true+ },+ "permissions": [+ "webRequest",+ "webRequestBlocking",+ "webNavigation",+ "tabs",+ "cookies",+ "privacy",+ "proxy",+ "<all_urls>"+ ],+ "web_accessible_resources": [+ "content.js"+ ],+ "incognito": "not_allowed"+}diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 61b8d0fd9..a19c67f7f 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -196,6 +196,7 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwrithrow new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.'+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');}+const playwrightLauncher = new PlaywrightLauncher(launchContext, config);browserPoolOptions.browserPlugins = [diff --git a/packages/playwright-crawler/src/internals/playwright-launcher.ts b/packages/playwright-crawler/src/internals/playwright-launcher.tsindex c2797e453..9dabd57d7 100644--- a/packages/playwright-crawler/src/internals/playwright-launcher.ts+++ b/packages/playwright-crawler/src/internals/playwright-launcher.ts@@ -55,6 +55,12 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti*/useIncognitoPages?: boolean;+ /**+ * @experimental+ * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading.+ */+ experimentalContainers?: boolean;+/*** Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.tsindex 329addac6..070465c6d 100644--- a/test/browser-pool/browser-plugins/plugins.test.ts+++ b/test/browser-pool/browser-plugins/plugins.test.ts@@ -130,7 +130,7 @@ const runPluginTest = <expect(false).toBe(true);} catch (error: any) {- expect(error.message).toBe('A new page can be created with provided context only when using incognito pages.');+ expect(error.message).toBe('A new page can be created with provided context only when using incognito pages or experimental containers.');}});});diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore b/test/e2e/playwright-chromium-experimental-containers/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfilenew file mode 100644index 000000000..2a1510bd0--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile@@ -0,0 +1,22 @@+FROM node:16 AS builder++COPY /packages ./packages+COPY /package*.json ./+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional++FROM apify/actor-node-playwright-chrome:16++RUN rm -r node_modules+COPY --from=builder /node_modules ./node_modules+COPY --from=builder /packages ./packages+COPY --from=builder /package*.json ./+COPY /apify.json ./+COPY /main.js ./++RUN echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --versiondiff --git a/test/e2e/playwright-chromium-experimental-containers/actor/apify.json b/test/e2e/playwright-chromium-experimental-containers/actor/apify.jsonnew file mode 100644index 000000000..7f43847ae--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-playwright-chromium-experimental-containers",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/main.js b/test/e2e/playwright-chromium-experimental-containers/actor/main.jsnew file mode 100644index 000000000..7a1f860a9--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/actor/main.js@@ -0,0 +1,26 @@+import { Actor } from 'apify';+import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';+import { ApifyStorageLocal } from '@apify/storage-local';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const crawler = new PlaywrightCrawler({+ proxyConfiguration: await Actor.createProxyConfiguration(),+ launchContext: {+ experimentalContainers: true,+ },+ preNavigationHooks: [(_ctx, goToOptions) => {+ goToOptions.waitUntil = 'networkidle';+ }],+ async requestHandler({ page }) {+ const content = await page.content();+ await Dataset.pushData({ ip: content.match(/"clientIp":\s*"(.*)"/)?.[1] });+ },+ });++ await crawler.run(['https://api.apify.com/v2/browser-info?1', 'https://api.apify.com/v2/browser-info?2']);+}, mainOptions);diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/package.json b/test/e2e/playwright-chromium-experimental-containers/actor/package.jsonnew file mode 100644index 000000000..5ca93dd8c--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/actor/package.json@@ -0,0 +1,21 @@+{+ "name": "test-playwright-chromium-experimental-containers",+ "version": "0.0.1",+ "description": "Playwright Test - Chromium - Experimental containers",+ "dependencies": {+ "apify": "^3.0.0-beta.75",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser": "file:./packages/browser-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/playwright": "file:./packages/playwright-crawler",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "playwright": "*"+ },+ "scripts": { "start": "node main.js" },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/playwright-chromium-experimental-containers/test.mjs b/test/e2e/playwright-chromium-experimental-containers/test.mjsnew file mode 100644index 000000000..5af3d086e--- /dev/null+++ b/test/e2e/playwright-chromium-experimental-containers/test.mjs@@ -0,0 +1,16 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname, 16384);++await expect(datasetItems.length > 0, 'Has dataset items');++const ips = new Set();++for (const { ip } of datasetItems) {+ await expect(!ips.has(ip), 'Unique proxy ip');++ ips.add(ip);+}