Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(puppeteer): support `puppeteer@v22` (#2337)
Martin Adámek3cc360a1ea94
diff --git a/package.json b/package.jsonindex 96d74488c..3c0489b4b 100644--- a/package.json+++ b/package.json@@ -100,7 +100,7 @@"playwright": "1.41.2","portastic": "^1.0.1","proxy": "^1.0.2",- "puppeteer": "21.10.0",+ "puppeteer": "22.0.0","rimraf": "^5.0.0","tsx": "^4.4.0","turbo": "1.12.4",diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex 79248061c..4522b9317 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -1,5 +1,7 @@import { tryCancel } from '@apify/timeout';import type { Cookie } from '@crawlee/types';+// @ts-expect-error not exposed on type level+import { CdpBrowser } from 'puppeteer';import type Puppeteer from 'puppeteer';import type * as PuppeteerTypes from 'puppeteer';@@ -61,7 +63,9 @@ export class PuppeteerController extends BrowserController<}try {- const context = await this.browser.createIncognitoBrowserContext(contextOptions);+ const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;+ const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';+ const context = await (this.browser as any)[method](contextOptions) as PuppeteerTypes.BrowserContext;tryCancel();const page = await context.newPage();tryCancel();diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex e7c9ae5c1..8bae5aefa 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -1,6 +1,8 @@import type { Dictionary } from '@crawlee/types';import type Puppeteer from 'puppeteer';import type * as PuppeteerTypes from 'puppeteer';+// @ts-expect-error not exposed on type level+import { CdpBrowser } from 'puppeteer';import type { PuppeteerNewPageOptions } from './puppeteer-controller';import { PuppeteerController } from './puppeteer-controller';@@ -22,6 +24,7 @@ export class PuppeteerPlugin extends BrowserPlugin<protected async _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,): Promise<PuppeteerTypes.Browser> {+ const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;const {launchOptions,userDataDir,@@ -44,8 +47,8 @@ export class PuppeteerPlugin extends BrowserPlugin<}}- if (launchOptions!.headless === true) {- launchOptions!.headless = 'new';+ if (launchOptions!.headless === true && oldPuppeteerVersion) {+ launchOptions!.headless = 'new' as any;}let browser: PuppeteerTypes.Browser;@@ -98,11 +101,12 @@ export class PuppeteerPlugin extends BrowserPlugin<}});- const boundMethods = (['newPage', 'close', 'userAgent', 'createIncognitoBrowserContext', 'version', 'on', 'process'] as const)+ const boundMethods = (['newPage', 'close', 'userAgent', 'createIncognitoBrowserContext', 'createBrowserContext', 'version', 'on', 'process'] as const).reduce((map, method) => {- map[method] = browser[method]?.bind(browser);+ map[method] = browser[method as 'close']?.bind(browser);return map;}, {} as Dictionary);+ const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';browser = new Proxy(browser, {get: (target, property: keyof typeof browser, receiver) => {@@ -114,9 +118,9 @@ export class PuppeteerPlugin extends BrowserPlugin<const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);try {- const context = await browser.createIncognitoBrowserContext({+ const context = await (browser as any)[method]({proxyServer: anonymizedProxyUrl ?? proxyUrl,- });+ }) as PuppeteerTypes.BrowserContext;page = await context.newPage(...args);diff --git a/yarn.lock b/yarn.lockindex 2772b3ce0..902e9b48f 100644--- a/yarn.lock+++ b/yarn.lock@@ -694,7 +694,7 @@ __metadata:playwright: "npm:1.41.2"portastic: "npm:^1.0.1"proxy: "npm:^1.0.2"- puppeteer: "npm:21.10.0"+ puppeteer: "npm:22.0.0"rimraf: "npm:^5.0.0"tsx: "npm:^4.4.0"turbo: "npm:1.12.4"@@ -1558,9 +1558,9 @@ __metadata:languageName: nodelinkType: hard-"@puppeteer/browsers@npm:1.9.1":- version: 1.9.1- resolution: "@puppeteer/browsers@npm:1.9.1"+"@puppeteer/browsers@npm:2.0.0":+ version: 2.0.0+ resolution: "@puppeteer/browsers@npm:2.0.0"dependencies:debug: "npm:4.3.4"extract-zip: "npm:2.0.1"@@ -1571,7 +1571,7 @@ __metadata:yargs: "npm:17.7.2"bin:browsers: lib/cjs/main-cli.js- checksum: 10c0/8cbde5ec8060c2bdfdc0d7149711cf0cccc6648ed61d0b969cd305918108e7973ce8a7ff3f7533fcc3c49552afe1955755cd7a3924c1fdbea7330d48094b35ee+ checksum: 10c0/72e17739f43ad4c56d8b68249736ff303c0ebdfd10d7426f7084740c221fd0584034ad340cc4709dfec13ad497f5e93a619a9a7ee84729d5497cb7096f01b94blanguageName: nodelinkType: hard@@ -3515,15 +3515,15 @@ __metadata:languageName: nodelinkType: hard-"chromium-bidi@npm:0.5.6":- version: 0.5.6- resolution: "chromium-bidi@npm:0.5.6"+"chromium-bidi@npm:0.5.8":+ version: 0.5.8+ resolution: "chromium-bidi@npm:0.5.8"dependencies:mitt: "npm:3.0.1"urlpattern-polyfill: "npm:10.0.0"peerDependencies:devtools-protocol: "*"- checksum: 10c0/076fe42ed4eec6361a0c100bd17b91747ccc064cb89114431568c99daebf4b1f38c6466a4a2d17e7e08badc2a907f53245231f39344419a0f5c35c856f6ff717+ checksum: 10c0/30a1827a6fd2e57fa1c07c7bc209c10b9e9748872b8403b901ca8ebdc7cc1ca879db223810567c5bf057c98fde5299631e58bfbc29cf7366ae9b38b215af3086languageName: nodelinkType: hard@@ -9755,30 +9755,30 @@ __metadata:languageName: nodelinkType: hard-"puppeteer-core@npm:21.10.0":- version: 21.10.0- resolution: "puppeteer-core@npm:21.10.0"+"puppeteer-core@npm:22.0.0":+ version: 22.0.0+ resolution: "puppeteer-core@npm:22.0.0"dependencies:- "@puppeteer/browsers": "npm:1.9.1"- chromium-bidi: "npm:0.5.6"+ "@puppeteer/browsers": "npm:2.0.0"+ chromium-bidi: "npm:0.5.8"cross-fetch: "npm:4.0.0"debug: "npm:4.3.4"devtools-protocol: "npm:0.0.1232444"ws: "npm:8.16.0"- checksum: 10c0/301e4430d22b4659b77bfffe02bf3ee0ddf0fa095a44e782f17b473f53b2fb015f2deed080b4acfa4b75a50b889fe2295df7ce82eebd161bfbeb9d749529eaad+ checksum: 10c0/e6eee1a02e3c69a4f427f1a7924b363f41c00fe0faff02ac21459cd2ab306dad24284cb04635f46cdfebef4a8e335530816b361b6a6067cb288a07a2fae693cflanguageName: nodelinkType: hard-"puppeteer@npm:21.10.0":- version: 21.10.0- resolution: "puppeteer@npm:21.10.0"+"puppeteer@npm:22.0.0":+ version: 22.0.0+ resolution: "puppeteer@npm:22.0.0"dependencies:- "@puppeteer/browsers": "npm:1.9.1"+ "@puppeteer/browsers": "npm:2.0.0"cosmiconfig: "npm:9.0.0"- puppeteer-core: "npm:21.10.0"+ puppeteer-core: "npm:22.0.0"bin:puppeteer: lib/esm/puppeteer/node/cli.js- checksum: 10c0/a12b0352f84d78e1c379b3efab4c3a98394a1e051c5675ee59c23ad4812d127ccf9b679ad379019d113ab2da8128c469b5dda41059527ec8a83793c232d28347+ checksum: 10c0/8c2909db2c9c030f2a26f9179b5150e836941c676db57ec360d1389f981d2dcde2a66ecb242bffb375d3c7303827c4ee5b4148eda165ec834564cfadd0dcd963languageName: nodelinkType: hard