Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(puppeteer): support `puppeteer@v22` (#2337)

Martin Adámek3cc360a1ea94
diff --git a/package.json b/package.json
index 96d74488c..3c0489b4b 100644
--- a/package.json
+++ b/package.json
@@ -100,7 +100,7 @@
"playwright": "1.41.2",
"portastic": "^1.0.1",
"proxy": "^1.0.2",
- "puppeteer": "21.10.0",
+ "puppeteer": "22.0.0",
"rimraf": "^5.0.0",
"tsx": "^4.4.0",
"turbo": "1.12.4",
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
index 79248061c..4522b9317 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
@@ -1,5 +1,7 @@
import { tryCancel } from '@apify/timeout';
import type { Cookie } from '@crawlee/types';
+// @ts-expect-error not exposed on type level
+import { CdpBrowser } from 'puppeteer';
import type Puppeteer from 'puppeteer';
import type * as PuppeteerTypes from 'puppeteer';
@@ -61,7 +63,9 @@ export class PuppeteerController extends BrowserController<
}
try {
- const context = await this.browser.createIncognitoBrowserContext(contextOptions);
+ const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;
+ const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';
+ const context = await (this.browser as any)[method](contextOptions) as PuppeteerTypes.BrowserContext;
tryCancel();
const page = await context.newPage();
tryCancel();
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index e7c9ae5c1..8bae5aefa 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -1,6 +1,8 @@
import type { Dictionary } from '@crawlee/types';
import type Puppeteer from 'puppeteer';
import type * as PuppeteerTypes from 'puppeteer';
+// @ts-expect-error not exposed on type level
+import { CdpBrowser } from 'puppeteer';
import type { PuppeteerNewPageOptions } from './puppeteer-controller';
import { PuppeteerController } from './puppeteer-controller';
@@ -22,6 +24,7 @@ export class PuppeteerPlugin extends BrowserPlugin<
protected async _launch(
launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,
): Promise<PuppeteerTypes.Browser> {
+ const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;
const {
launchOptions,
userDataDir,
@@ -44,8 +47,8 @@ export class PuppeteerPlugin extends BrowserPlugin<
}
}
- if (launchOptions!.headless === true) {
- launchOptions!.headless = 'new';
+ if (launchOptions!.headless === true && oldPuppeteerVersion) {
+ launchOptions!.headless = 'new' as any;
}
let browser: PuppeteerTypes.Browser;
@@ -98,11 +101,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
}
});
- const boundMethods = (['newPage', 'close', 'userAgent', 'createIncognitoBrowserContext', 'version', 'on', 'process'] as const)
+ const boundMethods = (['newPage', 'close', 'userAgent', 'createIncognitoBrowserContext', 'createBrowserContext', 'version', 'on', 'process'] as const)
.reduce((map, method) => {
- map[method] = browser[method]?.bind(browser);
+ map[method] = browser[method as 'close']?.bind(browser);
return map;
}, {} as Dictionary);
+ const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';
browser = new Proxy(browser, {
get: (target, property: keyof typeof browser, receiver) => {
@@ -114,9 +118,9 @@ export class PuppeteerPlugin extends BrowserPlugin<
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
try {
- const context = await browser.createIncognitoBrowserContext({
+ const context = await (browser as any)[method]({
proxyServer: anonymizedProxyUrl ?? proxyUrl,
- });
+ }) as PuppeteerTypes.BrowserContext;
page = await context.newPage(...args);
diff --git a/yarn.lock b/yarn.lock
index 2772b3ce0..902e9b48f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -694,7 +694,7 @@ __metadata:
playwright: "npm:1.41.2"
portastic: "npm:^1.0.1"
proxy: "npm:^1.0.2"
- puppeteer: "npm:21.10.0"
+ puppeteer: "npm:22.0.0"
rimraf: "npm:^5.0.0"
tsx: "npm:^4.4.0"
turbo: "npm:1.12.4"
@@ -1558,9 +1558,9 @@ __metadata:
languageName: node
linkType: hard
-"@puppeteer/browsers@npm:1.9.1":
- version: 1.9.1
- resolution: "@puppeteer/browsers@npm:1.9.1"
+"@puppeteer/browsers@npm:2.0.0":
+ version: 2.0.0
+ resolution: "@puppeteer/browsers@npm:2.0.0"
dependencies:
debug: "npm:4.3.4"
extract-zip: "npm:2.0.1"
@@ -1571,7 +1571,7 @@ __metadata:
yargs: "npm:17.7.2"
bin:
browsers: lib/cjs/main-cli.js
- checksum: 10c0/8cbde5ec8060c2bdfdc0d7149711cf0cccc6648ed61d0b969cd305918108e7973ce8a7ff3f7533fcc3c49552afe1955755cd7a3924c1fdbea7330d48094b35ee
+ checksum: 10c0/72e17739f43ad4c56d8b68249736ff303c0ebdfd10d7426f7084740c221fd0584034ad340cc4709dfec13ad497f5e93a619a9a7ee84729d5497cb7096f01b94b
languageName: node
linkType: hard
@@ -3515,15 +3515,15 @@ __metadata:
languageName: node
linkType: hard
-"chromium-bidi@npm:0.5.6":
- version: 0.5.6
- resolution: "chromium-bidi@npm:0.5.6"
+"chromium-bidi@npm:0.5.8":
+ version: 0.5.8
+ resolution: "chromium-bidi@npm:0.5.8"
dependencies:
mitt: "npm:3.0.1"
urlpattern-polyfill: "npm:10.0.0"
peerDependencies:
devtools-protocol: "*"
- checksum: 10c0/076fe42ed4eec6361a0c100bd17b91747ccc064cb89114431568c99daebf4b1f38c6466a4a2d17e7e08badc2a907f53245231f39344419a0f5c35c856f6ff717
+ checksum: 10c0/30a1827a6fd2e57fa1c07c7bc209c10b9e9748872b8403b901ca8ebdc7cc1ca879db223810567c5bf057c98fde5299631e58bfbc29cf7366ae9b38b215af3086
languageName: node
linkType: hard
@@ -9755,30 +9755,30 @@ __metadata:
languageName: node
linkType: hard
-"puppeteer-core@npm:21.10.0":
- version: 21.10.0
- resolution: "puppeteer-core@npm:21.10.0"
+"puppeteer-core@npm:22.0.0":
+ version: 22.0.0
+ resolution: "puppeteer-core@npm:22.0.0"
dependencies:
- "@puppeteer/browsers": "npm:1.9.1"
- chromium-bidi: "npm:0.5.6"
+ "@puppeteer/browsers": "npm:2.0.0"
+ chromium-bidi: "npm:0.5.8"
cross-fetch: "npm:4.0.0"
debug: "npm:4.3.4"
devtools-protocol: "npm:0.0.1232444"
ws: "npm:8.16.0"
- checksum: 10c0/301e4430d22b4659b77bfffe02bf3ee0ddf0fa095a44e782f17b473f53b2fb015f2deed080b4acfa4b75a50b889fe2295df7ce82eebd161bfbeb9d749529eaad
+ checksum: 10c0/e6eee1a02e3c69a4f427f1a7924b363f41c00fe0faff02ac21459cd2ab306dad24284cb04635f46cdfebef4a8e335530816b361b6a6067cb288a07a2fae693cf
languageName: node
linkType: hard
-"puppeteer@npm:21.10.0":
- version: 21.10.0
- resolution: "puppeteer@npm:21.10.0"
+"puppeteer@npm:22.0.0":
+ version: 22.0.0
+ resolution: "puppeteer@npm:22.0.0"
dependencies:
- "@puppeteer/browsers": "npm:1.9.1"
+ "@puppeteer/browsers": "npm:2.0.0"
cosmiconfig: "npm:9.0.0"
- puppeteer-core: "npm:21.10.0"
+ puppeteer-core: "npm:22.0.0"
bin:
puppeteer: lib/esm/puppeteer/node/cli.js
- checksum: 10c0/a12b0352f84d78e1c379b3efab4c3a98394a1e051c5675ee59c23ad4812d127ccf9b679ad379019d113ab2da8128c469b5dda41059527ec8a83793c232d28347
+ checksum: 10c0/8c2909db2c9c030f2a26f9179b5150e836941c676db57ec360d1389f981d2dcde2a66ecb242bffb375d3c7303827c4ee5b4148eda165ec834564cfadd0dcd963
languageName: node
linkType: hard