Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: bump puppeteer support to 17.x (#1519)

Vlad Frangub97a85282b64
diff --git a/package-lock.json b/package-lock.json
index 4d7a07411..d18d8d560 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -55,7 +55,7 @@
"playwright": "1.25.1",
"portastic": "^1.0.1",
"proxy": "^1.0.2",
- "puppeteer": "15.1.1",
+ "puppeteer": "17.0.0",
"ts-jest": "^28.0.8",
"ts-node": "^10.9.1",
"turbo": "1.4.5",
@@ -6579,9 +6579,9 @@
}
},
"node_modules/devtools-protocol": {
- "version": "0.0.1011705",
- "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1011705.tgz",
- "integrity": "sha512-OKvTvu9n3swmgYshvsyVHYX0+aPzCoYUnyXUacfQMmFtBtBKewV/gT4I9jkAbpTqtTi2E4S9MXLlvzBDUlqg0Q==",
+ "version": "0.0.1019158",
+ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz",
+ "integrity": "sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ==",
"devOptional": true
},
"node_modules/dezalgo": {
@@ -13525,33 +13525,32 @@
}
},
"node_modules/puppeteer": {
- "version": "15.1.1",
- "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-15.1.1.tgz",
- "integrity": "sha512-XMysu48uIcaYad/IelRTX3yxpHkcNdhdzPegnBEz9h1uEQfLhFcMJnjyvus51Sm+OPwr2gaKQhtyuIVaVKqd0Q==",
+ "version": "17.0.0",
+ "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-17.0.0.tgz",
+ "integrity": "sha512-T2rdzlPxnPezF218kywFP3O+0YI5/8Kl8riNUicGb+KuMyDTrqRjhSOSDp6coQ1T4QYPBARTFp4EMBepMOzAQA==",
"devOptional": true,
"hasInstallScript": true,
"dependencies": {
"cross-fetch": "3.1.5",
"debug": "4.3.4",
- "devtools-protocol": "0.0.1011705",
+ "devtools-protocol": "0.0.1019158",
"extract-zip": "2.0.1",
"https-proxy-agent": "5.0.1",
- "pkg-dir": "4.2.0",
"progress": "2.0.3",
"proxy-from-env": "1.1.0",
"rimraf": "3.0.2",
"tar-fs": "2.1.1",
"unbzip2-stream": "1.4.3",
- "ws": "8.8.0"
+ "ws": "8.8.1"
},
"engines": {
"node": ">=14.1.0"
}
},
"node_modules/puppeteer/node_modules/ws": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz",
- "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==",
+ "version": "8.8.1",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz",
+ "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==",
"devOptional": true,
"engines": {
"node": ">=10.0.0"
@@ -16414,7 +16413,7 @@
},
"peerDependencies": {
"playwright": "^1.21.1",
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"playwright": {
@@ -16529,7 +16528,7 @@
},
"peerDependencies": {
"playwright": "^1.21.1",
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"playwright": {
@@ -16652,7 +16651,7 @@
"node": ">=16.0.0"
},
"peerDependencies": {
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"puppeteer": {
@@ -22486,9 +22485,9 @@
"dev": true
},
"devtools-protocol": {
- "version": "0.0.1011705",
- "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1011705.tgz",
- "integrity": "sha512-OKvTvu9n3swmgYshvsyVHYX0+aPzCoYUnyXUacfQMmFtBtBKewV/gT4I9jkAbpTqtTi2E4S9MXLlvzBDUlqg0Q==",
+ "version": "0.0.1019158",
+ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz",
+ "integrity": "sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ==",
"devOptional": true
},
"dezalgo": {
@@ -27715,29 +27714,28 @@
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
},
"puppeteer": {
- "version": "15.1.1",
- "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-15.1.1.tgz",
- "integrity": "sha512-XMysu48uIcaYad/IelRTX3yxpHkcNdhdzPegnBEz9h1uEQfLhFcMJnjyvus51Sm+OPwr2gaKQhtyuIVaVKqd0Q==",
+ "version": "17.0.0",
+ "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-17.0.0.tgz",
+ "integrity": "sha512-T2rdzlPxnPezF218kywFP3O+0YI5/8Kl8riNUicGb+KuMyDTrqRjhSOSDp6coQ1T4QYPBARTFp4EMBepMOzAQA==",
"devOptional": true,
"requires": {
"cross-fetch": "3.1.5",
"debug": "4.3.4",
- "devtools-protocol": "0.0.1011705",
+ "devtools-protocol": "0.0.1019158",
"extract-zip": "2.0.1",
"https-proxy-agent": "5.0.1",
- "pkg-dir": "4.2.0",
"progress": "2.0.3",
"proxy-from-env": "1.1.0",
"rimraf": "3.0.2",
"tar-fs": "2.1.1",
"unbzip2-stream": "1.4.3",
- "ws": "8.8.0"
+ "ws": "8.8.1"
},
"dependencies": {
"ws": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz",
- "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==",
+ "version": "8.8.1",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz",
+ "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==",
"devOptional": true,
"requires": {}
}
diff --git a/package.json b/package.json
index 6694e9f50..5fdae02aa 100644
--- a/package.json
+++ b/package.json
@@ -93,7 +93,7 @@
"playwright": "1.25.1",
"portastic": "^1.0.1",
"proxy": "^1.0.2",
- "puppeteer": "15.1.1",
+ "puppeteer": "17.0.0",
"ts-jest": "^28.0.8",
"ts-node": "^10.9.1",
"turbo": "1.4.5",
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index 399af4440..187725aec 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -52,7 +52,7 @@
},
"peerDependencies": {
"playwright": "^1.21.1",
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"playwright": {
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index edaa140e2..ca4570e78 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -70,7 +70,7 @@
},
"peerDependencies": {
"playwright": "^1.21.1",
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"playwright": {
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index fadac2006..48667e152 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -64,7 +64,7 @@
"ow": "^0.28.1"
},
"peerDependencies": {
- "puppeteer": ">= 9.x <= 15.1"
+ "puppeteer": ">= 9.x <= 17.0"
},
"peerDependenciesMeta": {
"puppeteer": {
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index 3584f1d83..c34b1dd8f 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -22,6 +22,7 @@ import { readFile } from 'fs/promises';
import ow from 'ow';
import vm from 'vm';
import { LruCache } from '@apify/datastructures';
+import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
import type { Page, HTTPResponse, ResponseForRequest, HTTPRequest as PuppeteerRequest } from 'puppeteer';
import log_ from '@apify/log';
import type { Request } from '@crawlee/core';
@@ -237,12 +238,36 @@ export async function blockRequests(page: Page, options: BlockRequestsOptions =
const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
- if (page._client instanceof Function) {
- await page._client().send('Network.setBlockedURLs', { urls: patternsToBlock });
- } else {
- // @ts-expect-error for older puppeteer (<14.4)
- await page._client.send('Network.setBlockedURLs', { urls: patternsToBlock });
+ // We use CDP commands instead of request interception as the latter disables caching, which is not ideal
+ await sendCDPCommand(page, 'Network.setBlockedURLs', { urls: patternsToBlock });
+}
+
+/**
+ * @internal
+ */
+export async function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(
+ page: Page,
+ command: T,
+ ...args: ProtocolMapping.Commands[T]['paramsType']
+): Promise<ProtocolMapping.Commands[T]['returnType']> {
+ // In puppeteer 16.x and 17.x, the `_client` method is completely omitted from the types. It's still there and works the same way, but it is hidden.
+
+ // Puppeteer <= 17
+ if (Reflect.has(page, '_client')) {
+ const client = Reflect.get(page, '_client');
+
+ if (typeof client === 'function') {
+ return client.call(page).send(command, ...args);
+ }
+
+ return client.send(command, ...args);
}
+
+ const jsonPath = require.resolve('puppeteer/package.json');
+ const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
+
+ // eslint-disable-next-line max-len
+ throw new Error(`Cannot detect CDP client for Puppeteer ${parsed.version}. You should report this to Crawlee, mentioning the puppeteer version you are using.`);
}
/**