Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(deps): update dependency puppeteer to v20.9.0
renovate[bot]f193e11924ad
diff --git a/package.json b/package.jsonindex 4aa170465..19586e087 100644--- a/package.json+++ b/package.json@@ -93,7 +93,7 @@"playwright": "1.36.1","portastic": "^1.0.1","proxy": "^1.0.2",- "puppeteer": "20.8.3",+ "puppeteer": "20.9.0","rimraf": "^5.0.0","ts-jest": "^29.0.3","ts-node": "^10.9.1",diff --git a/yarn.lock b/yarn.lockindex 5dc778295..b1361c9ef 100644--- a/yarn.lock+++ b/yarn.lock@@ -1035,7 +1035,7 @@ __metadata:playwright: 1.36.1portastic: ^1.0.1proxy: ^1.0.2- puppeteer: 20.8.3+ puppeteer: 20.9.0rimraf: ^5.0.0ts-jest: ^29.0.3ts-node: ^10.9.1@@ -1906,17 +1906,14 @@ __metadata:languageName: nodelinkType: hard-"@puppeteer/browsers@npm:1.4.5":- version: 1.4.5- resolution: "@puppeteer/browsers@npm:1.4.5"+"@puppeteer/browsers@npm:1.4.6":+ version: 1.4.6+ resolution: "@puppeteer/browsers@npm:1.4.6"dependencies:debug: 4.3.4extract-zip: 2.0.1- http-proxy-agent: 7.0.0- https-proxy-agent: 7.0.1progress: 2.0.3- proxy-from-env: 1.1.0- socks-proxy-agent: 8.0.1+ proxy-agent: 6.3.0tar-fs: 3.0.4unbzip2-stream: 1.4.3yargs: 17.7.1@@ -1927,7 +1924,7 @@ __metadata:optional: truebin:browsers: lib/cjs/main-cli.js- checksum: f3740f184b076233e8ab9f47e11e074dff6efca617e93d080b305d84e21f142fbb7bc59590afd8034c253ec3a72335c1338bcbcc360576ca14946fd6e16f78d3+ checksum: 29569dd8a8a41737bb0dd40cce6279cfc8764afc6242d2f9d8ae610bed7e466fc77eeb27b9b3ac53dd04927a1a0e26389f282f6ba057210979b36ab455009d64languageName: nodelinkType: hard@@ -2029,6 +2026,13 @@ __metadata:languageName: nodelinkType: hard+"@tootallnate/quickjs-emscripten@npm:^0.23.0":+ version: 0.23.0+ resolution: "@tootallnate/quickjs-emscripten@npm:0.23.0"+ checksum: c350a2947ffb80b22e14ff35099fd582d1340d65723384a0fd0515e905e2534459ad2f301a43279a37308a27c99273c932e64649abd57d0bb3ca8c557150eccc+ languageName: node+ linkType: hard+"@tsconfig/node10@npm:^1.0.7":version: 1.0.9resolution: "@tsconfig/node10@npm:1.0.9"@@ -3173,6 +3177,15 @@ __metadata:languageName: nodelinkType: hard+"ast-types@npm:^0.13.4":+ version: 0.13.4+ resolution: "ast-types@npm:0.13.4"+ dependencies:+ tslib: ^2.0.1+ checksum: 5a51f7b70588ecced3601845a0e203279ca2f5fdc184416a0a1640c93ec0a267241d6090a328e78eebb8de81f8754754e0a4f1558ba2a3d638f8ccbd0b1f0eff+ languageName: node+ linkType: hard+"astral-regex@npm:^2.0.0":version: 2.0.0resolution: "astral-regex@npm:2.0.0"@@ -3350,6 +3363,13 @@ __metadata:languageName: nodelinkType: hard+"basic-ftp@npm:^5.0.2":+ version: 5.0.3+ resolution: "basic-ftp@npm:5.0.3"+ checksum: 8b04e88eb85a64de9311721bb0707c9cd70453eefdd854cab85438e6f46fb6c597ddad57ed1acf0a9ede3c677b14e657f51051688a5f23d6f3ea7b5d9073b850+ languageName: node+ linkType: hard+"before-after-hook@npm:^2.2.0":version: 2.2.3resolution: "before-after-hook@npm:2.2.3"@@ -4415,6 +4435,13 @@ __metadata:languageName: nodelinkType: hard+"data-uri-to-buffer@npm:^5.0.1":+ version: 5.0.1+ resolution: "data-uri-to-buffer@npm:5.0.1"+ checksum: 10958f89c0047b84bd86d572b6b77c9bf238ebe7b55a9a9ab04c90fbf5ab1881783b72e31dc0febdffd30ec914930244f2f728e3629bb8911d922baba129426f+ languageName: node+ linkType: hard+"data-urls@npm:^4.0.0":version: 4.0.0resolution: "data-urls@npm:4.0.0"@@ -4605,6 +4632,17 @@ __metadata:languageName: nodelinkType: hard+"degenerator@npm:^5.0.0":+ version: 5.0.1+ resolution: "degenerator@npm:5.0.1"+ dependencies:+ ast-types: ^0.13.4+ escodegen: ^2.1.0+ esprima: ^4.0.1+ checksum: a64fa39cdf6c2edd75188157d32338ee9de7193d7dbb2aeb4acb1eb30fa4a15ed80ba8dae9bd4d7b085472cf174a5baf81adb761aaa8e326771392c922084152+ languageName: node+ linkType: hard+"delayed-stream@npm:~1.0.0":version: 1.0.0resolution: "delayed-stream@npm:1.0.0"@@ -4668,10 +4706,10 @@ __metadata:languageName: nodelinkType: hard-"devtools-protocol@npm:0.0.1135028":- version: 0.0.1135028- resolution: "devtools-protocol@npm:0.0.1135028"- checksum: ee647140ac843ba3a7a4b6436588bf81adbb1f035489a9e71985b27c1f489779c63865ddb70a98beb3f4cea9dd53f1e049f41163c46d4555a76d36868b5882d1+"devtools-protocol@npm:0.0.1147663":+ version: 0.0.1147663+ resolution: "devtools-protocol@npm:0.0.1147663"+ checksum: 0631f2b6c6cd7f56e7d62a85bfc291f7e167f0f2de90969ef61fb24e2bd546b2e9530043d2bc3fe6c4d7a9e00473004272d2c2832a10a05e4b75c03a22f549fclanguageName: nodelinkType: hard@@ -5084,6 +5122,24 @@ __metadata:languageName: nodelinkType: hard+"escodegen@npm:^2.1.0":+ version: 2.1.0+ resolution: "escodegen@npm:2.1.0"+ dependencies:+ esprima: ^4.0.1+ estraverse: ^5.2.0+ esutils: ^2.0.2+ source-map: ~0.6.1+ dependenciesMeta:+ source-map:+ optional: true+ bin:+ escodegen: bin/escodegen.js+ esgenerate: bin/esgenerate.js+ checksum: 096696407e161305cd05aebb95134ad176708bc5cb13d0dcc89a5fcbb959b8ed757e7f2591a5f8036f8f4952d4a724de0df14cd419e29212729fa6df5ce16bf6+ languageName: node+ linkType: hard+"eslint-config-airbnb-base@npm:^15.0.0":version: 15.0.0resolution: "eslint-config-airbnb-base@npm:15.0.0"@@ -5334,7 +5390,7 @@ __metadata:languageName: nodelinkType: hard-"esprima@npm:^4.0.0":+"esprima@npm:^4.0.0, esprima@npm:^4.0.1":version: 4.0.1resolution: "esprima@npm:4.0.1"bin:@@ -5879,6 +5935,17 @@ __metadata:languageName: nodelinkType: hard+"fs-extra@npm:^8.1.0":+ version: 8.1.0+ resolution: "fs-extra@npm:8.1.0"+ dependencies:+ graceful-fs: ^4.2.0+ jsonfile: ^4.0.0+ universalify: ^0.1.0+ checksum: bf44f0e6cea59d5ce071bba4c43ca76d216f89e402dc6285c128abc0902e9b8525135aa808adad72c9d5d218e9f4bcc63962815529ff2f684ad532172a284880+ languageName: node+ linkType: hard+"fs-minipass@npm:^2.0.0":version: 2.1.0resolution: "fs-minipass@npm:2.1.0"@@ -6082,6 +6149,18 @@ __metadata:languageName: nodelinkType: hard+"get-uri@npm:^6.0.1":+ version: 6.0.1+ resolution: "get-uri@npm:6.0.1"+ dependencies:+ basic-ftp: ^5.0.2+ data-uri-to-buffer: ^5.0.1+ debug: ^4.3.4+ fs-extra: ^8.1.0+ checksum: a8aec70e1c67386fbe67f66e344ecd671a19f4cfc8e0f0e14d070563af5123d540e77fbceb6e26566f29846fac864d2862699ab134d307f85c85e7d72ce23d14+ languageName: node+ linkType: hard+"git-raw-commits@npm:^2.0.11":version: 2.0.11resolution: "git-raw-commits@npm:2.0.11"@@ -6600,16 +6679,6 @@ __metadata:languageName: nodelinkType: hard-"http-proxy-agent@npm:7.0.0":- version: 7.0.0- resolution: "http-proxy-agent@npm:7.0.0"- dependencies:- agent-base: ^7.1.0- debug: ^4.3.4- checksum: 48d4fac997917e15f45094852b63b62a46d0c8a4f0b9c6c23ca26d27b8df8d178bed88389e604745e748bd9a01f5023e25093722777f0593c3f052009ff438b6- languageName: node- linkType: hard-"http-proxy-agent@npm:^5.0.0":version: 5.0.0resolution: "http-proxy-agent@npm:5.0.0"@@ -6621,6 +6690,16 @@ __metadata:languageName: nodelinkType: hard+"http-proxy-agent@npm:^7.0.0":+ version: 7.0.0+ resolution: "http-proxy-agent@npm:7.0.0"+ dependencies:+ agent-base: ^7.1.0+ debug: ^4.3.4+ checksum: 48d4fac997917e15f45094852b63b62a46d0c8a4f0b9c6c23ca26d27b8df8d178bed88389e604745e748bd9a01f5023e25093722777f0593c3f052009ff438b6+ languageName: node+ linkType: hard+"http2-wrapper@npm:^2.1.10, http2-wrapper@npm:^2.1.11":version: 2.2.0resolution: "http2-wrapper@npm:2.2.0"@@ -6631,16 +6710,6 @@ __metadata:languageName: nodelinkType: hard-"https-proxy-agent@npm:7.0.1":- version: 7.0.1- resolution: "https-proxy-agent@npm:7.0.1"- dependencies:- agent-base: ^7.0.2- debug: 4- checksum: 2d765c31865071373771f53abdd72912567b76015a4eff61094f586194192950cd89257d50f0e621807a16c083bc8cad5852e3885c6ba154d2ce721a18fac248- languageName: node- linkType: hard-"https-proxy-agent@npm:^5.0.0, https-proxy-agent@npm:^5.0.1":version: 5.0.1resolution: "https-proxy-agent@npm:5.0.1"@@ -6651,6 +6720,16 @@ __metadata:languageName: nodelinkType: hard+"https-proxy-agent@npm:^7.0.0":+ version: 7.0.1+ resolution: "https-proxy-agent@npm:7.0.1"+ dependencies:+ agent-base: ^7.0.2+ debug: 4+ checksum: 2d765c31865071373771f53abdd72912567b76015a4eff61094f586194192950cd89257d50f0e621807a16c083bc8cad5852e3885c6ba154d2ce721a18fac248+ languageName: node+ linkType: hard+"human-signals@npm:^2.1.0":version: 2.1.0resolution: "human-signals@npm:2.1.0"@@ -6872,6 +6951,13 @@ __metadata:languageName: nodelinkType: hard+"ip@npm:^1.1.8":+ version: 1.1.8+ resolution: "ip@npm:1.1.8"+ checksum: a2ade53eb339fb0cbe9e69a44caab10d6e3784662285eb5d2677117ee4facc33a64679051c35e0dfdb1a3983a51ce2f5d2cb36446d52e10d01881789b76e28fb+ languageName: node+ linkType: hard+"ip@npm:^2.0.0":version: 2.0.0resolution: "ip@npm:2.0.0"@@ -7985,6 +8071,18 @@ __metadata:languageName: nodelinkType: hard+"jsonfile@npm:^4.0.0":+ version: 4.0.0+ resolution: "jsonfile@npm:4.0.0"+ dependencies:+ graceful-fs: ^4.1.6+ dependenciesMeta:+ graceful-fs:+ optional: true+ checksum: 6447d6224f0d31623eef9b51185af03ac328a7553efcee30fa423d98a9e276ca08db87d71e17f2310b0263fd3ffa6c2a90a6308367f661dc21580f9469897c9e+ languageName: node+ linkType: hard+"jsonfile@npm:^6.0.1":version: 6.1.0resolution: "jsonfile@npm:6.1.0"@@ -8484,7 +8582,7 @@ __metadata:languageName: nodelinkType: hard-"lru-cache@npm:^7.4.4, lru-cache@npm:^7.5.1, lru-cache@npm:^7.7.1":+"lru-cache@npm:^7.14.1, lru-cache@npm:^7.4.4, lru-cache@npm:^7.5.1, lru-cache@npm:^7.7.1":version: 7.18.3resolution: "lru-cache@npm:7.18.3"checksum: e550d772384709deea3f141af34b6d4fa392e2e418c1498c078de0ee63670f1f46f5eee746e8ef7e69e1c895af0d4224e62ee33e66a543a14763b0f2e74c1356@@ -8994,6 +9092,13 @@ __metadata:languageName: nodelinkType: hard+"netmask@npm:^2.0.2":+ version: 2.0.2+ resolution: "netmask@npm:2.0.2"+ checksum: c65cb8d3f7ea5669edddb3217e4c96910a60d0d9a4b52d9847ff6b28b2d0277cd8464eee0ef85133cdee32605c57940cacdd04a9a019079b091b6bba4cb0ec22+ languageName: node+ linkType: hard+"node-addon-api@npm:^3.2.1":version: 3.2.1resolution: "node-addon-api@npm:3.2.1"@@ -9731,6 +9836,33 @@ __metadata:languageName: nodelinkType: hard+"pac-proxy-agent@npm:^7.0.0":+ version: 7.0.0+ resolution: "pac-proxy-agent@npm:7.0.0"+ dependencies:+ "@tootallnate/quickjs-emscripten": ^0.23.0+ agent-base: ^7.0.2+ debug: ^4.3.4+ get-uri: ^6.0.1+ http-proxy-agent: ^7.0.0+ https-proxy-agent: ^7.0.0+ pac-resolver: ^7.0.0+ socks-proxy-agent: ^8.0.1+ checksum: 45fe10ae58b1700d5419a9e5b525fb261b866ed6a65c1382fe45c3d5af9f81d9a58250d407941a363b1955e0315f3d97e02a2f20e4c7e2ba793bd46585db7ec8+ languageName: node+ linkType: hard++"pac-resolver@npm:^7.0.0":+ version: 7.0.0+ resolution: "pac-resolver@npm:7.0.0"+ dependencies:+ degenerator: ^5.0.0+ ip: ^1.1.8+ netmask: ^2.0.2+ checksum: fa3a898c09848e93e35f5e23443fea36ddb393a851c76a23664a5bf3fcbe58ff77a0bcdae1e4f01b9ea87ea493c52e14d97a0fe39f92474d14cd45559c6e3cde+ languageName: node+ linkType: hard+"pacote@npm:^15.2.0":version: 15.2.0resolution: "pacote@npm:15.2.0"@@ -10146,6 +10278,22 @@ __metadata:languageName: nodelinkType: hard+"proxy-agent@npm:6.3.0":+ version: 6.3.0+ resolution: "proxy-agent@npm:6.3.0"+ dependencies:+ agent-base: ^7.0.2+ debug: ^4.3.4+ http-proxy-agent: ^7.0.0+ https-proxy-agent: ^7.0.0+ lru-cache: ^7.14.1+ pac-proxy-agent: ^7.0.0+ proxy-from-env: ^1.1.0+ socks-proxy-agent: ^8.0.1+ checksum: e3fb0633d665e352ed4efe23ae5616b8301423dfa4ff1c5975d093da8a636181a97391f7a91c6a7ffae17c1a305df855e95507f73bcdafda8876198c64b88f5b+ languageName: node+ linkType: hard+"proxy-chain@npm:^2.0.1":version: 2.3.0resolution: "proxy-chain@npm:2.3.0"@@ -10155,7 +10303,7 @@ __metadata:languageName: nodelinkType: hard-"proxy-from-env@npm:1.1.0, proxy-from-env@npm:^1.1.0":+"proxy-from-env@npm:^1.1.0":version: 1.1.0resolution: "proxy-from-env@npm:1.1.0"checksum: ed7fcc2ba0a33404958e34d95d18638249a68c430e30fcb6c478497d72739ba64ce9810a24f53a7d921d0c065e5b78e3822759800698167256b04659366ca4d4@@ -10199,33 +10347,33 @@ __metadata:languageName: nodelinkType: hard-"puppeteer-core@npm:20.8.3":- version: 20.8.3- resolution: "puppeteer-core@npm:20.8.3"+"puppeteer-core@npm:20.9.0":+ version: 20.9.0+ resolution: "puppeteer-core@npm:20.9.0"dependencies:- "@puppeteer/browsers": 1.4.5+ "@puppeteer/browsers": 1.4.6chromium-bidi: 0.4.16cross-fetch: 4.0.0debug: 4.3.4- devtools-protocol: 0.0.1135028+ devtools-protocol: 0.0.1147663ws: 8.13.0peerDependencies:typescript: ">= 4.7.4"peerDependenciesMeta:typescript:optional: true- checksum: 29efe38c100ee8cec413115141d8088a53eccd2dfc1bde806df024c957506b0b7a31d3c5ecc35277c79d299d2aaf2d6a29eb704c61443cdae230be0e037fc246+ checksum: d298598445b0f2032c02d0ed7d1d18a8d2d2fcaf6fc31fc96e93e2669a7fc6fbee0338bd9b8c8f8822887f18a8fb680b77bb56e96fe1928baadb52292bbd93b4languageName: nodelinkType: hard-"puppeteer@npm:20.8.3":- version: 20.8.3- resolution: "puppeteer@npm:20.8.3"+"puppeteer@npm:20.9.0":+ version: 20.9.0+ resolution: "puppeteer@npm:20.9.0"dependencies:- "@puppeteer/browsers": 1.4.5+ "@puppeteer/browsers": 1.4.6cosmiconfig: 8.2.0- puppeteer-core: 20.8.3- checksum: a51ba8f0e266175a42d6e2be965652f3d896c0112d4e7a9f26358f6f4342a1c92661063699374ddb21ab81e9413e7e755be1fe84473bfe94b3e53f4d9c9bbd3f+ puppeteer-core: 20.9.0+ checksum: c328eca0a649d5e5d68ee2874c25254b8498ab8bbfa1731b444c2dd69380c2f6df1741b3ad427d448fb8e5bcdd259cc9c1ed749dddfc634a64a6a607403fe53clanguageName: nodelinkType: hard@@ -11005,17 +11153,6 @@ __metadata:languageName: nodelinkType: hard-"socks-proxy-agent@npm:8.0.1":- version: 8.0.1- resolution: "socks-proxy-agent@npm:8.0.1"- dependencies:- agent-base: ^7.0.1- debug: ^4.3.4- socks: ^2.7.1- checksum: f6538fd16cb545094d20b9a1ae97bb2c4ddd150622ad7cc6b64c89c889d8847b7bac179757838ce5487cbac49a499537e3991c975fe13b152b76b10027470dfb- languageName: node- linkType: hard-"socks-proxy-agent@npm:^7.0.0":version: 7.0.0resolution: "socks-proxy-agent@npm:7.0.0"@@ -11027,6 +11164,17 @@ __metadata:languageName: nodelinkType: hard+"socks-proxy-agent@npm:^8.0.1":+ version: 8.0.1+ resolution: "socks-proxy-agent@npm:8.0.1"+ dependencies:+ agent-base: ^7.0.1+ debug: ^4.3.4+ socks: ^2.7.1+ checksum: f6538fd16cb545094d20b9a1ae97bb2c4ddd150622ad7cc6b64c89c889d8847b7bac179757838ce5487cbac49a499537e3991c975fe13b152b76b10027470dfb+ languageName: node+ linkType: hard+"socks@npm:^2.6.2, socks@npm:^2.7.1":version: 2.7.1resolution: "socks@npm:2.7.1"@@ -11056,7 +11204,7 @@ __metadata:languageName: nodelinkType: hard-"source-map@npm:^0.6.0, source-map@npm:^0.6.1":+"source-map@npm:^0.6.0, source-map@npm:^0.6.1, source-map@npm:~0.6.1":version: 0.6.1resolution: "source-map@npm:0.6.1"checksum: 59ce8640cf3f3124f64ac289012c2b8bd377c238e316fb323ea22fbfe83da07d81e000071d7242cad7a23cd91c7de98e4df8830ec3f133cb6133a5f6e9f67bc2@@ -11803,7 +11951,7 @@ __metadata:languageName: nodelinkType: hard-"tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.0":+"tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.0":version: 2.6.0resolution: "tslib@npm:2.6.0"checksum: c01066038f950016a18106ddeca4649b4d76caa76ec5a31e2a26e10586a59fceb4ee45e96719bf6c715648e7c14085a81fee5c62f7e9ebee68e77a5396e5538f@@ -12111,6 +12259,13 @@ __metadata:languageName: nodelinkType: hard+"universalify@npm:^0.1.0":+ version: 0.1.2+ resolution: "universalify@npm:0.1.2"+ checksum: 40cdc60f6e61070fe658ca36016a8f4ec216b29bf04a55dce14e3710cc84c7448538ef4dad3728d0bfe29975ccd7bfb5f414c45e7b78883567fb31b246f02dff+ languageName: node+ linkType: hard+"universalify@npm:^0.2.0":version: 0.2.0resolution: "universalify@npm:0.2.0"