Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(deps): batch dependency version bumps (#3451)
Jindřich Bär47f0889ee111
diff --git a/package.json b/package.jsonindex d0144cb5b..a6d470472 100644--- a/package.json+++ b/package.json@@ -87,7 +87,7 @@"apify-node-curl-impersonate": "^1.0.15","basic-auth-parser": "^0.0.2","body-parser": "^2.0.0",- "camoufox-js": "^0.8.0",+ "camoufox-js": "^0.9.0","commitlint": "^20.0.0","cross-env": "^10.0.0","deep-equal": "^2.0.5",diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 0c5833abc..ed1c45e03 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -53,7 +53,7 @@"@crawlee/utils": "3.16.0","csv-stringify": "^6.2.0","fs-extra": "^11.0.0",- "got-scraping": "^4.0.0",+ "got-scraping": "^4.2.1","ow": "^0.28.1","tldts": "^7.0.0","tslib": "^2.4.0",diff --git a/packages/core/package.json b/packages/core/package.jsonindex 4e852bc36..e3290c108 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -66,7 +66,7 @@"@vladfrangu/async_event_emitter": "^2.2.2","csv-stringify": "^6.2.0","fs-extra": "^11.0.0",- "got-scraping": "^4.0.0",+ "got-scraping": "^4.2.1","json5": "^2.2.3","minimatch": "^9.0.0","ow": "^0.28.1",diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex e643802ef..f7317eded 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -61,7 +61,7 @@"@types/content-type": "^1.1.5","cheerio": "1.0.0-rc.12","content-type": "^1.0.4",- "got-scraping": "^4.0.0",+ "got-scraping": "^4.2.1","iconv-lite": "^0.7.0","mime-types": "^2.1.35","ow": "^0.28.1",diff --git a/packages/templates/templates/camoufox-ts/package.json b/packages/templates/templates/camoufox-ts/package.jsonindex 3acd6866a..fa68ac6ed 100644--- a/packages/templates/templates/camoufox-ts/package.json+++ b/packages/templates/templates/camoufox-ts/package.json@@ -4,7 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {- "camoufox-js": "^0.8.0",+ "camoufox-js": "^0.9.0","crawlee": "^3.0.0","playwright": "*"},diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 21cac1940..342de6f61 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -53,7 +53,7 @@"@types/sax": "^1.2.7","cheerio": "1.0.0-rc.12","file-type": "^20.0.0",- "got-scraping": "^4.0.3",+ "got-scraping": "^4.2.1","ow": "^0.28.1","robots-parser": "^3.0.1","sax": "^1.4.1",diff --git a/website/yarn.lock b/website/yarn.lockindex 11bfb1185..4187ac920 100644--- a/website/yarn.lock+++ b/website/yarn.lock@@ -12093,9 +12093,9 @@ __metadata:linkType: hard"lodash@npm:^4.17.20, lodash@npm:^4.17.21":- version: 4.17.21- resolution: "lodash@npm:4.17.21"- checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c+ version: 4.17.23+ resolution: "lodash@npm:4.17.23"+ checksum: 10c0/1264a90469f5bb95d4739c43eb6277d15b6d9e186df4ac68c3620443160fc669e2f14c11e7d8b2ccf078b81d06147c01a8ccced9aab9f9f63d50dcf8cace6bf6languageName: nodelinkType: harddiff --git a/yarn.lock b/yarn.lockindex b8f03d284..2f96e7aef 100644--- a/yarn.lock+++ b/yarn.lock@@ -796,7 +796,7 @@ __metadata:"@crawlee/utils": "npm:3.16.0"csv-stringify: "npm:^6.2.0"fs-extra: "npm:^11.0.0"- got-scraping: "npm:^4.0.0"+ got-scraping: "npm:^4.2.1"ow: "npm:^0.28.1"tldts: "npm:^7.0.0"tslib: "npm:^2.4.0"@@ -902,7 +902,7 @@ __metadata:"@vladfrangu/async_event_emitter": "npm:^2.2.2"csv-stringify: "npm:^6.2.0"fs-extra: "npm:^11.0.0"- got-scraping: "npm:^4.0.0"+ got-scraping: "npm:^4.2.1"json5: "npm:^2.2.3"minimatch: "npm:^9.0.0"ow: "npm:^0.28.1"@@ -926,7 +926,7 @@ __metadata:"@types/content-type": "npm:^1.1.5"cheerio: "npm:1.0.0-rc.12"content-type: "npm:^1.0.4"- got-scraping: "npm:^4.0.0"+ got-scraping: "npm:^4.2.1"iconv-lite: "npm:^0.7.0"mime-types: "npm:^2.1.35"ow: "npm:^0.28.1"@@ -1091,7 +1091,7 @@ __metadata:apify-node-curl-impersonate: "npm:^1.0.15"basic-auth-parser: "npm:^0.0.2"body-parser: "npm:^2.0.0"- camoufox-js: "npm:^0.8.0"+ camoufox-js: "npm:^0.9.0"commitlint: "npm:^20.0.0"cross-env: "npm:^10.0.0"deep-equal: "npm:^2.0.5"@@ -1182,7 +1182,7 @@ __metadata:"@types/whatwg-mimetype": "npm:^3.0.2"cheerio: "npm:1.0.0-rc.12"file-type: "npm:^20.0.0"- got-scraping: "npm:^4.0.3"+ got-scraping: "npm:^4.2.1"ow: "npm:^0.28.1"robots-parser: "npm:^3.0.1"sax: "npm:^1.4.1"@@ -1906,6 +1906,13 @@ __metadata:languageName: nodelinkType: hard+"@isaacs/cliui@npm:^9.0.0":+ version: 9.0.0+ resolution: "@isaacs/cliui@npm:9.0.0"+ checksum: 10c0/971063b7296419f85053dacd0a0285dcadaa3dfc139228b23e016c1a9848121ad4aa5e7fcca7522062014e1eb6239a7424188b9f2cba893a79c90aae5710319c+ languageName: node+ linkType: hard+"@isaacs/fs-minipass@npm:^4.0.0":version: 4.0.1resolution: "@isaacs/fs-minipass@npm:4.0.1"@@ -4490,6 +4497,15 @@ __metadata:languageName: nodelinkType: hard+"balanced-match@npm:^4.0.2":+ version: 4.0.2+ resolution: "balanced-match@npm:4.0.2"+ dependencies:+ jackspeak: "npm:^4.2.3"+ checksum: 10c0/493eee4bece3f8b270cea8d3d6d1122ce008dd6b0d5aca8a3f1e623be6897be18c926018eadc454bd719bb7cc46d939c39fa2a05fff86b30f65382f020f6926d+ languageName: node+ linkType: hard+"bare-events@npm:^2.5.4, bare-events@npm:^2.7.0":version: 2.8.2resolution: "bare-events@npm:2.8.2"@@ -4720,6 +4736,15 @@ __metadata:languageName: nodelinkType: hard+"brace-expansion@npm:^5.0.2":+ version: 5.0.2+ resolution: "brace-expansion@npm:5.0.2"+ dependencies:+ balanced-match: "npm:^4.0.2"+ checksum: 10c0/60c765e5df6fc0ceca3d5703202ae6779db61f28ea3bf93a04dbf0d50c22ef8e4644e09d0459c827077cd2d09ba8f199a04d92c36419fcf874601a5565013174+ languageName: node+ linkType: hard+"braces@npm:^3.0.3":version: 3.0.3resolution: "braces@npm:3.0.3"@@ -4960,15 +4985,16 @@ __metadata:languageName: nodelinkType: hard-"camoufox-js@npm:^0.8.0":- version: 0.8.0- resolution: "camoufox-js@npm:0.8.0"+"camoufox-js@npm:^0.9.0":+ version: 0.9.1+ resolution: "camoufox-js@npm:0.9.1"dependencies:adm-zip: "npm:^0.5.16"better-sqlite3: "npm:^12.2.0"commander: "npm:^14.0.0"fingerprint-generator: "npm:^2.1.66"- impit: "npm:^0.6.0"+ glob: "npm:^13.0.0"+ impit: "npm:^0.9.0"language-tags: "npm:^2.0.1"maxmind: "npm:^5.0.0"progress: "npm:^2.0.3"@@ -4978,7 +5004,7 @@ __metadata:playwright-core: "*"bin:camoufox-js: dist/__main__.js- checksum: 10c0/31c8a4b2e314d968b3a94d13be8fdf31be3d27598bfdac72a19ac91481c952bef6ee6cee4c9379e85562e87d5e4d72dc2158b33330e9c900e016fc0a114150fd+ checksum: 10c0/1dc6dd49b5e58930c5fc9cccd9f72e4040b39d348b139aa9c0b1ff95847ba8d85aff25d56e7643d0eb3ec98a9ed66c92814ad0a81a197ed6c84223519b4332f7languageName: nodelinkType: hard@@ -7909,6 +7935,17 @@ __metadata:languageName: nodelinkType: hard+"glob@npm:^13.0.0":+ version: 13.0.5+ resolution: "glob@npm:13.0.5"+ dependencies:+ minimatch: "npm:^10.2.1"+ minipass: "npm:^7.1.2"+ path-scurry: "npm:^2.0.0"+ checksum: 10c0/1388527676127f337877eaf3403d6c54d3fa5e5599e10c1532d73108435b4da66d8fff4b00eb5b306388090a180c6a92d70694df1c19171cf820e285fb1dfee5+ languageName: node+ linkType: hard+"glob@npm:^9.2.0":version: 9.3.5resolution: "glob@npm:9.3.5"@@ -8004,9 +8041,9 @@ __metadata:languageName: nodelinkType: hard-"got-scraping@npm:^4.0.0, got-scraping@npm:^4.0.3":- version: 4.1.2- resolution: "got-scraping@npm:4.1.2"+"got-scraping@npm:^4.2.1":+ version: 4.2.1+ resolution: "got-scraping@npm:4.2.1"dependencies:got: "npm:^14.2.1"header-generator: "npm:^2.1.41"@@ -8015,7 +8052,7 @@ __metadata:ow: "npm:^1.1.1"quick-lru: "npm:^7.0.0"tslib: "npm:^2.6.2"- checksum: 10c0/7667637ad9ad3c8506522b4721456e61392e51230f3d7c988a0ef58b223d806869adc1adc61e7b8299f533b154f26343f6960f8ade7ce944336ecb4b5f3b7057+ checksum: 10c0/26a5779a2dbb084cd36cd3be1b6d243e3ddd69d558c26d786b03c997b5faabc78d5109a2d9b0a28531bc54a87e82976db6ead68759bc3e953bc60fdc67e2514flanguageName: nodelinkType: hard@@ -8451,13 +8488,6 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-arm64@npm:0.6.1":- version: 0.6.1- resolution: "impit-darwin-arm64@npm:0.6.1"- conditions: os=darwin & cpu=arm64- languageName: node- linkType: hard-"impit-darwin-arm64@npm:0.9.0":version: 0.9.0resolution: "impit-darwin-arm64@npm:0.9.0"@@ -8465,13 +8495,6 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-x64@npm:0.6.1":- version: 0.6.1- resolution: "impit-darwin-x64@npm:0.6.1"- conditions: os=darwin & cpu=x64- languageName: node- linkType: hard-"impit-darwin-x64@npm:0.9.0":version: 0.9.0resolution: "impit-darwin-x64@npm:0.9.0"@@ -8479,13 +8502,6 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-gnu@npm:0.6.1":- version: 0.6.1- resolution: "impit-linux-arm64-gnu@npm:0.6.1"- conditions: os=linux & cpu=arm64 & libc=glibc- languageName: node- linkType: hard-"impit-linux-arm64-gnu@npm:0.9.0":version: 0.9.0resolution: "impit-linux-arm64-gnu@npm:0.9.0"@@ -8493,13 +8509,6 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-musl@npm:0.6.1":- version: 0.6.1- resolution: "impit-linux-arm64-musl@npm:0.6.1"- conditions: os=linux & cpu=arm64 & libc=musl- languageName: node- linkType: hard-"impit-linux-arm64-musl@npm:0.9.0":version: 0.9.0resolution: "impit-linux-arm64-musl@npm:0.9.0"@@ -8507,13 +8516,6 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-gnu@npm:0.6.1":- version: 0.6.1- resolution: "impit-linux-x64-gnu@npm:0.6.1"- conditions: os=linux & cpu=x64 & libc=glibc- languageName: node- linkType: hard-"impit-linux-x64-gnu@npm:0.9.0":version: 0.9.0resolution: "impit-linux-x64-gnu@npm:0.9.0"@@ -8521,13 +8523,6 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-musl@npm:0.6.1":- version: 0.6.1- resolution: "impit-linux-x64-musl@npm:0.6.1"- conditions: os=linux & cpu=x64 & libc=musl- languageName: node- linkType: hard-"impit-linux-x64-musl@npm:0.9.0":version: 0.9.0resolution: "impit-linux-x64-musl@npm:0.9.0"@@ -8535,13 +8530,6 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-arm64-msvc@npm:0.6.1":- version: 0.6.1- resolution: "impit-win32-arm64-msvc@npm:0.6.1"- conditions: os=win32 & cpu=arm64- languageName: node- linkType: hard-"impit-win32-arm64-msvc@npm:0.9.0":version: 0.9.0resolution: "impit-win32-arm64-msvc@npm:0.9.0"@@ -8549,13 +8537,6 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-x64-msvc@npm:0.6.1":- version: 0.6.1- resolution: "impit-win32-x64-msvc@npm:0.6.1"- conditions: os=win32 & cpu=x64- languageName: node- linkType: hard-"impit-win32-x64-msvc@npm:0.9.0":version: 0.9.0resolution: "impit-win32-x64-msvc@npm:0.9.0"@@ -8563,39 +8544,6 @@ __metadata:languageName: nodelinkType: hard-"impit@npm:^0.6.0":- version: 0.6.1- resolution: "impit@npm:0.6.1"- dependencies:- impit-darwin-arm64: "npm:0.6.1"- impit-darwin-x64: "npm:0.6.1"- impit-linux-arm64-gnu: "npm:0.6.1"- impit-linux-arm64-musl: "npm:0.6.1"- impit-linux-x64-gnu: "npm:0.6.1"- impit-linux-x64-musl: "npm:0.6.1"- impit-win32-arm64-msvc: "npm:0.6.1"- impit-win32-x64-msvc: "npm:0.6.1"- dependenciesMeta:- impit-darwin-arm64:- optional: true- impit-darwin-x64:- optional: true- impit-linux-arm64-gnu:- optional: true- impit-linux-arm64-musl:- optional: true- impit-linux-x64-gnu:- optional: true- impit-linux-x64-musl:- optional: true- impit-win32-arm64-msvc:- optional: true- impit-win32-x64-msvc:- optional: true- checksum: 10c0/deafa6540355ae6aa7dc8bee310b6e551d8928a4d2b412475238bf83486905bc41747b476f29ff27a1b805263a2ba9ed463d4534b73544c3825dd6946e1216e9- languageName: node- linkType: hard-"impit@npm:^0.9.0":version: 0.9.0resolution: "impit@npm:0.9.0"@@ -9330,6 +9278,15 @@ __metadata:languageName: nodelinkType: hard+"jackspeak@npm:^4.2.3":+ version: 4.2.3+ resolution: "jackspeak@npm:4.2.3"+ dependencies:+ "@isaacs/cliui": "npm:^9.0.0"+ checksum: 10c0/b5c0c414f1607c2aa0597f4bf2c03b8443897fccd5fd3c2b3e4f77d556b2bc7c3d3413828ba91e0789f6fb40ad90242f7f89fb20aee9e9d705bc1681f7564f67+ languageName: node+ linkType: hard+"jake@npm:^10.8.5":version: 10.9.4resolution: "jake@npm:10.9.4"@@ -10050,9 +10007,9 @@ __metadata:linkType: hard"lodash@npm:^4.17.21":- version: 4.17.21- resolution: "lodash@npm:4.17.21"- checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c+ version: 4.17.23+ resolution: "lodash@npm:4.17.23"+ checksum: 10c0/1264a90469f5bb95d4739c43eb6277d15b6d9e186df4ac68c3620443160fc669e2f14c11e7d8b2ccf078b81d06147c01a8ccced9aab9f9f63d50dcf8cace6bf6languageName: nodelinkType: hard@@ -10426,6 +10383,15 @@ __metadata:languageName: nodelinkType: hard+"minimatch@npm:^10.2.1":+ version: 10.2.1+ resolution: "minimatch@npm:10.2.1"+ dependencies:+ brace-expansion: "npm:^5.0.2"+ checksum: 10c0/86c3ed013630e820fda00336ee786a03098723b60bfae452de6306708fc83619df40a99dc6ec59c97d14e25b3b3371669a04e5bf508b1b00339b20229c4907d2+ languageName: node+ linkType: hard+"minimatch@npm:^3.0.4, minimatch@npm:^3.1.2":version: 3.1.2resolution: "minimatch@npm:3.1.2"