Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): update internal dependencies [skip ci]
Martin Adámek742f32efb9d3
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex a01b5292c..34ba8adea 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -48,9 +48,9 @@"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/core": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/core": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","got-scraping": "^3.2.9","ow": "^0.28.1","tldts": "^6.0.0",diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 1cd2b9cc9..dc1fd7cd3 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -54,10 +54,10 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.5.3",- "@crawlee/browser-pool": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/basic": "3.5.3",+ "@crawlee/browser-pool": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","ow": "^0.28.1","tslib": "^2.4.0"}diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 36a0cac43..b46a68207 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -38,8 +38,8 @@"dependencies": {"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0",- "@crawlee/core": "^3.5.3",- "@crawlee/types": "^3.5.3",+ "@crawlee/core": "3.5.3",+ "@crawlee/types": "3.5.3","fingerprint-generator": "^2.0.6","fingerprint-injector": "^2.0.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex 11caf83c2..5a839c1c5 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -53,8 +53,8 @@"access": "public"},"dependencies": {- "@crawlee/http": "^3.5.3",- "@crawlee/types": "^3.5.3",+ "@crawlee/http": "3.5.3",+ "@crawlee/types": "3.5.3","cheerio": "^1.0.0-rc.12","htmlparser2": "^9.0.0","tslib": "^2.4.0"diff --git a/packages/cli/package.json b/packages/cli/package.jsonindex badc9583f..a3cbe4a67 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "^3.5.3",+ "@crawlee/templates": "3.5.3","ansi-colors": "^4.1.3","fs-extra": "^11.0.0","inquirer": "^8.2.4",diff --git a/packages/core/package.json b/packages/core/package.jsonindex e77c9ba4f..fee79ec2e 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -59,9 +59,9 @@"@apify/pseudo_url": "^2.0.30","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/memory-storage": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/memory-storage": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","@sapphire/async-queue": "^1.5.0","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.2.2",diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 589c1d543..09e63fd83 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -54,18 +54,18 @@"access": "public"},"dependencies": {- "@crawlee/basic": "^3.5.3",- "@crawlee/browser": "^3.5.3",- "@crawlee/browser-pool": "^3.5.3",- "@crawlee/cheerio": "^3.5.3",- "@crawlee/cli": "^3.5.3",- "@crawlee/core": "^3.5.3",- "@crawlee/http": "^3.5.3",- "@crawlee/jsdom": "^3.5.3",- "@crawlee/linkedom": "^3.5.3",- "@crawlee/playwright": "^3.5.3",- "@crawlee/puppeteer": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/basic": "3.5.3",+ "@crawlee/browser": "3.5.3",+ "@crawlee/browser-pool": "3.5.3",+ "@crawlee/cheerio": "3.5.3",+ "@crawlee/cli": "3.5.3",+ "@crawlee/core": "3.5.3",+ "@crawlee/http": "3.5.3",+ "@crawlee/jsdom": "3.5.3",+ "@crawlee/linkedom": "3.5.3",+ "@crawlee/playwright": "3.5.3",+ "@crawlee/puppeteer": "3.5.3",+ "@crawlee/utils": "3.5.3","import-local": "^3.1.0","tslib": "^2.4.0"},diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 66e4cc114..a2eeb8327 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/basic": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/basic": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","@types/content-type": "^1.1.5","cheerio": "^1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex 3abed20dc..d993e8ff1 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "^3.5.3",- "@crawlee/types": "^3.5.3",+ "@crawlee/http": "3.5.3",+ "@crawlee/types": "3.5.3","@types/jsdom": "^21.0.0","cheerio": "^1.0.0-rc.12","jsdom": "^22.0.0",diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex f4d506ebf..e690edd5a 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "^3.5.3",- "@crawlee/types": "^3.5.3",+ "@crawlee/http": "3.5.3",+ "@crawlee/types": "3.5.3","linkedom": "^0.15.0","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 60eb7cdf6..f1c92bd88 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -49,7 +49,7 @@},"dependencies": {"@apify/log": "^2.4.0",- "@crawlee/types": "^3.5.3",+ "@crawlee/types": "3.5.3","@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4",diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 560e4a9af..992a2301a 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "^3.5.3",- "@crawlee/browser-pool": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/browser": "3.5.3",+ "@crawlee/browser-pool": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","cheerio": "^1.0.0-rc.12","idcac-playwright": "^0.1.2","jquery": "^3.6.0",diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex c667eeefd..b9828568a 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "^3.5.3",- "@crawlee/browser-pool": "^3.5.3",- "@crawlee/types": "^3.5.3",- "@crawlee/utils": "^3.5.3",+ "@crawlee/browser": "3.5.3",+ "@crawlee/browser-pool": "3.5.3",+ "@crawlee/types": "3.5.3",+ "@crawlee/utils": "3.5.3","cheerio": "^1.0.0-rc.12","devtools-protocol": "*","idcac-playwright": "^0.1.2",diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex cb2d3eea1..f29de3cf6 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.4.0","@apify/ps-tree": "^1.2.0",- "@crawlee/types": "^3.5.3",+ "@crawlee/types": "3.5.3","cheerio": "^1.0.0-rc.12","got-scraping": "^3.2.9","ow": "^0.28.1",diff --git a/yarn.lock b/yarn.lockindex 62a18b81c..f2a5278c6 100644--- a/yarn.lock+++ b/yarn.lock@@ -737,16 +737,16 @@ __metadata:languageName: nodelinkType: hard-"@crawlee/basic@^3.5.3, @crawlee/basic@workspace:packages/basic-crawler":+"@crawlee/[email protected], @crawlee/basic@workspace:packages/basic-crawler":version: 0.0.0-use.localresolution: "@crawlee/basic@workspace:packages/basic-crawler"dependencies:"@apify/log": ^2.4.0"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/core": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/core": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3got-scraping: ^3.2.9ow: ^0.28.1tldts: ^6.0.0@@ -755,14 +755,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser-pool@^3.5.3, @crawlee/browser-pool@workspace:packages/browser-pool":+"@crawlee/[email protected], @crawlee/browser-pool@workspace:packages/browser-pool":version: 0.0.0-use.localresolution: "@crawlee/browser-pool@workspace:packages/browser-pool"dependencies:"@apify/log": ^2.4.0"@apify/timeout": ^0.3.0- "@crawlee/core": ^3.5.3- "@crawlee/types": ^3.5.3+ "@crawlee/core": 3.5.3+ "@crawlee/types": 3.5.3fingerprint-generator: ^2.0.6fingerprint-injector: ^2.0.5lodash.merge: ^4.6.2@@ -784,37 +784,37 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser@^3.5.3, @crawlee/browser@workspace:packages/browser-crawler":+"@crawlee/[email protected], @crawlee/browser@workspace:packages/browser-crawler":version: 0.0.0-use.localresolution: "@crawlee/browser@workspace:packages/browser-crawler"dependencies:"@apify/timeout": ^0.3.0- "@crawlee/basic": ^3.5.3- "@crawlee/browser-pool": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/basic": 3.5.3+ "@crawlee/browser-pool": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3ow: ^0.28.1tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cheerio@^3.5.3, @crawlee/cheerio@workspace:packages/cheerio-crawler":+"@crawlee/[email protected], @crawlee/cheerio@workspace:packages/cheerio-crawler":version: 0.0.0-use.localresolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"dependencies:- "@crawlee/http": ^3.5.3- "@crawlee/types": ^3.5.3+ "@crawlee/http": 3.5.3+ "@crawlee/types": 3.5.3cheerio: ^1.0.0-rc.12htmlparser2: ^9.0.0tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cli@^3.5.3, @crawlee/cli@workspace:packages/cli":+"@crawlee/[email protected], @crawlee/cli@workspace:packages/cli":version: 0.0.0-use.localresolution: "@crawlee/cli@workspace:packages/cli"dependencies:- "@crawlee/templates": ^3.5.3+ "@crawlee/templates": 3.5.3ansi-colors: ^4.1.3fs-extra: ^11.0.0inquirer: ^8.2.4@@ -826,7 +826,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/core@^3.4.2, @crawlee/core@^3.5.3, @crawlee/core@workspace:packages/core":+"@crawlee/[email protected], @crawlee/core@^3.4.2, @crawlee/core@workspace:packages/core":version: 0.0.0-use.localresolution: "@crawlee/core@workspace:packages/core"dependencies:@@ -836,9 +836,9 @@ __metadata:"@apify/pseudo_url": ^2.0.30"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/memory-storage": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/memory-storage": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3"@sapphire/async-queue": ^1.5.0"@types/tough-cookie": ^4.0.2"@vladfrangu/async_event_emitter": ^2.2.2@@ -856,15 +856,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@^3.5.3, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/[email protected], @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/basic": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/basic": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3"@types/content-type": ^1.1.5cheerio: ^1.0.0-rc.12content-type: ^1.0.4@@ -877,14 +877,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/jsdom@^3.5.3, @crawlee/jsdom@workspace:packages/jsdom-crawler":+"@crawlee/[email protected], @crawlee/jsdom@workspace:packages/jsdom-crawler":version: 0.0.0-use.localresolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/http": ^3.5.3- "@crawlee/types": ^3.5.3+ "@crawlee/http": 3.5.3+ "@crawlee/types": 3.5.3"@types/jsdom": ^21.0.0cheerio: ^1.0.0-rc.12jsdom: ^22.0.0@@ -893,26 +893,26 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/linkedom@^3.5.3, @crawlee/linkedom@workspace:packages/linkedom-crawler":+"@crawlee/[email protected], @crawlee/linkedom@workspace:packages/linkedom-crawler":version: 0.0.0-use.localresolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/http": ^3.5.3- "@crawlee/types": ^3.5.3+ "@crawlee/http": 3.5.3+ "@crawlee/types": 3.5.3linkedom: ^0.15.0ow: ^0.28.2tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/memory-storage@^3.5.3, @crawlee/memory-storage@workspace:packages/memory-storage":+"@crawlee/[email protected], @crawlee/memory-storage@workspace:packages/memory-storage":version: 0.0.0-use.localresolution: "@crawlee/memory-storage@workspace:packages/memory-storage"dependencies:"@apify/log": ^2.4.0- "@crawlee/types": ^3.5.3+ "@crawlee/types": 3.5.3"@sapphire/async-queue": ^1.5.0"@sapphire/shapeshift": ^3.0.0content-type: ^1.0.4@@ -924,16 +924,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/playwright@^3.5.3, @crawlee/playwright@workspace:packages/playwright-crawler":+"@crawlee/[email protected], @crawlee/playwright@workspace:packages/playwright-crawler":version: 0.0.0-use.localresolution: "@crawlee/playwright@workspace:packages/playwright-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.4.0- "@crawlee/browser": ^3.5.3- "@crawlee/browser-pool": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/browser": 3.5.3+ "@crawlee/browser-pool": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3cheerio: ^1.0.0-rc.12idcac-playwright: ^0.1.2jquery: ^3.6.0@@ -947,16 +947,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/puppeteer@^3.5.3, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":+"@crawlee/[email protected], @crawlee/puppeteer@workspace:packages/puppeteer-crawler":version: 0.0.0-use.localresolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.4.0- "@crawlee/browser": ^3.5.3- "@crawlee/browser-pool": ^3.5.3- "@crawlee/types": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/browser": 3.5.3+ "@crawlee/browser-pool": 3.5.3+ "@crawlee/types": 3.5.3+ "@crawlee/utils": 3.5.3cheerio: ^1.0.0-rc.12devtools-protocol: "*"idcac-playwright: ^0.1.2@@ -1027,7 +1027,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/templates@^3.5.3, @crawlee/templates@workspace:packages/templates":+"@crawlee/[email protected], @crawlee/templates@workspace:packages/templates":version: 0.0.0-use.localresolution: "@crawlee/templates@workspace:packages/templates"dependencies:@@ -1039,7 +1039,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@^3.3.0, @crawlee/types@^3.4.2, @crawlee/types@^3.5.3, @crawlee/types@workspace:packages/types":+"@crawlee/[email protected], @crawlee/types@^3.3.0, @crawlee/types@^3.4.2, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -1047,13 +1047,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/utils@^3.4.2, @crawlee/utils@^3.5.3, @crawlee/utils@workspace:packages/utils":+"@crawlee/[email protected], @crawlee/utils@^3.4.2, @crawlee/utils@workspace:packages/utils":version: 0.0.0-use.localresolution: "@crawlee/utils@workspace:packages/utils"dependencies:"@apify/log": ^2.4.0"@apify/ps-tree": ^1.2.0- "@crawlee/types": ^3.5.3+ "@crawlee/types": 3.5.3cheerio: ^1.0.0-rc.12got-scraping: ^3.2.9ow: ^0.28.1@@ -4303,18 +4303,18 @@ __metadata:version: 0.0.0-use.localresolution: "crawlee@workspace:packages/crawlee"dependencies:- "@crawlee/basic": ^3.5.3- "@crawlee/browser": ^3.5.3- "@crawlee/browser-pool": ^3.5.3- "@crawlee/cheerio": ^3.5.3- "@crawlee/cli": ^3.5.3- "@crawlee/core": ^3.5.3- "@crawlee/http": ^3.5.3- "@crawlee/jsdom": ^3.5.3- "@crawlee/linkedom": ^3.5.3- "@crawlee/playwright": ^3.5.3- "@crawlee/puppeteer": ^3.5.3- "@crawlee/utils": ^3.5.3+ "@crawlee/basic": 3.5.3+ "@crawlee/browser": 3.5.3+ "@crawlee/browser-pool": 3.5.3+ "@crawlee/cheerio": 3.5.3+ "@crawlee/cli": 3.5.3+ "@crawlee/core": 3.5.3+ "@crawlee/http": 3.5.3+ "@crawlee/jsdom": 3.5.3+ "@crawlee/linkedom": 3.5.3+ "@crawlee/playwright": 3.5.3+ "@crawlee/puppeteer": 3.5.3+ "@crawlee/utils": 3.5.3import-local: ^3.1.0tslib: ^2.4.0peerDependencies: