Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): update internal dependencies [skip ci]
Apify Release Bot43786925843b
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 42093762a..f64c7eb2f 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -48,9 +48,9 @@"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/core": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/core": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","got-scraping": "^3.2.15","ow": "^0.28.1","tldts": "^6.0.0",diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 240152de5..7b77285ba 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -54,10 +54,10 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.5.8",- "@crawlee/browser-pool": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/basic": "3.5.8",+ "@crawlee/browser-pool": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","ow": "^0.28.1","tslib": "^2.4.0"}diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 5e4befefb..1ab09692f 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -38,8 +38,8 @@"dependencies": {"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0",- "@crawlee/core": "^3.5.8",- "@crawlee/types": "^3.5.8",+ "@crawlee/core": "3.5.8",+ "@crawlee/types": "3.5.8","fingerprint-generator": "^2.0.6","fingerprint-injector": "^2.0.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex 7ff96242b..955b6e597 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -53,8 +53,8 @@"access": "public"},"dependencies": {- "@crawlee/http": "^3.5.8",- "@crawlee/types": "^3.5.8",+ "@crawlee/http": "3.5.8",+ "@crawlee/types": "3.5.8","cheerio": "^1.0.0-rc.12","htmlparser2": "^9.0.0","tslib": "^2.4.0"diff --git a/packages/cli/package.json b/packages/cli/package.jsonindex da6b69c4f..a075a0d36 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "^3.5.8",+ "@crawlee/templates": "3.5.8","ansi-colors": "^4.1.3","fs-extra": "^11.0.0","inquirer": "^8.2.4",diff --git a/packages/core/package.json b/packages/core/package.jsonindex 3052610c8..b1134a45e 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -59,9 +59,9 @@"@apify/pseudo_url": "^2.0.30","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/memory-storage": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/memory-storage": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","@sapphire/async-queue": "^1.5.0","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.2.2",diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex a258c7db1..453380eb2 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -54,18 +54,18 @@"access": "public"},"dependencies": {- "@crawlee/basic": "^3.5.8",- "@crawlee/browser": "^3.5.8",- "@crawlee/browser-pool": "^3.5.8",- "@crawlee/cheerio": "^3.5.8",- "@crawlee/cli": "^3.5.8",- "@crawlee/core": "^3.5.8",- "@crawlee/http": "^3.5.8",- "@crawlee/jsdom": "^3.5.8",- "@crawlee/linkedom": "^3.5.8",- "@crawlee/playwright": "^3.5.8",- "@crawlee/puppeteer": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/basic": "3.5.8",+ "@crawlee/browser": "3.5.8",+ "@crawlee/browser-pool": "3.5.8",+ "@crawlee/cheerio": "3.5.8",+ "@crawlee/cli": "3.5.8",+ "@crawlee/core": "3.5.8",+ "@crawlee/http": "3.5.8",+ "@crawlee/jsdom": "3.5.8",+ "@crawlee/linkedom": "3.5.8",+ "@crawlee/playwright": "3.5.8",+ "@crawlee/puppeteer": "3.5.8",+ "@crawlee/utils": "3.5.8","import-local": "^3.1.0","tslib": "^2.4.0"},diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 08dcbe9ca..e90a71ac0 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/basic": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/basic": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","@types/content-type": "^1.1.5","cheerio": "^1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex cb8b9f539..2149bd69a 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "^3.5.8",- "@crawlee/types": "^3.5.8",+ "@crawlee/http": "3.5.8",+ "@crawlee/types": "3.5.8","@types/jsdom": "^21.0.0","cheerio": "^1.0.0-rc.12","jsdom": "^22.0.0",diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex 6fc35f62a..c7c8620eb 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "^3.5.8",- "@crawlee/types": "^3.5.8",+ "@crawlee/http": "3.5.8",+ "@crawlee/types": "3.5.8","linkedom": "^0.15.0","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex b7c1ccf8b..3531fab77 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -49,7 +49,7 @@},"dependencies": {"@apify/log": "^2.4.0",- "@crawlee/types": "^3.5.8",+ "@crawlee/types": "3.5.8","@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4",diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 5584ae3ae..047fb8ffd 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "^3.5.8",- "@crawlee/browser-pool": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/browser": "3.5.8",+ "@crawlee/browser-pool": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","cheerio": "^1.0.0-rc.12","idcac-playwright": "^0.1.2","jquery": "^3.6.0",diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex 0caa512f0..e4dc0d2bf 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "^3.5.8",- "@crawlee/browser-pool": "^3.5.8",- "@crawlee/types": "^3.5.8",- "@crawlee/utils": "^3.5.8",+ "@crawlee/browser": "3.5.8",+ "@crawlee/browser-pool": "3.5.8",+ "@crawlee/types": "3.5.8",+ "@crawlee/utils": "3.5.8","cheerio": "^1.0.0-rc.12","devtools-protocol": "*","idcac-playwright": "^0.1.2",diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 62ee6ac44..8980dfabd 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.4.0","@apify/ps-tree": "^1.2.0",- "@crawlee/types": "^3.5.8",+ "@crawlee/types": "3.5.8","cheerio": "^1.0.0-rc.12","got-scraping": "^3.2.15","ow": "^0.28.1",diff --git a/yarn.lock b/yarn.lockindex 115b78027..c3a933578 100644--- a/yarn.lock+++ b/yarn.lock@@ -379,16 +379,16 @@ __metadata:languageName: nodelinkType: hard-"@crawlee/basic@^3.5.8, @crawlee/basic@workspace:packages/basic-crawler":+"@crawlee/[email protected], @crawlee/basic@workspace:packages/basic-crawler":version: 0.0.0-use.localresolution: "@crawlee/basic@workspace:packages/basic-crawler"dependencies:"@apify/log": ^2.4.0"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/core": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/core": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8got-scraping: ^3.2.15ow: ^0.28.1tldts: ^6.0.0@@ -397,14 +397,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser-pool@^3.5.8, @crawlee/browser-pool@workspace:packages/browser-pool":+"@crawlee/[email protected], @crawlee/browser-pool@workspace:packages/browser-pool":version: 0.0.0-use.localresolution: "@crawlee/browser-pool@workspace:packages/browser-pool"dependencies:"@apify/log": ^2.4.0"@apify/timeout": ^0.3.0- "@crawlee/core": ^3.5.8- "@crawlee/types": ^3.5.8+ "@crawlee/core": 3.5.8+ "@crawlee/types": 3.5.8fingerprint-generator: ^2.0.6fingerprint-injector: ^2.0.5lodash.merge: ^4.6.2@@ -426,37 +426,37 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser@^3.5.8, @crawlee/browser@workspace:packages/browser-crawler":+"@crawlee/[email protected], @crawlee/browser@workspace:packages/browser-crawler":version: 0.0.0-use.localresolution: "@crawlee/browser@workspace:packages/browser-crawler"dependencies:"@apify/timeout": ^0.3.0- "@crawlee/basic": ^3.5.8- "@crawlee/browser-pool": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/basic": 3.5.8+ "@crawlee/browser-pool": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8ow: ^0.28.1tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cheerio@^3.5.8, @crawlee/cheerio@workspace:packages/cheerio-crawler":+"@crawlee/[email protected], @crawlee/cheerio@workspace:packages/cheerio-crawler":version: 0.0.0-use.localresolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"dependencies:- "@crawlee/http": ^3.5.8- "@crawlee/types": ^3.5.8+ "@crawlee/http": 3.5.8+ "@crawlee/types": 3.5.8cheerio: ^1.0.0-rc.12htmlparser2: ^9.0.0tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cli@^3.5.8, @crawlee/cli@workspace:packages/cli":+"@crawlee/[email protected], @crawlee/cli@workspace:packages/cli":version: 0.0.0-use.localresolution: "@crawlee/cli@workspace:packages/cli"dependencies:- "@crawlee/templates": ^3.5.8+ "@crawlee/templates": 3.5.8ansi-colors: ^4.1.3fs-extra: ^11.0.0inquirer: ^8.2.4@@ -468,7 +468,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/core@^3.5.6, @crawlee/core@^3.5.8, @crawlee/core@workspace:packages/core":+"@crawlee/[email protected], @crawlee/core@^3.5.6, @crawlee/core@workspace:packages/core":version: 0.0.0-use.localresolution: "@crawlee/core@workspace:packages/core"dependencies:@@ -478,9 +478,9 @@ __metadata:"@apify/pseudo_url": ^2.0.30"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/memory-storage": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/memory-storage": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8"@sapphire/async-queue": ^1.5.0"@types/tough-cookie": ^4.0.2"@vladfrangu/async_event_emitter": ^2.2.2@@ -498,15 +498,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@^3.5.8, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/[email protected], @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/basic": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/basic": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8"@types/content-type": ^1.1.5cheerio: ^1.0.0-rc.12content-type: ^1.0.4@@ -519,14 +519,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/jsdom@^3.5.8, @crawlee/jsdom@workspace:packages/jsdom-crawler":+"@crawlee/[email protected], @crawlee/jsdom@workspace:packages/jsdom-crawler":version: 0.0.0-use.localresolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/http": ^3.5.8- "@crawlee/types": ^3.5.8+ "@crawlee/http": 3.5.8+ "@crawlee/types": 3.5.8"@types/jsdom": ^21.0.0cheerio: ^1.0.0-rc.12jsdom: ^22.0.0@@ -535,26 +535,26 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/linkedom@^3.5.8, @crawlee/linkedom@workspace:packages/linkedom-crawler":+"@crawlee/[email protected], @crawlee/linkedom@workspace:packages/linkedom-crawler":version: 0.0.0-use.localresolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.7.10- "@crawlee/http": ^3.5.8- "@crawlee/types": ^3.5.8+ "@crawlee/http": 3.5.8+ "@crawlee/types": 3.5.8linkedom: ^0.15.0ow: ^0.28.2tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/memory-storage@^3.5.8, @crawlee/memory-storage@workspace:packages/memory-storage":+"@crawlee/[email protected], @crawlee/memory-storage@workspace:packages/memory-storage":version: 0.0.0-use.localresolution: "@crawlee/memory-storage@workspace:packages/memory-storage"dependencies:"@apify/log": ^2.4.0- "@crawlee/types": ^3.5.8+ "@crawlee/types": 3.5.8"@sapphire/async-queue": ^1.5.0"@sapphire/shapeshift": ^3.0.0content-type: ^1.0.4@@ -566,16 +566,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/playwright@^3.5.8, @crawlee/playwright@workspace:packages/playwright-crawler":+"@crawlee/[email protected], @crawlee/playwright@workspace:packages/playwright-crawler":version: 0.0.0-use.localresolution: "@crawlee/playwright@workspace:packages/playwright-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.4.0- "@crawlee/browser": ^3.5.8- "@crawlee/browser-pool": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/browser": 3.5.8+ "@crawlee/browser-pool": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8cheerio: ^1.0.0-rc.12idcac-playwright: ^0.1.2jquery: ^3.6.0@@ -589,16 +589,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/puppeteer@^3.5.8, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":+"@crawlee/[email protected], @crawlee/puppeteer@workspace:packages/puppeteer-crawler":version: 0.0.0-use.localresolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.4.0- "@crawlee/browser": ^3.5.8- "@crawlee/browser-pool": ^3.5.8- "@crawlee/types": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/browser": 3.5.8+ "@crawlee/browser-pool": 3.5.8+ "@crawlee/types": 3.5.8+ "@crawlee/utils": 3.5.8cheerio: ^1.0.0-rc.12devtools-protocol: "*"idcac-playwright: ^0.1.2@@ -675,7 +675,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/templates@^3.5.8, @crawlee/templates@workspace:packages/templates":+"@crawlee/[email protected], @crawlee/templates@workspace:packages/templates":version: 0.0.0-use.localresolution: "@crawlee/templates@workspace:packages/templates"dependencies:@@ -687,7 +687,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@^3.3.0, @crawlee/types@^3.5.6, @crawlee/types@^3.5.8, @crawlee/types@workspace:packages/types":+"@crawlee/[email protected], @crawlee/types@^3.3.0, @crawlee/types@^3.5.6, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -695,13 +695,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/utils@^3.5.6, @crawlee/utils@^3.5.8, @crawlee/utils@workspace:packages/utils":+"@crawlee/[email protected], @crawlee/utils@^3.5.6, @crawlee/utils@workspace:packages/utils":version: 0.0.0-use.localresolution: "@crawlee/utils@workspace:packages/utils"dependencies:"@apify/log": ^2.4.0"@apify/ps-tree": ^1.2.0- "@crawlee/types": ^3.5.8+ "@crawlee/types": 3.5.8cheerio: ^1.0.0-rc.12got-scraping: ^3.2.15ow: ^0.28.1@@ -3798,18 +3798,18 @@ __metadata:version: 0.0.0-use.localresolution: "crawlee@workspace:packages/crawlee"dependencies:- "@crawlee/basic": ^3.5.8- "@crawlee/browser": ^3.5.8- "@crawlee/browser-pool": ^3.5.8- "@crawlee/cheerio": ^3.5.8- "@crawlee/cli": ^3.5.8- "@crawlee/core": ^3.5.8- "@crawlee/http": ^3.5.8- "@crawlee/jsdom": ^3.5.8- "@crawlee/linkedom": ^3.5.8- "@crawlee/playwright": ^3.5.8- "@crawlee/puppeteer": ^3.5.8- "@crawlee/utils": ^3.5.8+ "@crawlee/basic": 3.5.8+ "@crawlee/browser": 3.5.8+ "@crawlee/browser-pool": 3.5.8+ "@crawlee/cheerio": 3.5.8+ "@crawlee/cli": 3.5.8+ "@crawlee/core": 3.5.8+ "@crawlee/http": 3.5.8+ "@crawlee/jsdom": 3.5.8+ "@crawlee/linkedom": 3.5.8+ "@crawlee/playwright": 3.5.8+ "@crawlee/puppeteer": 3.5.8+ "@crawlee/utils": 3.5.8import-local: ^3.1.0tslib: ^2.4.0peerDependencies: