Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: bump version to 3.0.2
Martin Adámek4c9e5c3f052b
diff --git a/lerna.json b/lerna.jsonindex 0a16c7fcd..a7cfa5818 100644--- a/lerna.json+++ b/lerna.json@@ -2,7 +2,7 @@"packages": ["packages/*"],- "version": "3.0.1",+ "version": "3.0.2","command": {"version": {"conventionalCommits": true,diff --git a/package.json b/package.jsonindex e55641276..e610cf68f 100644--- a/package.json+++ b/package.json@@ -1,7 +1,7 @@{"name": "crawlee","private": true,- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","workspaces": ["packages/*"diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 0ee693100..f4ba3eb6d 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -48,9 +48,9 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/core": "^3.0.1",- "@crawlee/types": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/core": "^3.0.2",+ "@crawlee/types": "^3.0.2",+ "@crawlee/utils": "^3.0.2","got-scraping": "^3.2.9","ow": "^0.28.1"}diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 0046516a2..9b5896765 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,11 +54,11 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.0.1",- "@crawlee/browser-pool": "^3.0.1",- "@crawlee/core": "^3.0.1",- "@crawlee/types": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/basic": "^3.0.2",+ "@crawlee/browser-pool": "^3.0.2",+ "@crawlee/core": "^3.0.2",+ "@crawlee/types": "^3.0.2",+ "@crawlee/utils": "^3.0.2","ow": "^0.28.1"}}diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 57c255592..d71f69d56 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser-pool",- "version": "3.0.1",+ "version": "3.0.2","description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.","engines": {"node": ">=16.0.0"@@ -38,7 +38,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0",- "@crawlee/types": "^3.0.1",+ "@crawlee/types": "^3.0.2","fingerprint-generator": "^2.0.0-dev.5","fingerprint-injector": "^2.0.0-dev.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex a13768d43..a9ddf4861 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,7 +55,7 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/basic": "^3.0.1",+ "@crawlee/basic": "^3.0.2","@types/content-type": "^1.1.5","cheerio": "1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/cli/package.json b/packages/cli/package.jsonindex a75d308da..d8dcaff91 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cli",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "^3.0.1",+ "@crawlee/templates": "^3.0.2","ansi-colors": "^4.1.3","fs-extra": "^10.1.0","inquirer": "^8.2.4",diff --git a/packages/core/package.json b/packages/core/package.jsonindex b5ac5e7f1..b7494db36 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/core",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -58,9 +58,9 @@"@apify/pseudo_url": "^2.0.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/memory-storage": "^3.0.1",- "@crawlee/types": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/memory-storage": "^3.0.2",+ "@crawlee/types": "^3.0.2",+ "@crawlee/utils": "^3.0.2","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex e909dcbad..272bf7c58 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -56,14 +56,14 @@"access": "public"},"dependencies": {- "@crawlee/cli": "^3.0.1",- "@crawlee/core": "^3.0.1",- "@crawlee/basic": "^3.0.1",- "@crawlee/browser": "^3.0.1",- "@crawlee/cheerio": "^3.0.1",- "@crawlee/playwright": "^3.0.1",- "@crawlee/puppeteer": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/cli": "^3.0.2",+ "@crawlee/core": "^3.0.2",+ "@crawlee/basic": "^3.0.2",+ "@crawlee/browser": "^3.0.2",+ "@crawlee/cheerio": "^3.0.2",+ "@crawlee/playwright": "^3.0.2",+ "@crawlee/puppeteer": "^3.0.2",+ "@crawlee/utils": "^3.0.2","import-local": "^3.1.0"},"peerDependencies": {diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 6f094efe7..6f2858631 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/memory-storage",- "version": "3.0.1",+ "version": "3.0.2","description": "A simple in-memory storage implementation of the Apify API","engines": {"node": ">= 16"@@ -49,8 +49,8 @@},"dependencies": {"@apify/log": "^2.0.0",- "@crawlee/types": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/types": "^3.0.2",+ "@crawlee/utils": "^3.0.2","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^10.1.0",diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex b2f96d886..f761206cd 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/log": "^2.0.0","@apify/datastructures": "^2.0.0",- "@crawlee/browser": "^3.0.1",- "@crawlee/browser-pool": "^3.0.1",- "@crawlee/core": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/browser": "^3.0.2",+ "@crawlee/browser-pool": "^3.0.2",+ "@crawlee/core": "^3.0.2",+ "@crawlee/utils": "^3.0.2","cheerio": "1.0.0-rc.12","jquery": "^3.6.0","ow": "^0.28.1"diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex 8161ec35b..cdb108d2f 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.0.1",+ "version": "3.0.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.0.1",- "@crawlee/browser-pool": "^3.0.1",- "@crawlee/types": "^3.0.1",- "@crawlee/utils": "^3.0.1",+ "@crawlee/browser": "^3.0.2",+ "@crawlee/browser-pool": "^3.0.2",+ "@crawlee/types": "^3.0.2",+ "@crawlee/utils": "^3.0.2","cheerio": "1.0.0-rc.12","jquery": "^3.6.0","ow": "^0.28.1"diff --git a/packages/templates/package.json b/packages/templates/package.jsonindex 060d9b296..7ed404913 100644--- a/packages/templates/package.json+++ b/packages/templates/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/templates",- "version": "3.0.1",+ "version": "3.0.2","description": "Templates for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/types/package.json b/packages/types/package.jsonindex 855db7598..259d0bede 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/types",- "version": "3.0.1",+ "version": "3.0.2","description": "Shared types for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 942ca0430..1906471f6 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/utils",- "version": "3.0.1",+ "version": "3.0.2","description": "A set of shared utilities that can be used by crawlers","engines": {"node": ">=16.0.0"@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/ps-tree": "^1.1.4",- "@crawlee/types": "^3.0.1",+ "@crawlee/types": "^3.0.2","cheerio": "1.0.0-rc.12","content-type": "^1.0.4","got-scraping": "^3.2.9",