Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: bump the patch version when computing canary versions automatically
Martin Adámek84aa0c5f6f3d
diff --git a/lerna.json b/lerna.jsonindex dbd65120e..1272c1037 100644--- a/lerna.json+++ b/lerna.json@@ -2,7 +2,7 @@"packages": ["packages/*"],- "version": "3.0.4",+ "version": "3.0.3","command": {"version": {"conventionalCommits": true,diff --git a/package.json b/package.jsonindex 93c2e8318..227d6abc2 100644--- a/package.json+++ b/package.json@@ -1,7 +1,7 @@{"name": "crawlee","private": true,- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","workspaces": ["packages/*"diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex ccbc29ea9..de511cb64 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -48,9 +48,9 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/core": "^3.0.4",- "@crawlee/types": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/core": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@crawlee/utils": "^3.0.3","got-scraping": "^3.2.9","ow": "^0.28.1"}diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex a25869189..88ffb86dd 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,11 +54,11 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.0.4",- "@crawlee/browser-pool": "^3.0.4",- "@crawlee/core": "^3.0.4",- "@crawlee/types": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/basic": "^3.0.3",+ "@crawlee/browser-pool": "^3.0.3",+ "@crawlee/core": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@crawlee/utils": "^3.0.3","ow": "^0.28.1"}}diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 28a821847..1fc4d21e9 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser-pool",- "version": "3.0.4",+ "version": "3.0.3","description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.","engines": {"node": ">=16.0.0"@@ -38,7 +38,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0",- "@crawlee/types": "^3.0.4",+ "@crawlee/types": "^3.0.3","fingerprint-generator": "^2.0.5","fingerprint-injector": "^2.0.4","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex e3432ab6c..a12cc5b3b 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -53,8 +53,8 @@"access": "public"},"dependencies": {- "@crawlee/http": "^3.0.4",- "@crawlee/types": "^3.0.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3","cheerio": "1.0.0-rc.12","htmlparser2": "^8.0.1"}diff --git a/packages/cli/package.json b/packages/cli/package.jsonindex 5057b3166..1bad8ea0e 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cli",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "^3.0.4",+ "@crawlee/templates": "^3.0.3","ansi-colors": "^4.1.3","fs-extra": "^10.1.0","inquirer": "^8.2.4",diff --git a/packages/core/package.json b/packages/core/package.jsonindex 325dd684f..e058c94ea 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/core",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -58,9 +58,9 @@"@apify/pseudo_url": "^2.0.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/memory-storage": "^3.0.4",- "@crawlee/types": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/memory-storage": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@crawlee/utils": "^3.0.3","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",diff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex de025727e..674c48412 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -56,16 +56,16 @@"access": "public"},"dependencies": {- "@crawlee/cli": "^3.0.4",- "@crawlee/core": "^3.0.4",- "@crawlee/basic": "^3.0.4",- "@crawlee/browser": "^3.0.4",- "@crawlee/http": "^3.0.4",- "@crawlee/cheerio": "^3.0.4",- "@crawlee/jsdom": "^3.0.4",- "@crawlee/playwright": "^3.0.4",- "@crawlee/puppeteer": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/cli": "^3.0.3",+ "@crawlee/core": "^3.0.3",+ "@crawlee/basic": "^3.0.3",+ "@crawlee/browser": "^3.0.3",+ "@crawlee/http": "^3.0.3",+ "@crawlee/cheerio": "^3.0.3",+ "@crawlee/jsdom": "^3.0.3",+ "@crawlee/playwright": "^3.0.3",+ "@crawlee/puppeteer": "^3.0.3",+ "@crawlee/utils": "^3.0.3","import-local": "^3.1.0"},"peerDependencies": {diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 220bad699..21cfe707e 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/http",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/basic": "^3.0.4",- "@crawlee/types": "^3.0.4",+ "@crawlee/basic": "^3.0.3",+ "@crawlee/types": "^3.0.3","@types/content-type": "^1.1.5","content-type": "^1.0.4","got-scraping": "^3.2.9",diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex de91e11bb..469008ef6 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/jsdom",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,8 +54,8 @@},"dependencies": {"@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.0.4",- "@crawlee/types": "^3.0.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3","@types/jsdom": "^20.0.0","jsdom": "^20.0.0"}diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 97a99669f..fd1ddcf05 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/memory-storage",- "version": "3.0.4",+ "version": "3.0.3","description": "A simple in-memory storage implementation of the Apify API","engines": {"node": ">= 16"@@ -49,8 +49,8 @@},"dependencies": {"@apify/log": "^2.0.0",- "@crawlee/types": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/types": "^3.0.3",+ "@crawlee/utils": "^3.0.3","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^10.1.0",diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 8aa9956ea..db5cd726e 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/log": "^2.0.0","@apify/datastructures": "^2.0.0",- "@crawlee/browser": "^3.0.4",- "@crawlee/browser-pool": "^3.0.4",- "@crawlee/core": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/browser": "^3.0.3",+ "@crawlee/browser-pool": "^3.0.3",+ "@crawlee/core": "^3.0.3",+ "@crawlee/utils": "^3.0.3","cheerio": "1.0.0-rc.12","jquery": "^3.6.0","ow": "^0.28.1"diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex f9096cc02..15ec8abd8 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.0.4",+ "version": "3.0.3","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.0.4",- "@crawlee/browser-pool": "^3.0.4",- "@crawlee/types": "^3.0.4",- "@crawlee/utils": "^3.0.4",+ "@crawlee/browser": "^3.0.3",+ "@crawlee/browser-pool": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@crawlee/utils": "^3.0.3","cheerio": "1.0.0-rc.12","jquery": "^3.6.0","ow": "^0.28.1"diff --git a/packages/templates/package.json b/packages/templates/package.jsonindex 8af50029d..eac62f527 100644--- a/packages/templates/package.json+++ b/packages/templates/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/templates",- "version": "3.0.4",+ "version": "3.0.3","description": "Templates for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/types/package.json b/packages/types/package.jsonindex 5fd313414..715d8d662 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/types",- "version": "3.0.4",+ "version": "3.0.3","description": "Shared types for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 076e5dd1c..5b4a93e27 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/utils",- "version": "3.0.4",+ "version": "3.0.3","description": "A set of shared utilities that can be used by crawlers","engines": {"node": ">=16.0.0"@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/ps-tree": "^1.1.4",- "@crawlee/types": "^3.0.4",+ "@crawlee/types": "^3.0.3","cheerio": "1.0.0-rc.12","got-scraping": "^3.2.9","tslib": "^2.4.0"diff --git a/scripts/copy.ts b/scripts/copy.tsindex 19c499b1b..3f76246b5 100644--- a/scripts/copy.ts+++ b/scripts/copy.ts@@ -31,13 +31,17 @@ function getRootVersion(): string {}// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-dynamic-require,global-require- rootVersion = require(resolve(root, './packages/core/package.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1');+ rootVersion = require(resolve(root, './packages/crawlee/package.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1');++ const parts = rootVersion.split('.');+ parts[2] = `${+parts[2] + 1}`;+ rootVersion = parts.join('.');return rootVersion;}/**- * Checks next dev version number based on the `@crawlee/core` package via `npm show`.+ * Checks next dev version number based on the `crawlee` meta package via `npm show`.* We always use this package, so we ensure the version is the same for each package in the monorepo.*/function getNextVersion() {