Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: bump the patch version when computing canary versions automatically

Martin Adámek84aa0c5f6f3d
diff --git a/lerna.json b/lerna.json
index dbd65120e..1272c1037 100644
--- a/lerna.json
+++ b/lerna.json
@@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
- "version": "3.0.4",
+ "version": "3.0.3",
"command": {
"version": {
"conventionalCommits": true,
diff --git a/package.json b/package.json
index 93c2e8318..227d6abc2 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "crawlee",
"private": true,
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"workspaces": [
"packages/*"
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json
index ccbc29ea9..de511cb64 100644
--- a/packages/basic-crawler/package.json
+++ b/packages/basic-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/basic",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -48,9 +48,9 @@
"@apify/log": "^2.0.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.0.0",
- "@crawlee/core": "^3.0.4",
- "@crawlee/types": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/core": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"got-scraping": "^3.2.9",
"ow": "^0.28.1"
}
diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json
index a25869189..88ffb86dd 100644
--- a/packages/browser-crawler/package.json
+++ b/packages/browser-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,11 +54,11 @@
},
"dependencies": {
"@apify/timeout": "^0.3.0",
- "@crawlee/basic": "^3.0.4",
- "@crawlee/browser-pool": "^3.0.4",
- "@crawlee/core": "^3.0.4",
- "@crawlee/types": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/basic": "^3.0.3",
+ "@crawlee/browser-pool": "^3.0.3",
+ "@crawlee/core": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"ow": "^0.28.1"
}
}
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index 28a821847..1fc4d21e9 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser-pool",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -38,7 +38,7 @@
"dependencies": {
"@apify/log": "^2.0.0",
"@apify/timeout": "^0.3.0",
- "@crawlee/types": "^3.0.4",
+ "@crawlee/types": "^3.0.3",
"fingerprint-generator": "^2.0.5",
"fingerprint-injector": "^2.0.4",
"lodash.merge": "^4.6.2",
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index e3432ab6c..a12cc5b3b 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cheerio",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -53,8 +53,8 @@
"access": "public"
},
"dependencies": {
- "@crawlee/http": "^3.0.4",
- "@crawlee/types": "^3.0.4",
+ "@crawlee/http": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
"cheerio": "1.0.0-rc.12",
"htmlparser2": "^8.0.1"
}
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 5057b3166..1bad8ea0e 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cli",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -51,7 +51,7 @@
"access": "public"
},
"dependencies": {
- "@crawlee/templates": "^3.0.4",
+ "@crawlee/templates": "^3.0.3",
"ansi-colors": "^4.1.3",
"fs-extra": "^10.1.0",
"inquirer": "^8.2.4",
diff --git a/packages/core/package.json b/packages/core/package.json
index 325dd684f..e058c94ea 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/core",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -58,9 +58,9 @@
"@apify/pseudo_url": "^2.0.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.0.0",
- "@crawlee/memory-storage": "^3.0.4",
- "@crawlee/types": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/memory-storage": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
"fs-extra": "^10.1.0",
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index de025727e..674c48412 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -1,6 +1,6 @@
{
"name": "crawlee",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -56,16 +56,16 @@
"access": "public"
},
"dependencies": {
- "@crawlee/cli": "^3.0.4",
- "@crawlee/core": "^3.0.4",
- "@crawlee/basic": "^3.0.4",
- "@crawlee/browser": "^3.0.4",
- "@crawlee/http": "^3.0.4",
- "@crawlee/cheerio": "^3.0.4",
- "@crawlee/jsdom": "^3.0.4",
- "@crawlee/playwright": "^3.0.4",
- "@crawlee/puppeteer": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/cli": "^3.0.3",
+ "@crawlee/core": "^3.0.3",
+ "@crawlee/basic": "^3.0.3",
+ "@crawlee/browser": "^3.0.3",
+ "@crawlee/http": "^3.0.3",
+ "@crawlee/cheerio": "^3.0.3",
+ "@crawlee/jsdom": "^3.0.3",
+ "@crawlee/playwright": "^3.0.3",
+ "@crawlee/puppeteer": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"import-local": "^3.1.0"
},
"peerDependencies": {
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index 220bad699..21cfe707e 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/http",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.0.0",
- "@crawlee/basic": "^3.0.4",
- "@crawlee/types": "^3.0.4",
+ "@crawlee/basic": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
"@types/content-type": "^1.1.5",
"content-type": "^1.0.4",
"got-scraping": "^3.2.9",
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index de91e11bb..469008ef6 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/jsdom",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,8 +54,8 @@
},
"dependencies": {
"@apify/utilities": "^2.1.4",
- "@crawlee/http": "^3.0.4",
- "@crawlee/types": "^3.0.4",
+ "@crawlee/http": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
"@types/jsdom": "^20.0.0",
"jsdom": "^20.0.0"
}
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index 97a99669f..fd1ddcf05 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/memory-storage",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "A simple in-memory storage implementation of the Apify API",
"engines": {
"node": ">= 16"
@@ -49,8 +49,8 @@
},
"dependencies": {
"@apify/log": "^2.0.0",
- "@crawlee/types": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/types": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
"fs-extra": "^10.1.0",
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 8aa9956ea..db5cd726e 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/playwright",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/log": "^2.0.0",
"@apify/datastructures": "^2.0.0",
- "@crawlee/browser": "^3.0.4",
- "@crawlee/browser-pool": "^3.0.4",
- "@crawlee/core": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/browser": "^3.0.3",
+ "@crawlee/browser-pool": "^3.0.3",
+ "@crawlee/core": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"cheerio": "1.0.0-rc.12",
"jquery": "^3.6.0",
"ow": "^0.28.1"
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index f9096cc02..15ec8abd8 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/puppeteer",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.0.0",
- "@crawlee/browser": "^3.0.4",
- "@crawlee/browser-pool": "^3.0.4",
- "@crawlee/types": "^3.0.4",
- "@crawlee/utils": "^3.0.4",
+ "@crawlee/browser": "^3.0.3",
+ "@crawlee/browser-pool": "^3.0.3",
+ "@crawlee/types": "^3.0.3",
+ "@crawlee/utils": "^3.0.3",
"cheerio": "1.0.0-rc.12",
"jquery": "^3.6.0",
"ow": "^0.28.1"
diff --git a/packages/templates/package.json b/packages/templates/package.json
index 8af50029d..eac62f527 100644
--- a/packages/templates/package.json
+++ b/packages/templates/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/templates",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "Templates for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/types/package.json b/packages/types/package.json
index 5fd313414..715d8d662 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/types",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "Shared types for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/utils/package.json b/packages/utils/package.json
index 076e5dd1c..5b4a93e27 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/utils",
- "version": "3.0.4",
+ "version": "3.0.3",
"description": "A set of shared utilities that can be used by crawlers",
"engines": {
"node": ">=16.0.0"
@@ -49,7 +49,7 @@
"dependencies": {
"@apify/log": "^2.0.0",
"@apify/ps-tree": "^1.1.4",
- "@crawlee/types": "^3.0.4",
+ "@crawlee/types": "^3.0.3",
"cheerio": "1.0.0-rc.12",
"got-scraping": "^3.2.9",
"tslib": "^2.4.0"
diff --git a/scripts/copy.ts b/scripts/copy.ts
index 19c499b1b..3f76246b5 100644
--- a/scripts/copy.ts
+++ b/scripts/copy.ts
@@ -31,13 +31,17 @@ function getRootVersion(): string {
}
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-dynamic-require,global-require
- rootVersion = require(resolve(root, './packages/core/package.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1');
+ rootVersion = require(resolve(root, './packages/crawlee/package.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1');
+
+ const parts = rootVersion.split('.');
+ parts[2] = `${+parts[2] + 1}`;
+ rootVersion = parts.join('.');
return rootVersion;
}
/**
- * Checks next dev version number based on the `@crawlee/core` package via `npm show`.
+ * Checks next dev version number based on the `crawlee` meta package via `npm show`.
* We always use this package, so we ensure the version is the same for each package in the monorepo.
*/
function getNextVersion() {