Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(release): v3.10.5

Apify Release Bot07ada112dee2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1d1ca5e64..cd7d3dba3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,20 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* allow creating new adaptive crawler instance without any parameters ([9b7f595](https://github.com/apify/crawlee/commit/9b7f595a2d70cab5c50e188581b21b0ef7e51780))
+* declare missing peer dependencies in `@crawlee/browser` package ([#2532](https://github.com/apify/crawlee/issues/2532)) ([3357c7f](https://github.com/apify/crawlee/commit/3357c7fc5ab071b12f72097c190dbee9990e3751))
+* fix detection of HTTP site when using the `useState` in adaptive crawler ([#2530](https://github.com/apify/crawlee/issues/2530)) ([7e195c1](https://github.com/apify/crawlee/commit/7e195c17cf1d9beae7f6f068fe505f1334a3a5b3))
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
diff --git a/lerna.json b/lerna.json
index 5adaf697e..3532bdbd0 100644
--- a/lerna.json
+++ b/lerna.json
@@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
- "version": "3.10.4",
+ "version": "3.10.5",
"command": {
"version": {
"conventionalCommits": true,
diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.md
index 1df64b4a3..e8553a5b3 100644
--- a/packages/basic-crawler/CHANGELOG.md
+++ b/packages/basic-crawler/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json
index 2713bfb4e..ef63c0b1e 100644
--- a/packages/basic-crawler/package.json
+++ b/packages/basic-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/basic",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -48,9 +48,9 @@
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/core": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/core": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"csv-stringify": "^6.2.0",
"fs-extra": "^11.0.0",
"got-scraping": "^4.0.0",
diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.md
index b1f0e506b..1b0a4d1ec 100644
--- a/packages/browser-crawler/CHANGELOG.md
+++ b/packages/browser-crawler/CHANGELOG.md
@@ -3,6 +3,18 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* declare missing peer dependencies in `@crawlee/browser` package ([#2532](https://github.com/apify/crawlee/issues/2532)) ([3357c7f](https://github.com/apify/crawlee/commit/3357c7fc5ab071b12f72097c190dbee9990e3751))
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/browser
diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json
index de24abd7e..2bae555b2 100644
--- a/packages/browser-crawler/package.json
+++ b/packages/browser-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,10 +54,10 @@
},
"dependencies": {
"@apify/timeout": "^0.3.0",
- "@crawlee/basic": "3.10.4",
- "@crawlee/browser-pool": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/basic": "^3.10.5",
+ "@crawlee/browser-pool": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"ow": "^0.28.1",
"tslib": "^2.4.0",
"type-fest": "^4.0.0"
diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.md
index 071a9e13b..eceaad9c8 100644
--- a/packages/browser-pool/CHANGELOG.md
+++ b/packages/browser-pool/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/browser-pool
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/browser-pool
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index a0d11da50..5a9164964 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser-pool",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -38,8 +38,8 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
- "@crawlee/core": "3.10.4",
- "@crawlee/types": "3.10.4",
+ "@crawlee/core": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
"fingerprint-generator": "^2.0.6",
"fingerprint-injector": "^2.0.5",
"lodash.merge": "^4.6.2",
diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.md
index 7f052b8fd..b73ebf5de 100644
--- a/packages/cheerio-crawler/CHANGELOG.md
+++ b/packages/cheerio-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/cheerio
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/cheerio
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index f18fefcd0..9a894edfa 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cheerio",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -53,9 +53,9 @@
"access": "public"
},
"dependencies": {
- "@crawlee/http": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/http": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"cheerio": "^1.0.0-rc.12",
"htmlparser2": "^9.0.0",
"tslib": "^2.4.0"
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
index f1e8821a9..7c1b54fcf 100644
--- a/packages/cli/CHANGELOG.md
+++ b/packages/cli/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/cli
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/cli
diff --git a/packages/cli/package.json b/packages/cli/package.json
index a00c9013f..ba44facf8 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cli",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -51,7 +51,7 @@
"access": "public"
},
"dependencies": {
- "@crawlee/templates": "3.10.4",
+ "@crawlee/templates": "^3.10.5",
"ansi-colors": "^4.1.3",
"fs-extra": "^11.0.0",
"inquirer": "^8.2.4",
diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md
index efafd9f39..967235398 100644
--- a/packages/core/CHANGELOG.md
+++ b/packages/core/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
diff --git a/packages/core/package.json b/packages/core/package.json
index 697fda7bc..935accb52 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/core",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -59,9 +59,9 @@
"@apify/pseudo_url": "^2.0.30",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/memory-storage": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/memory-storage": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"@sapphire/async-queue": "^1.5.1",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.2.2",
diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.md
index 389e71d70..d8303105b 100644
--- a/packages/crawlee/CHANGELOG.md
+++ b/packages/crawlee/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package crawlee
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package crawlee
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index 42efc3858..f72b22ecf 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -1,6 +1,6 @@
{
"name": "crawlee",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,18 +54,18 @@
"access": "public"
},
"dependencies": {
- "@crawlee/basic": "3.10.4",
- "@crawlee/browser": "3.10.4",
- "@crawlee/browser-pool": "3.10.4",
- "@crawlee/cheerio": "3.10.4",
- "@crawlee/cli": "3.10.4",
- "@crawlee/core": "3.10.4",
- "@crawlee/http": "3.10.4",
- "@crawlee/jsdom": "3.10.4",
- "@crawlee/linkedom": "3.10.4",
- "@crawlee/playwright": "3.10.4",
- "@crawlee/puppeteer": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/basic": "^3.10.5",
+ "@crawlee/browser": "^3.10.5",
+ "@crawlee/browser-pool": "^3.10.5",
+ "@crawlee/cheerio": "^3.10.5",
+ "@crawlee/cli": "^3.10.5",
+ "@crawlee/core": "^3.10.5",
+ "@crawlee/http": "^3.10.5",
+ "@crawlee/jsdom": "^3.10.5",
+ "@crawlee/linkedom": "^3.10.5",
+ "@crawlee/playwright": "^3.10.5",
+ "@crawlee/puppeteer": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"import-local": "^3.1.0",
"tslib": "^2.4.0"
},
diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.md
index 6d28967a8..8468ef7d7 100644
--- a/packages/http-crawler/CHANGELOG.md
+++ b/packages/http-crawler/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index da915e08c..75c457100 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/http",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,9 +55,9 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/basic": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/basic": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"@types/content-type": "^1.1.5",
"cheerio": "^1.0.0-rc.12",
"content-type": "^1.0.4",
diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.md
index e970a1621..0157d92ac 100644
--- a/packages/jsdom-crawler/CHANGELOG.md
+++ b/packages/jsdom-crawler/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* declare missing peer dependencies in `@crawlee/browser` package ([#2532](https://github.com/apify/crawlee/issues/2532)) ([3357c7f](https://github.com/apify/crawlee/commit/3357c7fc5ab071b12f72097c190dbee9990e3751))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/jsdom
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index 2bab4b5a0..98e568df2 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/jsdom",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,9 +55,9 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/http": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/http": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"@types/jsdom": "^21.0.0",
"cheerio": "^1.0.0-rc.12",
"jsdom": "^24.0.0",
diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.md
index 18587978f..406455e29 100644
--- a/packages/linkedom-crawler/CHANGELOG.md
+++ b/packages/linkedom-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/linkedom
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/linkedom
diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.json
index 6bc32e64a..b2f34a9d4 100644
--- a/packages/linkedom-crawler/package.json
+++ b/packages/linkedom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/linkedom",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/http": "3.10.4",
- "@crawlee/types": "3.10.4",
+ "@crawlee/http": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
"linkedom": "^0.18.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.md
index 78d64f926..affa9a424 100644
--- a/packages/memory-storage/CHANGELOG.md
+++ b/packages/memory-storage/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/memory-storage
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/memory-storage
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index 6ab735f68..d02f0437f 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/memory-storage",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "A simple in-memory storage implementation of the Apify API",
"engines": {
"node": ">= 16"
@@ -49,7 +49,7 @@
},
"dependencies": {
"@apify/log": "^2.4.0",
- "@crawlee/types": "3.10.4",
+ "@crawlee/types": "^3.10.5",
"@sapphire/async-queue": "^1.5.0",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.md
index db83a5dfd..516370482 100644
--- a/packages/playwright-crawler/CHANGELOG.md
+++ b/packages/playwright-crawler/CHANGELOG.md
@@ -3,6 +3,19 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+
+### Bug Fixes
+
+* allow creating new adaptive crawler instance without any parameters ([9b7f595](https://github.com/apify/crawlee/commit/9b7f595a2d70cab5c50e188581b21b0ef7e51780))
+* fix detection of HTTP site when using the `useState` in adaptive crawler ([#2530](https://github.com/apify/crawlee/issues/2530)) ([7e195c1](https://github.com/apify/crawlee/commit/7e195c17cf1d9beae7f6f068fe505f1334a3a5b3))
+* mark `context.request.loadedUrl` and `id` as required inside the request handler ([#2531](https://github.com/apify/crawlee/issues/2531)) ([2b54660](https://github.com/apify/crawlee/commit/2b546600691d84852a2f9ef42f273cecf818d66d))
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 29e2183f4..71dc7c5ac 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/playwright",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -56,11 +56,11 @@
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.1",
- "@crawlee/browser": "3.10.4",
- "@crawlee/browser-pool": "3.10.4",
- "@crawlee/core": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/browser": "^3.10.5",
+ "@crawlee/browser-pool": "^3.10.5",
+ "@crawlee/core": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"cheerio": "^1.0.0-rc.12",
"idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.md
index 9d94de6be..aa39f41a8 100644
--- a/packages/puppeteer-crawler/CHANGELOG.md
+++ b/packages/puppeteer-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/puppeteer
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/puppeteer
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index e97f6dce9..bd7a38cba 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/puppeteer",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
- "@crawlee/browser": "3.10.4",
- "@crawlee/browser-pool": "3.10.4",
- "@crawlee/types": "3.10.4",
- "@crawlee/utils": "3.10.4",
+ "@crawlee/browser": "^3.10.5",
+ "@crawlee/browser-pool": "^3.10.5",
+ "@crawlee/types": "^3.10.5",
+ "@crawlee/utils": "^3.10.5",
"cheerio": "^1.0.0-rc.12",
"devtools-protocol": "*",
"idcac-playwright": "^0.1.2",
diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.md
index f75da4e5f..00040283c 100644
--- a/packages/templates/CHANGELOG.md
+++ b/packages/templates/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/templates
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/templates
diff --git a/packages/templates/package.json b/packages/templates/package.json
index 655121372..5999d6b6a 100644
--- a/packages/templates/package.json
+++ b/packages/templates/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/templates",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "Templates for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
index 45db32366..54e3d71dc 100644
--- a/packages/types/CHANGELOG.md
+++ b/packages/types/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/types
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/types
diff --git a/packages/types/package.json b/packages/types/package.json
index 90c5b0513..27b2ad433 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/types",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "Shared types for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md
index cb8965c12..a4e6a6488 100644
--- a/packages/utils/CHANGELOG.md
+++ b/packages/utils/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.5](https://github.com/apify/crawlee/compare/v3.10.4...v3.10.5) (2024-06-12)
+
+**Note:** Version bump only for package @crawlee/utils
+
+
+
+
+
## [3.10.4](https://github.com/apify/crawlee/compare/v3.10.3...v3.10.4) (2024-06-11)
**Note:** Version bump only for package @crawlee/utils
diff --git a/packages/utils/package.json b/packages/utils/package.json
index 86e221996..43632c058 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/utils",
- "version": "3.10.4",
+ "version": "3.10.5",
"description": "A set of shared utilities that can be used by crawlers",
"engines": {
"node": ">=16.0.0"
@@ -49,7 +49,7 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/ps-tree": "^1.2.0",
- "@crawlee/types": "3.10.4",
+ "@crawlee/types": "^3.10.5",
"@types/sax": "^1.2.7",
"cheerio": "^1.0.0-rc.12",
"file-type": "^19.0.0",
diff --git a/yarn.lock b/yarn.lock
index 8e360f06f..3ec69cecf 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -486,16 +486,16 @@ __metadata:
languageName: node
linkType: hard
-"@crawlee/basic@npm:3.10.4, @crawlee/basic@workspace:packages/basic-crawler":
+"@crawlee/basic@npm:^3.10.5, @crawlee/basic@workspace:packages/basic-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/basic@workspace:packages/basic-crawler"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/core": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/core": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
csv-stringify: "npm:^6.2.0"
fs-extra: "npm:^11.0.0"
got-scraping: "npm:^4.0.0"
@@ -506,14 +506,14 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser-pool@npm:3.10.4, @crawlee/browser-pool@workspace:packages/browser-pool":
+"@crawlee/browser-pool@npm:^3.10.5, @crawlee/browser-pool@workspace:packages/browser-pool":
version: 0.0.0-use.local
resolution: "@crawlee/browser-pool@workspace:packages/browser-pool"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.0"
- "@crawlee/core": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
+ "@crawlee/core": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
fingerprint-generator: "npm:^2.0.6"
fingerprint-injector: "npm:^2.0.5"
lodash.merge: "npm:^4.6.2"
@@ -535,15 +535,15 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser@npm:3.10.4, @crawlee/browser@workspace:packages/browser-crawler":
+"@crawlee/browser@npm:^3.10.5, @crawlee/browser@workspace:packages/browser-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/browser@workspace:packages/browser-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
- "@crawlee/basic": "npm:3.10.4"
- "@crawlee/browser-pool": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/basic": "npm:^3.10.5"
+ "@crawlee/browser-pool": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
ow: "npm:^0.28.1"
tslib: "npm:^2.4.0"
type-fest: "npm:^4.0.0"
@@ -558,24 +558,24 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/cheerio@npm:3.10.4, @crawlee/cheerio@workspace:packages/cheerio-crawler":
+"@crawlee/cheerio@npm:^3.10.5, @crawlee/cheerio@workspace:packages/cheerio-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"
dependencies:
- "@crawlee/http": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/http": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
cheerio: "npm:^1.0.0-rc.12"
htmlparser2: "npm:^9.0.0"
tslib: "npm:^2.4.0"
languageName: unknown
linkType: soft
-"@crawlee/cli@npm:3.10.4, @crawlee/cli@workspace:packages/cli":
+"@crawlee/cli@npm:^3.10.5, @crawlee/cli@workspace:packages/cli":
version: 0.0.0-use.local
resolution: "@crawlee/cli@workspace:packages/cli"
dependencies:
- "@crawlee/templates": "npm:3.10.4"
+ "@crawlee/templates": "npm:^3.10.5"
ansi-colors: "npm:^4.1.3"
fs-extra: "npm:^11.0.0"
inquirer: "npm:^8.2.4"
@@ -587,7 +587,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/core@npm:3.10.4, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
+"@crawlee/core@npm:^3.10.5, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
version: 0.0.0-use.local
resolution: "@crawlee/core@workspace:packages/core"
dependencies:
@@ -597,9 +597,9 @@ __metadata:
"@apify/pseudo_url": "npm:^2.0.30"
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/memory-storage": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/memory-storage": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
"@sapphire/async-queue": "npm:^1.5.1"
"@types/tough-cookie": "npm:^4.0.2"
"@vladfrangu/async_event_emitter": "npm:^2.2.2"
@@ -618,15 +618,15 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/http@npm:3.10.4, @crawlee/http@workspace:packages/http-crawler":
+"@crawlee/http@npm:^3.10.5, @crawlee/http@workspace:packages/http-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/http@workspace:packages/http-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/basic": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/basic": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
"@types/content-type": "npm:^1.1.5"
cheerio: "npm:^1.0.0-rc.12"
content-type: "npm:^1.0.4"
@@ -639,15 +639,15 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/jsdom@npm:3.10.4, @crawlee/jsdom@workspace:packages/jsdom-crawler":
+"@crawlee/jsdom@npm:^3.10.5, @crawlee/jsdom@workspace:packages/jsdom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/http": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/http": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
"@types/jsdom": "npm:^21.0.0"
cheerio: "npm:^1.0.0-rc.12"
jsdom: "npm:^24.0.0"
@@ -656,26 +656,26 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/linkedom@npm:3.10.4, @crawlee/linkedom@workspace:packages/linkedom-crawler":
+"@crawlee/linkedom@npm:^3.10.5, @crawlee/linkedom@workspace:packages/linkedom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/http": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
+ "@crawlee/http": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
linkedom: "npm:^0.18.0"
ow: "npm:^0.28.2"
tslib: "npm:^2.4.0"
languageName: unknown
linkType: soft
-"@crawlee/memory-storage@npm:3.10.4, @crawlee/memory-storage@workspace:packages/memory-storage":
+"@crawlee/memory-storage@npm:^3.10.5, @crawlee/memory-storage@workspace:packages/memory-storage":
version: 0.0.0-use.local
resolution: "@crawlee/memory-storage@workspace:packages/memory-storage"
dependencies:
"@apify/log": "npm:^2.4.0"
- "@crawlee/types": "npm:3.10.4"
+ "@crawlee/types": "npm:^3.10.5"
"@sapphire/async-queue": "npm:^1.5.0"
"@sapphire/shapeshift": "npm:^3.0.0"
content-type: "npm:^1.0.4"
@@ -687,18 +687,18 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/playwright@npm:3.10.4, @crawlee/playwright@workspace:packages/playwright-crawler":
+"@crawlee/playwright@npm:^3.10.5, @crawlee/playwright@workspace:packages/playwright-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/playwright@workspace:packages/playwright-crawler"
dependencies:
"@apify/datastructures": "npm:^2.0.0"
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.1"
- "@crawlee/browser": "npm:3.10.4"
- "@crawlee/browser-pool": "npm:3.10.4"
- "@crawlee/core": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/browser": "npm:^3.10.5"
+ "@crawlee/browser-pool": "npm:^3.10.5"
+ "@crawlee/core": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
cheerio: "npm:^1.0.0-rc.12"
idcac-playwright: "npm:^0.1.2"
jquery: "npm:^3.6.0"
@@ -716,16 +716,16 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/puppeteer@npm:3.10.4, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
+"@crawlee/puppeteer@npm:^3.10.5, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"
dependencies:
"@apify/datastructures": "npm:^2.0.0"
"@apify/log": "npm:^2.4.0"
- "@crawlee/browser": "npm:3.10.4"
- "@crawlee/browser-pool": "npm:3.10.4"
- "@crawlee/types": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/browser": "npm:^3.10.5"
+ "@crawlee/browser-pool": "npm:^3.10.5"
+ "@crawlee/types": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
cheerio: "npm:^1.0.0-rc.12"
devtools-protocol: "npm:*"
idcac-playwright: "npm:^0.1.2"
@@ -806,7 +806,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/templates@npm:3.10.4, @crawlee/templates@workspace:packages/templates":
+"@crawlee/templates@npm:^3.10.5, @crawlee/templates@workspace:packages/templates":
version: 0.0.0-use.local
resolution: "@crawlee/templates@workspace:packages/templates"
dependencies:
@@ -818,7 +818,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/types@npm:3.10.4, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":
+"@crawlee/types@npm:^3.10.5, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":
version: 0.0.0-use.local
resolution: "@crawlee/types@workspace:packages/types"
dependencies:
@@ -826,13 +826,13 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/utils@npm:3.10.4, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":
+"@crawlee/utils@npm:^3.10.5, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":
version: 0.0.0-use.local
resolution: "@crawlee/utils@workspace:packages/utils"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/ps-tree": "npm:^1.2.0"
- "@crawlee/types": "npm:3.10.4"
+ "@crawlee/types": "npm:^3.10.5"
"@types/sax": "npm:^1.2.7"
"@types/whatwg-mimetype": "npm:^3.0.2"
cheerio: "npm:^1.0.0-rc.12"
@@ -4335,18 +4335,18 @@ __metadata:
version: 0.0.0-use.local
resolution: "crawlee@workspace:packages/crawlee"
dependencies:
- "@crawlee/basic": "npm:3.10.4"
- "@crawlee/browser": "npm:3.10.4"
- "@crawlee/browser-pool": "npm:3.10.4"
- "@crawlee/cheerio": "npm:3.10.4"
- "@crawlee/cli": "npm:3.10.4"
- "@crawlee/core": "npm:3.10.4"
- "@crawlee/http": "npm:3.10.4"
- "@crawlee/jsdom": "npm:3.10.4"
- "@crawlee/linkedom": "npm:3.10.4"
- "@crawlee/playwright": "npm:3.10.4"
- "@crawlee/puppeteer": "npm:3.10.4"
- "@crawlee/utils": "npm:3.10.4"
+ "@crawlee/basic": "npm:^3.10.5"
+ "@crawlee/browser": "npm:^3.10.5"
+ "@crawlee/browser-pool": "npm:^3.10.5"
+ "@crawlee/cheerio": "npm:^3.10.5"
+ "@crawlee/cli": "npm:^3.10.5"
+ "@crawlee/core": "npm:^3.10.5"
+ "@crawlee/http": "npm:^3.10.5"
+ "@crawlee/jsdom": "npm:^3.10.5"
+ "@crawlee/linkedom": "npm:^3.10.5"
+ "@crawlee/playwright": "npm:^3.10.5"
+ "@crawlee/puppeteer": "npm:^3.10.5"
+ "@crawlee/utils": "npm:^3.10.5"
import-local: "npm:^3.1.0"
tslib: "npm:^2.4.0"
peerDependencies: