Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(release): v3.4.2

Apify Release Botef4cf657d134
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba0589adb..395df35cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,23 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Bug Fixes
+
+* **basic-crawler:** limit `internalTimeoutMillis` in addition to `requestHandlerTimeoutMillis` ([#1981](https://github.com/apify/crawlee/issues/1981)) ([8122622](https://github.com/apify/crawlee/commit/8122622c3054a0e0e0c1869ba462276cbead8090)), closes [#1766](https://github.com/apify/crawlee/issues/1766)
+
+
+### Features
+
+* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)
+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
diff --git a/lerna.json b/lerna.json
index fbffc6aac..be7d39a90 100644
--- a/lerna.json
+++ b/lerna.json
@@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
- "version": "3.4.1",
+ "version": "3.4.2",
"command": {
"version": {
"conventionalCommits": true,
diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.md
index 91c8a25d7..b8dda4ffa 100644
--- a/packages/basic-crawler/CHANGELOG.md
+++ b/packages/basic-crawler/CHANGELOG.md
@@ -3,6 +3,23 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Bug Fixes
+
+* **basic-crawler:** limit `internalTimeoutMillis` in addition to `requestHandlerTimeoutMillis` ([#1981](https://github.com/apify/crawlee/issues/1981)) ([8122622](https://github.com/apify/crawlee/commit/8122622c3054a0e0e0c1869ba462276cbead8090)), closes [#1766](https://github.com/apify/crawlee/issues/1766)
+
+
+### Features
+
+* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)
+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/basic
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json
index b316b318e..5ea5ba1d1 100644
--- a/packages/basic-crawler/package.json
+++ b/packages/basic-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/basic",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -48,9 +48,9 @@
"@apify/log": "^2.0.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.0.0",
- "@crawlee/core": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/core": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"got-scraping": "^3.2.9",
"ow": "^0.28.1",
"tslib": "^2.4.0",
diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.md
index 415fbbf47..6dd6db148 100644
--- a/packages/browser-crawler/CHANGELOG.md
+++ b/packages/browser-crawler/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Features
+
+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/browser
diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json
index 6762ca86d..1d30a3165 100644
--- a/packages/browser-crawler/package.json
+++ b/packages/browser-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,10 +54,10 @@
},
"dependencies": {
"@apify/timeout": "^0.3.0",
- "@crawlee/basic": "^3.4.1",
- "@crawlee/browser-pool": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/basic": "^3.4.2",
+ "@crawlee/browser-pool": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"ow": "^0.28.1",
"tslib": "^2.4.0"
}
diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.md
index b9eb4222a..4b6cfb65d 100644
--- a/packages/browser-pool/CHANGELOG.md
+++ b/packages/browser-pool/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/browser-pool
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/browser-pool
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index e774a9a01..b7f8bdc95 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser-pool",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -38,7 +38,7 @@
"dependencies": {
"@apify/log": "^2.0.0",
"@apify/timeout": "^0.3.0",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/types": "^3.4.2",
"fingerprint-generator": "^2.0.6",
"fingerprint-injector": "^2.0.5",
"lodash.merge": "^4.6.2",
diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.md
index e714cc28f..a9bd95cb1 100644
--- a/packages/cheerio-crawler/CHANGELOG.md
+++ b/packages/cheerio-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/cheerio
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/cheerio
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index c233447af..75f3a2e20 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cheerio",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -53,8 +53,8 @@
"access": "public"
},
"dependencies": {
- "@crawlee/http": "^3.4.1",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/http": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
"cheerio": "^1.0.0-rc.12",
"htmlparser2": "^9.0.0",
"tslib": "^2.4.0"
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
index fc24e15bf..74facbb9a 100644
--- a/packages/cli/CHANGELOG.md
+++ b/packages/cli/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/cli
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/cli
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 3e3958369..d91b5986d 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cli",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -51,7 +51,7 @@
"access": "public"
},
"dependencies": {
- "@crawlee/templates": "^3.4.1",
+ "@crawlee/templates": "^3.4.2",
"ansi-colors": "^4.1.3",
"fs-extra": "^11.0.0",
"inquirer": "^8.2.4",
diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md
index 240e1f03a..2303ded86 100644
--- a/packages/core/CHANGELOG.md
+++ b/packages/core/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Features
+
+* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
diff --git a/packages/core/package.json b/packages/core/package.json
index 9b656dfcb..ef30d7f8a 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/core",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -59,9 +59,9 @@
"@apify/pseudo_url": "^2.0.14",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.3.3",
- "@crawlee/memory-storage": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/memory-storage": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"@sapphire/async-queue": "^1.5.0",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.md
index 09cd0ab4e..39cb8f60e 100644
--- a/packages/crawlee/CHANGELOG.md
+++ b/packages/crawlee/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package crawlee
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package crawlee
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index 0aa8bb29e..5bb31a289 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -1,6 +1,6 @@
{
"name": "crawlee",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,17 +54,17 @@
"access": "public"
},
"dependencies": {
- "@crawlee/basic": "^3.4.1",
- "@crawlee/browser": "^3.4.1",
- "@crawlee/cheerio": "^3.4.1",
- "@crawlee/cli": "^3.4.1",
- "@crawlee/core": "^3.4.1",
- "@crawlee/http": "^3.4.1",
- "@crawlee/jsdom": "^3.4.1",
- "@crawlee/linkedom": "^3.4.1",
- "@crawlee/playwright": "^3.4.1",
- "@crawlee/puppeteer": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/basic": "^3.4.2",
+ "@crawlee/browser": "^3.4.2",
+ "@crawlee/cheerio": "^3.4.2",
+ "@crawlee/cli": "^3.4.2",
+ "@crawlee/core": "^3.4.2",
+ "@crawlee/http": "^3.4.2",
+ "@crawlee/jsdom": "^3.4.2",
+ "@crawlee/linkedom": "^3.4.2",
+ "@crawlee/playwright": "^3.4.2",
+ "@crawlee/puppeteer": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"import-local": "^3.1.0",
"tslib": "^2.4.0"
},
diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.md
index c675d497d..c61d8c054 100644
--- a/packages/http-crawler/CHANGELOG.md
+++ b/packages/http-crawler/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Features
+
+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index 5beb41dbc..765152f61 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/http",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,9 +55,9 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.0.0",
- "@crawlee/basic": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/basic": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"@types/content-type": "^1.1.5",
"cheerio": "^1.0.0-rc.12",
"content-type": "^1.0.4",
diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.md
index 81fe9ade3..fc94018bf 100644
--- a/packages/jsdom-crawler/CHANGELOG.md
+++ b/packages/jsdom-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/jsdom
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index 1d386ff1e..fd33346a3 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/jsdom",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.1.4",
- "@crawlee/http": "^3.4.1",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/http": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
"@types/jsdom": "^21.0.0",
"cheerio": "^1.0.0-rc.12",
"jsdom": "^22.0.0",
diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.md
index 3f7172e71..6184c64c9 100644
--- a/packages/linkedom-crawler/CHANGELOG.md
+++ b/packages/linkedom-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/linkedom
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.json
index 48434ce3d..bda01c3a7 100644
--- a/packages/linkedom-crawler/package.json
+++ b/packages/linkedom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/linkedom",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.1.4",
- "@crawlee/http": "^3.4.1",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/http": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
"linkedom": "^0.15.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.md
index caf5ae671..30462526b 100644
--- a/packages/memory-storage/CHANGELOG.md
+++ b/packages/memory-storage/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/memory-storage
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/memory-storage
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index 4a704c2db..af0464f84 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/memory-storage",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "A simple in-memory storage implementation of the Apify API",
"engines": {
"node": ">= 16"
@@ -49,7 +49,7 @@
},
"dependencies": {
"@apify/log": "^2.0.0",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/types": "^3.4.2",
"@sapphire/async-queue": "^1.5.0",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.md
index 9d20fbe4e..016843e2c 100644
--- a/packages/playwright-crawler/CHANGELOG.md
+++ b/packages/playwright-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/playwright
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/playwright
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index da40b7f18..2b687a9e9 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/playwright",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.0.0",
- "@crawlee/browser": "^3.4.1",
- "@crawlee/browser-pool": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/browser": "^3.4.2",
+ "@crawlee/browser-pool": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"cheerio": "^1.0.0-rc.12",
"jquery": "^3.6.0",
"ow": "^0.28.1",
diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.md
index 53ae62178..d8a978f08 100644
--- a/packages/puppeteer-crawler/CHANGELOG.md
+++ b/packages/puppeteer-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/puppeteer
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/puppeteer
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 2ac7efda9..506640e67 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/puppeteer",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.0.0",
- "@crawlee/browser": "^3.4.1",
- "@crawlee/browser-pool": "^3.4.1",
- "@crawlee/types": "^3.4.1",
- "@crawlee/utils": "^3.4.1",
+ "@crawlee/browser": "^3.4.2",
+ "@crawlee/browser-pool": "^3.4.2",
+ "@crawlee/types": "^3.4.2",
+ "@crawlee/utils": "^3.4.2",
"cheerio": "^1.0.0-rc.12",
"devtools-protocol": "*",
"jquery": "^3.6.0",
diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.md
index 8b0e8425c..1f27d461e 100644
--- a/packages/templates/CHANGELOG.md
+++ b/packages/templates/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/templates
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/templates
diff --git a/packages/templates/package.json b/packages/templates/package.json
index 66fd640fc..b8430993d 100644
--- a/packages/templates/package.json
+++ b/packages/templates/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/templates",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "Templates for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
index e6107f2e0..2526afd6c 100644
--- a/packages/types/CHANGELOG.md
+++ b/packages/types/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+**Note:** Version bump only for package @crawlee/types
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/types
diff --git a/packages/types/package.json b/packages/types/package.json
index 2eebc700d..5aae7c92c 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/types",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "Shared types for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md
index c5be272e0..690b52a2f 100644
--- a/packages/utils/CHANGELOG.md
+++ b/packages/utils/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)
+
+
+### Features
+
+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))
+
+
+
+
+
## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)
**Note:** Version bump only for package @crawlee/utils
diff --git a/packages/utils/package.json b/packages/utils/package.json
index 5f5910494..3feff10dd 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/utils",
- "version": "3.4.1",
+ "version": "3.4.2",
"description": "A set of shared utilities that can be used by crawlers",
"engines": {
"node": ">=16.0.0"
@@ -49,7 +49,7 @@
"dependencies": {
"@apify/log": "^2.0.0",
"@apify/ps-tree": "^1.1.4",
- "@crawlee/types": "^3.4.1",
+ "@crawlee/types": "^3.4.2",
"cheerio": "^1.0.0-rc.12",
"got-scraping": "^3.2.9",
"ow": "^0.28.1",
diff --git a/yarn.lock b/yarn.lock
index 71e6e5007..1797c47f5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -738,16 +738,16 @@ __metadata:
languageName: node
linkType: hard
-"@crawlee/basic@^3.4.1, @crawlee/basic@workspace:packages/basic-crawler":
+"@crawlee/basic@^3.4.2, @crawlee/basic@workspace:packages/basic-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/basic@workspace:packages/basic-crawler"
dependencies:
"@apify/log": ^2.0.0
"@apify/timeout": ^0.3.0
"@apify/utilities": ^2.0.0
- "@crawlee/core": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/core": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
got-scraping: ^3.2.9
ow: ^0.28.1
tslib: ^2.4.0
@@ -755,13 +755,13 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser-pool@^3.4.1, @crawlee/browser-pool@workspace:packages/browser-pool":
+"@crawlee/browser-pool@^3.4.2, @crawlee/browser-pool@workspace:packages/browser-pool":
version: 0.0.0-use.local
resolution: "@crawlee/browser-pool@workspace:packages/browser-pool"
dependencies:
"@apify/log": ^2.0.0
"@apify/timeout": ^0.3.0
- "@crawlee/types": ^3.4.1
+ "@crawlee/types": ^3.4.2
fingerprint-generator: ^2.0.6
fingerprint-injector: ^2.0.5
lodash.merge: ^4.6.2
@@ -783,37 +783,37 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser@^3.4.1, @crawlee/browser@workspace:packages/browser-crawler":
+"@crawlee/browser@^3.4.2, @crawlee/browser@workspace:packages/browser-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/browser@workspace:packages/browser-crawler"
dependencies:
"@apify/timeout": ^0.3.0
- "@crawlee/basic": ^3.4.1
- "@crawlee/browser-pool": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/basic": ^3.4.2
+ "@crawlee/browser-pool": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
ow: ^0.28.1
tslib: ^2.4.0
languageName: unknown
linkType: soft
-"@crawlee/cheerio@^3.4.1, @crawlee/cheerio@workspace:packages/cheerio-crawler":
+"@crawlee/cheerio@^3.4.2, @crawlee/cheerio@workspace:packages/cheerio-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"
dependencies:
- "@crawlee/http": ^3.4.1
- "@crawlee/types": ^3.4.1
+ "@crawlee/http": ^3.4.2
+ "@crawlee/types": ^3.4.2
cheerio: ^1.0.0-rc.12
htmlparser2: ^9.0.0
tslib: ^2.4.0
languageName: unknown
linkType: soft
-"@crawlee/cli@^3.4.1, @crawlee/cli@workspace:packages/cli":
+"@crawlee/cli@^3.4.2, @crawlee/cli@workspace:packages/cli":
version: 0.0.0-use.local
resolution: "@crawlee/cli@workspace:packages/cli"
dependencies:
- "@crawlee/templates": ^3.4.1
+ "@crawlee/templates": ^3.4.2
ansi-colors: ^4.1.3
fs-extra: ^11.0.0
inquirer: ^8.2.4
@@ -825,7 +825,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/core@^3.3.0, @crawlee/core@^3.4.1, @crawlee/core@workspace:packages/core":
+"@crawlee/core@^3.3.0, @crawlee/core@^3.4.2, @crawlee/core@workspace:packages/core":
version: 0.0.0-use.local
resolution: "@crawlee/core@workspace:packages/core"
dependencies:
@@ -835,9 +835,9 @@ __metadata:
"@apify/pseudo_url": ^2.0.14
"@apify/timeout": ^0.3.0
"@apify/utilities": ^2.3.3
- "@crawlee/memory-storage": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/memory-storage": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
"@sapphire/async-queue": ^1.5.0
"@types/tough-cookie": ^4.0.2
"@vladfrangu/async_event_emitter": ^2.0.0
@@ -855,15 +855,15 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/http@^3.4.1, @crawlee/http@workspace:packages/http-crawler":
+"@crawlee/http@^3.4.2, @crawlee/http@workspace:packages/http-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/http@workspace:packages/http-crawler"
dependencies:
"@apify/timeout": ^0.3.0
"@apify/utilities": ^2.0.0
- "@crawlee/basic": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/basic": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
"@types/content-type": ^1.1.5
cheerio: ^1.0.0-rc.12
content-type: ^1.0.4
@@ -876,14 +876,14 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/jsdom@^3.4.1, @crawlee/jsdom@workspace:packages/jsdom-crawler":
+"@crawlee/jsdom@^3.4.2, @crawlee/jsdom@workspace:packages/jsdom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"
dependencies:
"@apify/timeout": ^0.3.0
"@apify/utilities": ^2.1.4
- "@crawlee/http": ^3.4.1
- "@crawlee/types": ^3.4.1
+ "@crawlee/http": ^3.4.2
+ "@crawlee/types": ^3.4.2
"@types/jsdom": ^21.0.0
cheerio: ^1.0.0-rc.12
jsdom: ^22.0.0
@@ -892,26 +892,26 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/linkedom@^3.4.1, @crawlee/linkedom@workspace:packages/linkedom-crawler":
+"@crawlee/linkedom@^3.4.2, @crawlee/linkedom@workspace:packages/linkedom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"
dependencies:
"@apify/timeout": ^0.3.0
"@apify/utilities": ^2.1.4
- "@crawlee/http": ^3.4.1
- "@crawlee/types": ^3.4.1
+ "@crawlee/http": ^3.4.2
+ "@crawlee/types": ^3.4.2
linkedom: ^0.15.0
ow: ^0.28.2
tslib: ^2.4.0
languageName: unknown
linkType: soft
-"@crawlee/memory-storage@^3.4.1, @crawlee/memory-storage@workspace:packages/memory-storage":
+"@crawlee/memory-storage@^3.4.2, @crawlee/memory-storage@workspace:packages/memory-storage":
version: 0.0.0-use.local
resolution: "@crawlee/memory-storage@workspace:packages/memory-storage"
dependencies:
"@apify/log": ^2.0.0
- "@crawlee/types": ^3.4.1
+ "@crawlee/types": ^3.4.2
"@sapphire/async-queue": ^1.5.0
"@sapphire/shapeshift": ^3.0.0
content-type: ^1.0.4
@@ -923,16 +923,16 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/playwright@^3.4.1, @crawlee/playwright@workspace:packages/playwright-crawler":
+"@crawlee/playwright@^3.4.2, @crawlee/playwright@workspace:packages/playwright-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/playwright@workspace:packages/playwright-crawler"
dependencies:
"@apify/datastructures": ^2.0.0
"@apify/log": ^2.0.0
- "@crawlee/browser": ^3.4.1
- "@crawlee/browser-pool": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/browser": ^3.4.2
+ "@crawlee/browser-pool": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
cheerio: ^1.0.0-rc.12
jquery: ^3.6.0
ow: ^0.28.1
@@ -945,16 +945,16 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/puppeteer@^3.4.1, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
+"@crawlee/puppeteer@^3.4.2, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"
dependencies:
"@apify/datastructures": ^2.0.0
"@apify/log": ^2.0.0
- "@crawlee/browser": ^3.4.1
- "@crawlee/browser-pool": ^3.4.1
- "@crawlee/types": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/browser": ^3.4.2
+ "@crawlee/browser-pool": ^3.4.2
+ "@crawlee/types": ^3.4.2
+ "@crawlee/utils": ^3.4.2
cheerio: ^1.0.0-rc.12
devtools-protocol: "*"
jquery: ^3.6.0
@@ -1024,7 +1024,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/templates@^3.4.1, @crawlee/templates@workspace:packages/templates":
+"@crawlee/templates@^3.4.2, @crawlee/templates@workspace:packages/templates":
version: 0.0.0-use.local
resolution: "@crawlee/templates@workspace:packages/templates"
dependencies:
@@ -1036,7 +1036,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/types@^3.3.0, @crawlee/types@^3.4.1, @crawlee/types@workspace:packages/types":
+"@crawlee/types@^3.3.0, @crawlee/types@^3.4.2, @crawlee/types@workspace:packages/types":
version: 0.0.0-use.local
resolution: "@crawlee/types@workspace:packages/types"
dependencies:
@@ -1044,13 +1044,13 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/utils@^3.3.0, @crawlee/utils@^3.4.1, @crawlee/utils@workspace:packages/utils":
+"@crawlee/utils@^3.3.0, @crawlee/utils@^3.4.2, @crawlee/utils@workspace:packages/utils":
version: 0.0.0-use.local
resolution: "@crawlee/utils@workspace:packages/utils"
dependencies:
"@apify/log": ^2.0.0
"@apify/ps-tree": ^1.1.4
- "@crawlee/types": ^3.4.1
+ "@crawlee/types": ^3.4.2
cheerio: ^1.0.0-rc.12
got-scraping: ^3.2.9
ow: ^0.28.1
@@ -4232,17 +4232,17 @@ __metadata:
version: 0.0.0-use.local
resolution: "crawlee@workspace:packages/crawlee"
dependencies:
- "@crawlee/basic": ^3.4.1
- "@crawlee/browser": ^3.4.1
- "@crawlee/cheerio": ^3.4.1
- "@crawlee/cli": ^3.4.1
- "@crawlee/core": ^3.4.1
- "@crawlee/http": ^3.4.1
- "@crawlee/jsdom": ^3.4.1
- "@crawlee/linkedom": ^3.4.1
- "@crawlee/playwright": ^3.4.1
- "@crawlee/puppeteer": ^3.4.1
- "@crawlee/utils": ^3.4.1
+ "@crawlee/basic": ^3.4.2
+ "@crawlee/browser": ^3.4.2
+ "@crawlee/cheerio": ^3.4.2
+ "@crawlee/cli": ^3.4.2
+ "@crawlee/core": ^3.4.2
+ "@crawlee/http": ^3.4.2
+ "@crawlee/jsdom": ^3.4.2
+ "@crawlee/linkedom": ^3.4.2
+ "@crawlee/playwright": ^3.4.2
+ "@crawlee/puppeteer": ^3.4.2
+ "@crawlee/utils": ^3.4.2
import-local: ^3.1.0
tslib: ^2.4.0
peerDependencies: