Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): v3.4.2
Apify Release Botef4cf657d134
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ba0589adb..395df35cb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,23 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Bug Fixes++* **basic-crawler:** limit `internalTimeoutMillis` in addition to `requestHandlerTimeoutMillis` ([#1981](https://github.com/apify/crawlee/issues/1981)) ([8122622](https://github.com/apify/crawlee/commit/8122622c3054a0e0e0c1869ba462276cbead8090)), closes [#1766](https://github.com/apify/crawlee/issues/1766)+++### Features++* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)diff --git a/lerna.json b/lerna.jsonindex fbffc6aac..be7d39a90 100644--- a/lerna.json+++ b/lerna.json@@ -2,7 +2,7 @@"packages": ["packages/*"],- "version": "3.4.1",+ "version": "3.4.2","command": {"version": {"conventionalCommits": true,diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.mdindex 91c8a25d7..b8dda4ffa 100644--- a/packages/basic-crawler/CHANGELOG.md+++ b/packages/basic-crawler/CHANGELOG.md@@ -3,6 +3,23 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Bug Fixes++* **basic-crawler:** limit `internalTimeoutMillis` in addition to `requestHandlerTimeoutMillis` ([#1981](https://github.com/apify/crawlee/issues/1981)) ([8122622](https://github.com/apify/crawlee/commit/8122622c3054a0e0e0c1869ba462276cbead8090)), closes [#1766](https://github.com/apify/crawlee/issues/1766)+++### Features++* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)+* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/basicdiff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex b316b318e..5ea5ba1d1 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -48,9 +48,9 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/core": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/core": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","got-scraping": "^3.2.9","ow": "^0.28.1","tslib": "^2.4.0",diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.mdindex 415fbbf47..6dd6db148 100644--- a/packages/browser-crawler/CHANGELOG.md+++ b/packages/browser-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Features++* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/browserdiff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 6762ca86d..1d30a3165 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,10 +54,10 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.4.1",- "@crawlee/browser-pool": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/basic": "^3.4.2",+ "@crawlee/browser-pool": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","ow": "^0.28.1","tslib": "^2.4.0"}diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.mdindex b9eb4222a..4b6cfb65d 100644--- a/packages/browser-pool/CHANGELOG.md+++ b/packages/browser-pool/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/browser-pool+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/browser-pooldiff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex e774a9a01..b7f8bdc95 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser-pool",- "version": "3.4.1",+ "version": "3.4.2","description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.","engines": {"node": ">=16.0.0"@@ -38,7 +38,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0",- "@crawlee/types": "^3.4.1",+ "@crawlee/types": "^3.4.2","fingerprint-generator": "^2.0.6","fingerprint-injector": "^2.0.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.mdindex e714cc28f..a9bd95cb1 100644--- a/packages/cheerio-crawler/CHANGELOG.md+++ b/packages/cheerio-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/cheerio+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/cheeriodiff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex c233447af..75f3a2e20 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -53,8 +53,8 @@"access": "public"},"dependencies": {- "@crawlee/http": "^3.4.1",- "@crawlee/types": "^3.4.1",+ "@crawlee/http": "^3.4.2",+ "@crawlee/types": "^3.4.2","cheerio": "^1.0.0-rc.12","htmlparser2": "^9.0.0","tslib": "^2.4.0"diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.mdindex fc24e15bf..74facbb9a 100644--- a/packages/cli/CHANGELOG.md+++ b/packages/cli/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/cli+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/clidiff --git a/packages/cli/package.json b/packages/cli/package.jsonindex 3e3958369..d91b5986d 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cli",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "^3.4.1",+ "@crawlee/templates": "^3.4.2","ansi-colors": "^4.1.3","fs-extra": "^11.0.0","inquirer": "^8.2.4",diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.mdindex 240e1f03a..2303ded86 100644--- a/packages/core/CHANGELOG.md+++ b/packages/core/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Features++* **core:** add `RequestQueue.addRequestsBatched()` that is non-blocking ([#1996](https://github.com/apify/crawlee/issues/1996)) ([c85485d](https://github.com/apify/crawlee/commit/c85485d6ca2bb61cfebb24a2ad99e0b3ba5c069b)), closes [#1995](https://github.com/apify/crawlee/issues/1995)+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)diff --git a/packages/core/package.json b/packages/core/package.jsonindex 9b656dfcb..ef30d7f8a 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/core",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -59,9 +59,9 @@"@apify/pseudo_url": "^2.0.14","@apify/timeout": "^0.3.0","@apify/utilities": "^2.3.3",- "@crawlee/memory-storage": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/memory-storage": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","@sapphire/async-queue": "^1.5.0","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0",diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.mdindex 09cd0ab4e..39cb8f60e 100644--- a/packages/crawlee/CHANGELOG.md+++ b/packages/crawlee/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package crawlee+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package crawleediff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 0aa8bb29e..5bb31a289 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,17 +54,17 @@"access": "public"},"dependencies": {- "@crawlee/basic": "^3.4.1",- "@crawlee/browser": "^3.4.1",- "@crawlee/cheerio": "^3.4.1",- "@crawlee/cli": "^3.4.1",- "@crawlee/core": "^3.4.1",- "@crawlee/http": "^3.4.1",- "@crawlee/jsdom": "^3.4.1",- "@crawlee/linkedom": "^3.4.1",- "@crawlee/playwright": "^3.4.1",- "@crawlee/puppeteer": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/basic": "^3.4.2",+ "@crawlee/browser": "^3.4.2",+ "@crawlee/cheerio": "^3.4.2",+ "@crawlee/cli": "^3.4.2",+ "@crawlee/core": "^3.4.2",+ "@crawlee/http": "^3.4.2",+ "@crawlee/jsdom": "^3.4.2",+ "@crawlee/linkedom": "^3.4.2",+ "@crawlee/playwright": "^3.4.2",+ "@crawlee/puppeteer": "^3.4.2",+ "@crawlee/utils": "^3.4.2","import-local": "^3.1.0","tslib": "^2.4.0"},diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.mdindex c675d497d..c61d8c054 100644--- a/packages/http-crawler/CHANGELOG.md+++ b/packages/http-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Features++* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 5beb41dbc..765152f61 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/http",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/basic": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/basic": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","@types/content-type": "^1.1.5","cheerio": "^1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.mdindex 81fe9ade3..fc94018bf 100644--- a/packages/jsdom-crawler/CHANGELOG.md+++ b/packages/jsdom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/jsdom+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex 1d386ff1e..fd33346a3 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/jsdom",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.4.1",- "@crawlee/types": "^3.4.1",+ "@crawlee/http": "^3.4.2",+ "@crawlee/types": "^3.4.2","@types/jsdom": "^21.0.0","cheerio": "^1.0.0-rc.12","jsdom": "^22.0.0",diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.mdindex 3f7172e71..6184c64c9 100644--- a/packages/linkedom-crawler/CHANGELOG.md+++ b/packages/linkedom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/linkedom+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex 48434ce3d..bda01c3a7 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/linkedom",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.4.1",- "@crawlee/types": "^3.4.1",+ "@crawlee/http": "^3.4.2",+ "@crawlee/types": "^3.4.2","linkedom": "^0.15.0","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.mdindex caf5ae671..30462526b 100644--- a/packages/memory-storage/CHANGELOG.md+++ b/packages/memory-storage/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/memory-storage+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/memory-storagediff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 4a704c2db..af0464f84 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/memory-storage",- "version": "3.4.1",+ "version": "3.4.2","description": "A simple in-memory storage implementation of the Apify API","engines": {"node": ">= 16"@@ -49,7 +49,7 @@},"dependencies": {"@apify/log": "^2.0.0",- "@crawlee/types": "^3.4.1",+ "@crawlee/types": "^3.4.2","@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4",diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.mdindex 9d20fbe4e..016843e2c 100644--- a/packages/playwright-crawler/CHANGELOG.md+++ b/packages/playwright-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/playwright+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/playwrightdiff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex da40b7f18..2b687a9e9 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.4.1",- "@crawlee/browser-pool": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/browser": "^3.4.2",+ "@crawlee/browser-pool": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","cheerio": "^1.0.0-rc.12","jquery": "^3.6.0","ow": "^0.28.1",diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.mdindex 53ae62178..d8a978f08 100644--- a/packages/puppeteer-crawler/CHANGELOG.md+++ b/packages/puppeteer-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/puppeteer+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/puppeteerdiff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex 2ac7efda9..506640e67 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.4.1",+ "version": "3.4.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.4.1",- "@crawlee/browser-pool": "^3.4.1",- "@crawlee/types": "^3.4.1",- "@crawlee/utils": "^3.4.1",+ "@crawlee/browser": "^3.4.2",+ "@crawlee/browser-pool": "^3.4.2",+ "@crawlee/types": "^3.4.2",+ "@crawlee/utils": "^3.4.2","cheerio": "^1.0.0-rc.12","devtools-protocol": "*","jquery": "^3.6.0",diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.mdindex 8b0e8425c..1f27d461e 100644--- a/packages/templates/CHANGELOG.md+++ b/packages/templates/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/templates+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/templatesdiff --git a/packages/templates/package.json b/packages/templates/package.jsonindex 66fd640fc..b8430993d 100644--- a/packages/templates/package.json+++ b/packages/templates/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/templates",- "version": "3.4.1",+ "version": "3.4.2","description": "Templates for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.mdindex e6107f2e0..2526afd6c 100644--- a/packages/types/CHANGELOG.md+++ b/packages/types/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)++**Note:** Version bump only for package @crawlee/types+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/typesdiff --git a/packages/types/package.json b/packages/types/package.jsonindex 2eebc700d..5aae7c92c 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/types",- "version": "3.4.1",+ "version": "3.4.2","description": "Shared types for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.mdindex c5be272e0..690b52a2f 100644--- a/packages/utils/CHANGELOG.md+++ b/packages/utils/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.4.2](https://github.com/apify/crawlee/compare/v3.4.1...v3.4.2) (2023-07-19)+++### Features++* retryOnBlocked detects blocked webpage ([#1956](https://github.com/apify/crawlee/issues/1956)) ([766fa9b](https://github.com/apify/crawlee/commit/766fa9b88029e9243a7427075384c1abe85c70c8))+++++## [3.4.1](https://github.com/apify/crawlee/compare/v3.4.0...v3.4.1) (2023-07-13)**Note:** Version bump only for package @crawlee/utilsdiff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 5f5910494..3feff10dd 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/utils",- "version": "3.4.1",+ "version": "3.4.2","description": "A set of shared utilities that can be used by crawlers","engines": {"node": ">=16.0.0"@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.0.0","@apify/ps-tree": "^1.1.4",- "@crawlee/types": "^3.4.1",+ "@crawlee/types": "^3.4.2","cheerio": "^1.0.0-rc.12","got-scraping": "^3.2.9","ow": "^0.28.1",diff --git a/yarn.lock b/yarn.lockindex 71e6e5007..1797c47f5 100644--- a/yarn.lock+++ b/yarn.lock@@ -738,16 +738,16 @@ __metadata:languageName: nodelinkType: hard-"@crawlee/basic@^3.4.1, @crawlee/basic@workspace:packages/basic-crawler":+"@crawlee/basic@^3.4.2, @crawlee/basic@workspace:packages/basic-crawler":version: 0.0.0-use.localresolution: "@crawlee/basic@workspace:packages/basic-crawler"dependencies:"@apify/log": ^2.0.0"@apify/timeout": ^0.3.0"@apify/utilities": ^2.0.0- "@crawlee/core": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/core": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2got-scraping: ^3.2.9ow: ^0.28.1tslib: ^2.4.0@@ -755,13 +755,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser-pool@^3.4.1, @crawlee/browser-pool@workspace:packages/browser-pool":+"@crawlee/browser-pool@^3.4.2, @crawlee/browser-pool@workspace:packages/browser-pool":version: 0.0.0-use.localresolution: "@crawlee/browser-pool@workspace:packages/browser-pool"dependencies:"@apify/log": ^2.0.0"@apify/timeout": ^0.3.0- "@crawlee/types": ^3.4.1+ "@crawlee/types": ^3.4.2fingerprint-generator: ^2.0.6fingerprint-injector: ^2.0.5lodash.merge: ^4.6.2@@ -783,37 +783,37 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser@^3.4.1, @crawlee/browser@workspace:packages/browser-crawler":+"@crawlee/browser@^3.4.2, @crawlee/browser@workspace:packages/browser-crawler":version: 0.0.0-use.localresolution: "@crawlee/browser@workspace:packages/browser-crawler"dependencies:"@apify/timeout": ^0.3.0- "@crawlee/basic": ^3.4.1- "@crawlee/browser-pool": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/basic": ^3.4.2+ "@crawlee/browser-pool": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2ow: ^0.28.1tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cheerio@^3.4.1, @crawlee/cheerio@workspace:packages/cheerio-crawler":+"@crawlee/cheerio@^3.4.2, @crawlee/cheerio@workspace:packages/cheerio-crawler":version: 0.0.0-use.localresolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"dependencies:- "@crawlee/http": ^3.4.1- "@crawlee/types": ^3.4.1+ "@crawlee/http": ^3.4.2+ "@crawlee/types": ^3.4.2cheerio: ^1.0.0-rc.12htmlparser2: ^9.0.0tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/cli@^3.4.1, @crawlee/cli@workspace:packages/cli":+"@crawlee/cli@^3.4.2, @crawlee/cli@workspace:packages/cli":version: 0.0.0-use.localresolution: "@crawlee/cli@workspace:packages/cli"dependencies:- "@crawlee/templates": ^3.4.1+ "@crawlee/templates": ^3.4.2ansi-colors: ^4.1.3fs-extra: ^11.0.0inquirer: ^8.2.4@@ -825,7 +825,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/core@^3.3.0, @crawlee/core@^3.4.1, @crawlee/core@workspace:packages/core":+"@crawlee/core@^3.3.0, @crawlee/core@^3.4.2, @crawlee/core@workspace:packages/core":version: 0.0.0-use.localresolution: "@crawlee/core@workspace:packages/core"dependencies:@@ -835,9 +835,9 @@ __metadata:"@apify/pseudo_url": ^2.0.14"@apify/timeout": ^0.3.0"@apify/utilities": ^2.3.3- "@crawlee/memory-storage": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/memory-storage": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2"@sapphire/async-queue": ^1.5.0"@types/tough-cookie": ^4.0.2"@vladfrangu/async_event_emitter": ^2.0.0@@ -855,15 +855,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@^3.4.1, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/http@^3.4.2, @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.0.0- "@crawlee/basic": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/basic": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2"@types/content-type": ^1.1.5cheerio: ^1.0.0-rc.12content-type: ^1.0.4@@ -876,14 +876,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/jsdom@^3.4.1, @crawlee/jsdom@workspace:packages/jsdom-crawler":+"@crawlee/jsdom@^3.4.2, @crawlee/jsdom@workspace:packages/jsdom-crawler":version: 0.0.0-use.localresolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.1.4- "@crawlee/http": ^3.4.1- "@crawlee/types": ^3.4.1+ "@crawlee/http": ^3.4.2+ "@crawlee/types": ^3.4.2"@types/jsdom": ^21.0.0cheerio: ^1.0.0-rc.12jsdom: ^22.0.0@@ -892,26 +892,26 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/linkedom@^3.4.1, @crawlee/linkedom@workspace:packages/linkedom-crawler":+"@crawlee/linkedom@^3.4.2, @crawlee/linkedom@workspace:packages/linkedom-crawler":version: 0.0.0-use.localresolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"dependencies:"@apify/timeout": ^0.3.0"@apify/utilities": ^2.1.4- "@crawlee/http": ^3.4.1- "@crawlee/types": ^3.4.1+ "@crawlee/http": ^3.4.2+ "@crawlee/types": ^3.4.2linkedom: ^0.15.0ow: ^0.28.2tslib: ^2.4.0languageName: unknownlinkType: soft-"@crawlee/memory-storage@^3.4.1, @crawlee/memory-storage@workspace:packages/memory-storage":+"@crawlee/memory-storage@^3.4.2, @crawlee/memory-storage@workspace:packages/memory-storage":version: 0.0.0-use.localresolution: "@crawlee/memory-storage@workspace:packages/memory-storage"dependencies:"@apify/log": ^2.0.0- "@crawlee/types": ^3.4.1+ "@crawlee/types": ^3.4.2"@sapphire/async-queue": ^1.5.0"@sapphire/shapeshift": ^3.0.0content-type: ^1.0.4@@ -923,16 +923,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/playwright@^3.4.1, @crawlee/playwright@workspace:packages/playwright-crawler":+"@crawlee/playwright@^3.4.2, @crawlee/playwright@workspace:packages/playwright-crawler":version: 0.0.0-use.localresolution: "@crawlee/playwright@workspace:packages/playwright-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.0.0- "@crawlee/browser": ^3.4.1- "@crawlee/browser-pool": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/browser": ^3.4.2+ "@crawlee/browser-pool": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2cheerio: ^1.0.0-rc.12jquery: ^3.6.0ow: ^0.28.1@@ -945,16 +945,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/puppeteer@^3.4.1, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":+"@crawlee/puppeteer@^3.4.2, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":version: 0.0.0-use.localresolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"dependencies:"@apify/datastructures": ^2.0.0"@apify/log": ^2.0.0- "@crawlee/browser": ^3.4.1- "@crawlee/browser-pool": ^3.4.1- "@crawlee/types": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/browser": ^3.4.2+ "@crawlee/browser-pool": ^3.4.2+ "@crawlee/types": ^3.4.2+ "@crawlee/utils": ^3.4.2cheerio: ^1.0.0-rc.12devtools-protocol: "*"jquery: ^3.6.0@@ -1024,7 +1024,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/templates@^3.4.1, @crawlee/templates@workspace:packages/templates":+"@crawlee/templates@^3.4.2, @crawlee/templates@workspace:packages/templates":version: 0.0.0-use.localresolution: "@crawlee/templates@workspace:packages/templates"dependencies:@@ -1036,7 +1036,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@^3.3.0, @crawlee/types@^3.4.1, @crawlee/types@workspace:packages/types":+"@crawlee/types@^3.3.0, @crawlee/types@^3.4.2, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -1044,13 +1044,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/utils@^3.3.0, @crawlee/utils@^3.4.1, @crawlee/utils@workspace:packages/utils":+"@crawlee/utils@^3.3.0, @crawlee/utils@^3.4.2, @crawlee/utils@workspace:packages/utils":version: 0.0.0-use.localresolution: "@crawlee/utils@workspace:packages/utils"dependencies:"@apify/log": ^2.0.0"@apify/ps-tree": ^1.1.4- "@crawlee/types": ^3.4.1+ "@crawlee/types": ^3.4.2cheerio: ^1.0.0-rc.12got-scraping: ^3.2.9ow: ^0.28.1@@ -4232,17 +4232,17 @@ __metadata:version: 0.0.0-use.localresolution: "crawlee@workspace:packages/crawlee"dependencies:- "@crawlee/basic": ^3.4.1- "@crawlee/browser": ^3.4.1- "@crawlee/cheerio": ^3.4.1- "@crawlee/cli": ^3.4.1- "@crawlee/core": ^3.4.1- "@crawlee/http": ^3.4.1- "@crawlee/jsdom": ^3.4.1- "@crawlee/linkedom": ^3.4.1- "@crawlee/playwright": ^3.4.1- "@crawlee/puppeteer": ^3.4.1- "@crawlee/utils": ^3.4.1+ "@crawlee/basic": ^3.4.2+ "@crawlee/browser": ^3.4.2+ "@crawlee/cheerio": ^3.4.2+ "@crawlee/cli": ^3.4.2+ "@crawlee/core": ^3.4.2+ "@crawlee/http": ^3.4.2+ "@crawlee/jsdom": ^3.4.2+ "@crawlee/linkedom": ^3.4.2+ "@crawlee/playwright": ^3.4.2+ "@crawlee/puppeteer": ^3.4.2+ "@crawlee/utils": ^3.4.2import-local: ^3.1.0tslib: ^2.4.0peerDependencies: