Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): v3.10.2
Apify Release Bot7cb86d3f93f0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 46d685867..3e8942cee 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,23 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)+++### Bug Fixes++* Autodetect sitemap filetype from content ([#2497](https://github.com/apify/crawlee/issues/2497)) ([62a9f40](https://github.com/apify/crawlee/commit/62a9f4036dba92d07547af489ac8b6c7974faa6f)), closes [#2461](https://github.com/apify/crawlee/issues/2461)+* improve fix for double extension in KVS with HTML files ([#2505](https://github.com/apify/crawlee/issues/2505)) ([157927d](https://github.com/apify/crawlee/commit/157927d67f42342c20fdf01ef81bdafd7095f0b8)), closes [#2419](https://github.com/apify/crawlee/issues/2419)+++### Features++* Loading sitemaps from string ([#2496](https://github.com/apify/crawlee/issues/2496)) ([38ed0d6](https://github.com/apify/crawlee/commit/38ed0d6ad90a868df9c02632334fec8db9ef29a0)), closes [#2460](https://github.com/apify/crawlee/issues/2460)+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)diff --git a/lerna.json b/lerna.jsonindex 6166ed21e..3ed15df63 100644--- a/lerna.json+++ b/lerna.json@@ -1,17 +1,22 @@{- "packages": ["packages/*"],- "version": "3.10.1",- "command": {- "version": {- "conventionalCommits": true,- "createRelease": "github",- "message": "chore(release): %s"- },- "publish": {- "assets": []- }- },- "npmClient": "yarn",- "useNx": false,- "ignoreChanges": ["**/test/**", "**/*.md"]-}+ "packages": [+ "packages/*"+ ],+ "version": "3.10.2",+ "command": {+ "version": {+ "conventionalCommits": true,+ "createRelease": "github",+ "message": "chore(release): %s"+ },+ "publish": {+ "assets": []+ }+ },+ "npmClient": "yarn",+ "useNx": false,+ "ignoreChanges": [+ "**/test/**",+ "**/*.md"+ ]+}\ No newline at end of filediff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.mdindex b2e949572..d1e832b15 100644--- a/packages/basic-crawler/CHANGELOG.md+++ b/packages/basic-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/basic+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/basicdiff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 6a90d973a..c617a22c6 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -48,9 +48,9 @@"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/core": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/core": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","csv-stringify": "^6.2.0","fs-extra": "^11.0.0","got-scraping": "^4.0.0",diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.mdindex 167b767c7..38081ee08 100644--- a/packages/browser-crawler/CHANGELOG.md+++ b/packages/browser-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/browser+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/browserdiff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 23194deb4..865f9d12a 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,10 +54,10 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "3.10.1",- "@crawlee/browser-pool": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/basic": "^3.10.2",+ "@crawlee/browser-pool": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","ow": "^0.28.1","tslib": "^2.4.0","type-fest": "^4.0.0"diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.mdindex a99c58cda..3701e68eb 100644--- a/packages/browser-pool/CHANGELOG.md+++ b/packages/browser-pool/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/browser-pool+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/browser-pooldiff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 06bc0e884..9e90abdb9 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser-pool",- "version": "3.10.1",+ "version": "3.10.2","description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.","engines": {"node": ">=16.0.0"@@ -38,8 +38,8 @@"dependencies": {"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0",- "@crawlee/core": "3.10.1",- "@crawlee/types": "3.10.1",+ "@crawlee/core": "^3.10.2",+ "@crawlee/types": "^3.10.2","fingerprint-generator": "^2.0.6","fingerprint-injector": "^2.0.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.mdindex 58229ed8b..09b32128d 100644--- a/packages/cheerio-crawler/CHANGELOG.md+++ b/packages/cheerio-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/cheerio+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/cheeriodiff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex a80ff0330..e55bd4567 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -53,9 +53,9 @@"access": "public"},"dependencies": {- "@crawlee/http": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/http": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","cheerio": "^1.0.0-rc.12","htmlparser2": "^9.0.0","tslib": "^2.4.0"diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.mdindex 35347d1bb..249b3b1e2 100644--- a/packages/cli/CHANGELOG.md+++ b/packages/cli/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/cli+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/clidiff --git a/packages/cli/package.json b/packages/cli/package.jsonindex 47e46b734..c32a82108 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cli",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "3.10.1",+ "@crawlee/templates": "^3.10.2","ansi-colors": "^4.1.3","fs-extra": "^11.0.0","inquirer": "^8.2.4",diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.mdindex b51b4ff6a..0d9c10a6a 100644--- a/packages/core/CHANGELOG.md+++ b/packages/core/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/core+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)diff --git a/packages/core/package.json b/packages/core/package.jsonindex 42ebb28a7..74032408e 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/core",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -59,9 +59,9 @@"@apify/pseudo_url": "^2.0.30","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/memory-storage": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/memory-storage": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","@sapphire/async-queue": "^1.5.1","@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.2.2",diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.mdindex 663f6842c..e32f420c2 100644--- a/packages/crawlee/CHANGELOG.md+++ b/packages/crawlee/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package crawlee+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package crawleediff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 29ff4aa05..322747eae 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,18 +54,18 @@"access": "public"},"dependencies": {- "@crawlee/basic": "3.10.1",- "@crawlee/browser": "3.10.1",- "@crawlee/browser-pool": "3.10.1",- "@crawlee/cheerio": "3.10.1",- "@crawlee/cli": "3.10.1",- "@crawlee/core": "3.10.1",- "@crawlee/http": "3.10.1",- "@crawlee/jsdom": "3.10.1",- "@crawlee/linkedom": "3.10.1",- "@crawlee/playwright": "3.10.1",- "@crawlee/puppeteer": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/basic": "^3.10.2",+ "@crawlee/browser": "^3.10.2",+ "@crawlee/browser-pool": "^3.10.2",+ "@crawlee/cheerio": "^3.10.2",+ "@crawlee/cli": "^3.10.2",+ "@crawlee/core": "^3.10.2",+ "@crawlee/http": "^3.10.2",+ "@crawlee/jsdom": "^3.10.2",+ "@crawlee/linkedom": "^3.10.2",+ "@crawlee/playwright": "^3.10.2",+ "@crawlee/puppeteer": "^3.10.2",+ "@crawlee/utils": "^3.10.2","import-local": "^3.1.0","tslib": "^2.4.0"},diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.mdindex c450147a2..61c309802 100644--- a/packages/http-crawler/CHANGELOG.md+++ b/packages/http-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/http+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/httpdiff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 28ca437d4..1c223ca00 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/http",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/basic": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/basic": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","@types/content-type": "^1.1.5","cheerio": "^1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.mdindex 4f608feb3..30e3d93aa 100644--- a/packages/jsdom-crawler/CHANGELOG.md+++ b/packages/jsdom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/jsdom+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/jsdomdiff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex 1ec83f35f..5ceaef4ef 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/jsdom",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "3.10.1",- "@crawlee/types": "3.10.1",+ "@crawlee/http": "^3.10.2",+ "@crawlee/types": "^3.10.2","@types/jsdom": "^21.0.0","cheerio": "^1.0.0-rc.12","jsdom": "^24.0.0",diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.mdindex 1affa574f..ca4b220b7 100644--- a/packages/linkedom-crawler/CHANGELOG.md+++ b/packages/linkedom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/linkedom+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/linkedomdiff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex f44801606..963f4b043 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/linkedom",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "3.10.1",- "@crawlee/types": "3.10.1",+ "@crawlee/http": "^3.10.2",+ "@crawlee/types": "^3.10.2","linkedom": "^0.18.0","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.mdindex c65d38f75..4fffeeab5 100644--- a/packages/memory-storage/CHANGELOG.md+++ b/packages/memory-storage/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)+++### Bug Fixes++* improve fix for double extension in KVS with HTML files ([#2505](https://github.com/apify/crawlee/issues/2505)) ([157927d](https://github.com/apify/crawlee/commit/157927d67f42342c20fdf01ef81bdafd7095f0b8)), closes [#2419](https://github.com/apify/crawlee/issues/2419)+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/memory-storagediff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 876b32dcb..c8155cd6b 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/memory-storage",- "version": "3.10.1",+ "version": "3.10.2","description": "A simple in-memory storage implementation of the Apify API","engines": {"node": ">= 16"@@ -49,7 +49,7 @@},"dependencies": {"@apify/log": "^2.4.0",- "@crawlee/types": "3.10.1",+ "@crawlee/types": "^3.10.2","@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4",diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.mdindex 50eefc19e..096efb25d 100644--- a/packages/playwright-crawler/CHANGELOG.md+++ b/packages/playwright-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/playwright+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/playwrightdiff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 7f1842b43..86f6ef598 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -56,11 +56,11 @@"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0","@apify/timeout": "^0.3.1",- "@crawlee/browser": "3.10.1",- "@crawlee/browser-pool": "3.10.1",- "@crawlee/core": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/browser": "^3.10.2",+ "@crawlee/browser-pool": "^3.10.2",+ "@crawlee/core": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","cheerio": "^1.0.0-rc.12","idcac-playwright": "^0.1.2","jquery": "^3.6.0",diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.mdindex 3ccef9f8d..88d9d840a 100644--- a/packages/puppeteer-crawler/CHANGELOG.md+++ b/packages/puppeteer-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/puppeteer+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/puppeteerdiff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex 6b4688359..97bbfdfbc 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.10.1",+ "version": "3.10.2","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "3.10.1",- "@crawlee/browser-pool": "3.10.1",- "@crawlee/types": "3.10.1",- "@crawlee/utils": "3.10.1",+ "@crawlee/browser": "^3.10.2",+ "@crawlee/browser-pool": "^3.10.2",+ "@crawlee/types": "^3.10.2",+ "@crawlee/utils": "^3.10.2","cheerio": "^1.0.0-rc.12","devtools-protocol": "*","idcac-playwright": "^0.1.2",diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.mdindex f02e86c68..36785316b 100644--- a/packages/templates/CHANGELOG.md+++ b/packages/templates/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/templates+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/templatesdiff --git a/packages/templates/package.json b/packages/templates/package.jsonindex 4db916894..f5808170e 100644--- a/packages/templates/package.json+++ b/packages/templates/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/templates",- "version": "3.10.1",+ "version": "3.10.2","description": "Templates for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.mdindex 69f4b751f..220ecdf52 100644--- a/packages/types/CHANGELOG.md+++ b/packages/types/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)++**Note:** Version bump only for package @crawlee/types+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)**Note:** Version bump only for package @crawlee/typesdiff --git a/packages/types/package.json b/packages/types/package.jsonindex deebf3788..811512a01 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/types",- "version": "3.10.1",+ "version": "3.10.2","description": "Shared types for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.mdindex 135967b21..6b993fb33 100644--- a/packages/utils/CHANGELOG.md+++ b/packages/utils/CHANGELOG.md@@ -3,6 +3,22 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)+++### Bug Fixes++* Autodetect sitemap filetype from content ([#2497](https://github.com/apify/crawlee/issues/2497)) ([62a9f40](https://github.com/apify/crawlee/commit/62a9f4036dba92d07547af489ac8b6c7974faa6f)), closes [#2461](https://github.com/apify/crawlee/issues/2461)+++### Features++* Loading sitemaps from string ([#2496](https://github.com/apify/crawlee/issues/2496)) ([38ed0d6](https://github.com/apify/crawlee/commit/38ed0d6ad90a868df9c02632334fec8db9ef29a0)), closes [#2460](https://github.com/apify/crawlee/issues/2460)+++++## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)diff --git a/packages/utils/package.json b/packages/utils/package.jsonindex abf687ad2..74cb9e235 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/utils",- "version": "3.10.1",+ "version": "3.10.2","description": "A set of shared utilities that can be used by crawlers","engines": {"node": ">=16.0.0"@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.4.0","@apify/ps-tree": "^1.2.0",- "@crawlee/types": "3.10.1",+ "@crawlee/types": "^3.10.2","@types/sax": "^1.2.7","cheerio": "^1.0.0-rc.12","file-type": "^19.0.0",diff --git a/yarn.lock b/yarn.lockindex 68b79b46e..a59bfcb96 100644--- a/yarn.lock+++ b/yarn.lock@@ -486,16 +486,16 @@ __metadata:languageName: nodelinkType: hard-"@crawlee/basic@npm:3.10.1, @crawlee/basic@workspace:packages/basic-crawler":+"@crawlee/basic@npm:^3.10.2, @crawlee/basic@workspace:packages/basic-crawler":version: 0.0.0-use.localresolution: "@crawlee/basic@workspace:packages/basic-crawler"dependencies:"@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/core": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/core": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"csv-stringify: "npm:^6.2.0"fs-extra: "npm:^11.0.0"got-scraping: "npm:^4.0.0"@@ -506,14 +506,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser-pool@npm:3.10.1, @crawlee/browser-pool@workspace:packages/browser-pool":+"@crawlee/browser-pool@npm:^3.10.2, @crawlee/browser-pool@workspace:packages/browser-pool":version: 0.0.0-use.localresolution: "@crawlee/browser-pool@workspace:packages/browser-pool"dependencies:"@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.0"- "@crawlee/core": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"+ "@crawlee/core": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"fingerprint-generator: "npm:^2.0.6"fingerprint-injector: "npm:^2.0.5"lodash.merge: "npm:^4.6.2"@@ -535,39 +535,39 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser@npm:3.10.1, @crawlee/browser@workspace:packages/browser-crawler":+"@crawlee/browser@npm:^3.10.2, @crawlee/browser@workspace:packages/browser-crawler":version: 0.0.0-use.localresolution: "@crawlee/browser@workspace:packages/browser-crawler"dependencies:"@apify/timeout": "npm:^0.3.0"- "@crawlee/basic": "npm:3.10.1"- "@crawlee/browser-pool": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/basic": "npm:^3.10.2"+ "@crawlee/browser-pool": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"ow: "npm:^0.28.1"tslib: "npm:^2.4.0"type-fest: "npm:^4.0.0"languageName: unknownlinkType: soft-"@crawlee/cheerio@npm:3.10.1, @crawlee/cheerio@workspace:packages/cheerio-crawler":+"@crawlee/cheerio@npm:^3.10.2, @crawlee/cheerio@workspace:packages/cheerio-crawler":version: 0.0.0-use.localresolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"dependencies:- "@crawlee/http": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/http": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"cheerio: "npm:^1.0.0-rc.12"htmlparser2: "npm:^9.0.0"tslib: "npm:^2.4.0"languageName: unknownlinkType: soft-"@crawlee/cli@npm:3.10.1, @crawlee/cli@workspace:packages/cli":+"@crawlee/cli@npm:^3.10.2, @crawlee/cli@workspace:packages/cli":version: 0.0.0-use.localresolution: "@crawlee/cli@workspace:packages/cli"dependencies:- "@crawlee/templates": "npm:3.10.1"+ "@crawlee/templates": "npm:^3.10.2"ansi-colors: "npm:^4.1.3"fs-extra: "npm:^11.0.0"inquirer: "npm:^8.2.4"@@ -579,7 +579,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/core@npm:3.10.1, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":+"@crawlee/core@npm:^3.10.2, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":version: 0.0.0-use.localresolution: "@crawlee/core@workspace:packages/core"dependencies:@@ -589,9 +589,9 @@ __metadata:"@apify/pseudo_url": "npm:^2.0.30""@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/memory-storage": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/memory-storage": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2""@sapphire/async-queue": "npm:^1.5.1""@types/tough-cookie": "npm:^4.0.2""@vladfrangu/async_event_emitter": "npm:^2.2.2"@@ -610,15 +610,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@npm:3.10.1, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/http@npm:^3.10.2, @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/basic": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/basic": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2""@types/content-type": "npm:^1.1.5"cheerio: "npm:^1.0.0-rc.12"content-type: "npm:^1.0.4"@@ -631,14 +631,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/jsdom@npm:3.10.1, @crawlee/jsdom@workspace:packages/jsdom-crawler":+"@crawlee/jsdom@npm:^3.10.2, @crawlee/jsdom@workspace:packages/jsdom-crawler":version: 0.0.0-use.localresolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/http": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"+ "@crawlee/http": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2""@types/jsdom": "npm:^21.0.0"cheerio: "npm:^1.0.0-rc.12"jsdom: "npm:^24.0.0"@@ -647,26 +647,26 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/linkedom@npm:3.10.1, @crawlee/linkedom@workspace:packages/linkedom-crawler":+"@crawlee/linkedom@npm:^3.10.2, @crawlee/linkedom@workspace:packages/linkedom-crawler":version: 0.0.0-use.localresolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/http": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"+ "@crawlee/http": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"linkedom: "npm:^0.18.0"ow: "npm:^0.28.2"tslib: "npm:^2.4.0"languageName: unknownlinkType: soft-"@crawlee/memory-storage@npm:3.10.1, @crawlee/memory-storage@workspace:packages/memory-storage":+"@crawlee/memory-storage@npm:^3.10.2, @crawlee/memory-storage@workspace:packages/memory-storage":version: 0.0.0-use.localresolution: "@crawlee/memory-storage@workspace:packages/memory-storage"dependencies:"@apify/log": "npm:^2.4.0"- "@crawlee/types": "npm:3.10.1"+ "@crawlee/types": "npm:^3.10.2""@sapphire/async-queue": "npm:^1.5.0""@sapphire/shapeshift": "npm:^3.0.0"content-type: "npm:^1.0.4"@@ -678,18 +678,18 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/playwright@npm:3.10.1, @crawlee/playwright@workspace:packages/playwright-crawler":+"@crawlee/playwright@npm:^3.10.2, @crawlee/playwright@workspace:packages/playwright-crawler":version: 0.0.0-use.localresolution: "@crawlee/playwright@workspace:packages/playwright-crawler"dependencies:"@apify/datastructures": "npm:^2.0.0""@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.1"- "@crawlee/browser": "npm:3.10.1"- "@crawlee/browser-pool": "npm:3.10.1"- "@crawlee/core": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/browser": "npm:^3.10.2"+ "@crawlee/browser-pool": "npm:^3.10.2"+ "@crawlee/core": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"cheerio: "npm:^1.0.0-rc.12"idcac-playwright: "npm:^0.1.2"jquery: "npm:^3.6.0"@@ -707,16 +707,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/puppeteer@npm:3.10.1, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":+"@crawlee/puppeteer@npm:^3.10.2, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":version: 0.0.0-use.localresolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"dependencies:"@apify/datastructures": "npm:^2.0.0""@apify/log": "npm:^2.4.0"- "@crawlee/browser": "npm:3.10.1"- "@crawlee/browser-pool": "npm:3.10.1"- "@crawlee/types": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/browser": "npm:^3.10.2"+ "@crawlee/browser-pool": "npm:^3.10.2"+ "@crawlee/types": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"cheerio: "npm:^1.0.0-rc.12"devtools-protocol: "npm:*"idcac-playwright: "npm:^0.1.2"@@ -797,7 +797,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/templates@npm:3.10.1, @crawlee/templates@workspace:packages/templates":+"@crawlee/templates@npm:^3.10.2, @crawlee/templates@workspace:packages/templates":version: 0.0.0-use.localresolution: "@crawlee/templates@workspace:packages/templates"dependencies:@@ -809,7 +809,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@npm:3.10.1, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":+"@crawlee/types@npm:^3.10.2, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -817,13 +817,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/utils@npm:3.10.1, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":+"@crawlee/utils@npm:^3.10.2, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":version: 0.0.0-use.localresolution: "@crawlee/utils@workspace:packages/utils"dependencies:"@apify/log": "npm:^2.4.0""@apify/ps-tree": "npm:^1.2.0"- "@crawlee/types": "npm:3.10.1"+ "@crawlee/types": "npm:^3.10.2""@types/sax": "npm:^1.2.7""@types/whatwg-mimetype": "npm:^3.0.2"cheerio: "npm:^1.0.0-rc.12"@@ -4165,18 +4165,18 @@ __metadata:version: 0.0.0-use.localresolution: "crawlee@workspace:packages/crawlee"dependencies:- "@crawlee/basic": "npm:3.10.1"- "@crawlee/browser": "npm:3.10.1"- "@crawlee/browser-pool": "npm:3.10.1"- "@crawlee/cheerio": "npm:3.10.1"- "@crawlee/cli": "npm:3.10.1"- "@crawlee/core": "npm:3.10.1"- "@crawlee/http": "npm:3.10.1"- "@crawlee/jsdom": "npm:3.10.1"- "@crawlee/linkedom": "npm:3.10.1"- "@crawlee/playwright": "npm:3.10.1"- "@crawlee/puppeteer": "npm:3.10.1"- "@crawlee/utils": "npm:3.10.1"+ "@crawlee/basic": "npm:^3.10.2"+ "@crawlee/browser": "npm:^3.10.2"+ "@crawlee/browser-pool": "npm:^3.10.2"+ "@crawlee/cheerio": "npm:^3.10.2"+ "@crawlee/cli": "npm:^3.10.2"+ "@crawlee/core": "npm:^3.10.2"+ "@crawlee/http": "npm:^3.10.2"+ "@crawlee/jsdom": "npm:^3.10.2"+ "@crawlee/linkedom": "npm:^3.10.2"+ "@crawlee/playwright": "npm:^3.10.2"+ "@crawlee/puppeteer": "npm:^3.10.2"+ "@crawlee/utils": "npm:^3.10.2"import-local: "npm:^3.1.0"tslib: "npm:^2.4.0"peerDependencies: