Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(release): v3.10.2

Apify Release Bot7cb86d3f93f0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 46d685867..3e8942cee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,23 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+
+### Bug Fixes
+
+* Autodetect sitemap filetype from content ([#2497](https://github.com/apify/crawlee/issues/2497)) ([62a9f40](https://github.com/apify/crawlee/commit/62a9f4036dba92d07547af489ac8b6c7974faa6f)), closes [#2461](https://github.com/apify/crawlee/issues/2461)
+* improve fix for double extension in KVS with HTML files ([#2505](https://github.com/apify/crawlee/issues/2505)) ([157927d](https://github.com/apify/crawlee/commit/157927d67f42342c20fdf01ef81bdafd7095f0b8)), closes [#2419](https://github.com/apify/crawlee/issues/2419)
+
+
+### Features
+
+* Loading sitemaps from string ([#2496](https://github.com/apify/crawlee/issues/2496)) ([38ed0d6](https://github.com/apify/crawlee/commit/38ed0d6ad90a868df9c02632334fec8db9ef29a0)), closes [#2460](https://github.com/apify/crawlee/issues/2460)
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
diff --git a/lerna.json b/lerna.json
index 6166ed21e..3ed15df63 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,17 +1,22 @@
{
- "packages": ["packages/*"],
- "version": "3.10.1",
- "command": {
- "version": {
- "conventionalCommits": true,
- "createRelease": "github",
- "message": "chore(release): %s"
- },
- "publish": {
- "assets": []
- }
- },
- "npmClient": "yarn",
- "useNx": false,
- "ignoreChanges": ["**/test/**", "**/*.md"]
-}
+ "packages": [
+ "packages/*"
+ ],
+ "version": "3.10.2",
+ "command": {
+ "version": {
+ "conventionalCommits": true,
+ "createRelease": "github",
+ "message": "chore(release): %s"
+ },
+ "publish": {
+ "assets": []
+ }
+ },
+ "npmClient": "yarn",
+ "useNx": false,
+ "ignoreChanges": [
+ "**/test/**",
+ "**/*.md"
+ ]
+}
\ No newline at end of file
diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.md
index b2e949572..d1e832b15 100644
--- a/packages/basic-crawler/CHANGELOG.md
+++ b/packages/basic-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/basic
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/basic
diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json
index 6a90d973a..c617a22c6 100644
--- a/packages/basic-crawler/package.json
+++ b/packages/basic-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/basic",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -48,9 +48,9 @@
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/core": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/core": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"csv-stringify": "^6.2.0",
"fs-extra": "^11.0.0",
"got-scraping": "^4.0.0",
diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.md
index 167b767c7..38081ee08 100644
--- a/packages/browser-crawler/CHANGELOG.md
+++ b/packages/browser-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/browser
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/browser
diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json
index 23194deb4..865f9d12a 100644
--- a/packages/browser-crawler/package.json
+++ b/packages/browser-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,10 +54,10 @@
},
"dependencies": {
"@apify/timeout": "^0.3.0",
- "@crawlee/basic": "3.10.1",
- "@crawlee/browser-pool": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/basic": "^3.10.2",
+ "@crawlee/browser-pool": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"ow": "^0.28.1",
"tslib": "^2.4.0",
"type-fest": "^4.0.0"
diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.md
index a99c58cda..3701e68eb 100644
--- a/packages/browser-pool/CHANGELOG.md
+++ b/packages/browser-pool/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/browser-pool
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/browser-pool
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index 06bc0e884..9e90abdb9 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/browser-pool",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -38,8 +38,8 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
- "@crawlee/core": "3.10.1",
- "@crawlee/types": "3.10.1",
+ "@crawlee/core": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
"fingerprint-generator": "^2.0.6",
"fingerprint-injector": "^2.0.5",
"lodash.merge": "^4.6.2",
diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.md
index 58229ed8b..09b32128d 100644
--- a/packages/cheerio-crawler/CHANGELOG.md
+++ b/packages/cheerio-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/cheerio
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/cheerio
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index a80ff0330..e55bd4567 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cheerio",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -53,9 +53,9 @@
"access": "public"
},
"dependencies": {
- "@crawlee/http": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/http": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"cheerio": "^1.0.0-rc.12",
"htmlparser2": "^9.0.0",
"tslib": "^2.4.0"
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
index 35347d1bb..249b3b1e2 100644
--- a/packages/cli/CHANGELOG.md
+++ b/packages/cli/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/cli
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/cli
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 47e46b734..c32a82108 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/cli",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -51,7 +51,7 @@
"access": "public"
},
"dependencies": {
- "@crawlee/templates": "3.10.1",
+ "@crawlee/templates": "^3.10.2",
"ansi-colors": "^4.1.3",
"fs-extra": "^11.0.0",
"inquirer": "^8.2.4",
diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md
index b51b4ff6a..0d9c10a6a 100644
--- a/packages/core/CHANGELOG.md
+++ b/packages/core/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/core
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
diff --git a/packages/core/package.json b/packages/core/package.json
index 42ebb28a7..74032408e 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/core",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -59,9 +59,9 @@
"@apify/pseudo_url": "^2.0.30",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/memory-storage": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/memory-storage": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"@sapphire/async-queue": "^1.5.1",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.2.2",
diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.md
index 663f6842c..e32f420c2 100644
--- a/packages/crawlee/CHANGELOG.md
+++ b/packages/crawlee/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package crawlee
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package crawlee
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index 29ff4aa05..322747eae 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -1,6 +1,6 @@
{
"name": "crawlee",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -54,18 +54,18 @@
"access": "public"
},
"dependencies": {
- "@crawlee/basic": "3.10.1",
- "@crawlee/browser": "3.10.1",
- "@crawlee/browser-pool": "3.10.1",
- "@crawlee/cheerio": "3.10.1",
- "@crawlee/cli": "3.10.1",
- "@crawlee/core": "3.10.1",
- "@crawlee/http": "3.10.1",
- "@crawlee/jsdom": "3.10.1",
- "@crawlee/linkedom": "3.10.1",
- "@crawlee/playwright": "3.10.1",
- "@crawlee/puppeteer": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/basic": "^3.10.2",
+ "@crawlee/browser": "^3.10.2",
+ "@crawlee/browser-pool": "^3.10.2",
+ "@crawlee/cheerio": "^3.10.2",
+ "@crawlee/cli": "^3.10.2",
+ "@crawlee/core": "^3.10.2",
+ "@crawlee/http": "^3.10.2",
+ "@crawlee/jsdom": "^3.10.2",
+ "@crawlee/linkedom": "^3.10.2",
+ "@crawlee/playwright": "^3.10.2",
+ "@crawlee/puppeteer": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"import-local": "^3.1.0",
"tslib": "^2.4.0"
},
diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.md
index c450147a2..61c309802 100644
--- a/packages/http-crawler/CHANGELOG.md
+++ b/packages/http-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/http
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/http
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index 28ca437d4..1c223ca00 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/http",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,9 +55,9 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/basic": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/basic": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"@types/content-type": "^1.1.5",
"cheerio": "^1.0.0-rc.12",
"content-type": "^1.0.4",
diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.md
index 4f608feb3..30e3d93aa 100644
--- a/packages/jsdom-crawler/CHANGELOG.md
+++ b/packages/jsdom-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/jsdom
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/jsdom
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index 1ec83f35f..5ceaef4ef 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/jsdom",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/http": "3.10.1",
- "@crawlee/types": "3.10.1",
+ "@crawlee/http": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
"@types/jsdom": "^21.0.0",
"cheerio": "^1.0.0-rc.12",
"jsdom": "^24.0.0",
diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.md
index 1affa574f..ca4b220b7 100644
--- a/packages/linkedom-crawler/CHANGELOG.md
+++ b/packages/linkedom-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/linkedom
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/linkedom
diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.json
index f44801606..963f4b043 100644
--- a/packages/linkedom-crawler/package.json
+++ b/packages/linkedom-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/linkedom",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
- "@crawlee/http": "3.10.1",
- "@crawlee/types": "3.10.1",
+ "@crawlee/http": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
"linkedom": "^0.18.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.md
index c65d38f75..4fffeeab5 100644
--- a/packages/memory-storage/CHANGELOG.md
+++ b/packages/memory-storage/CHANGELOG.md
@@ -3,6 +3,17 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+
+### Bug Fixes
+
+* improve fix for double extension in KVS with HTML files ([#2505](https://github.com/apify/crawlee/issues/2505)) ([157927d](https://github.com/apify/crawlee/commit/157927d67f42342c20fdf01ef81bdafd7095f0b8)), closes [#2419](https://github.com/apify/crawlee/issues/2419)
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/memory-storage
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index 876b32dcb..c8155cd6b 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/memory-storage",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "A simple in-memory storage implementation of the Apify API",
"engines": {
"node": ">= 16"
@@ -49,7 +49,7 @@
},
"dependencies": {
"@apify/log": "^2.4.0",
- "@crawlee/types": "3.10.1",
+ "@crawlee/types": "^3.10.2",
"@sapphire/async-queue": "^1.5.0",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.md
index 50eefc19e..096efb25d 100644
--- a/packages/playwright-crawler/CHANGELOG.md
+++ b/packages/playwright-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/playwright
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/playwright
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 7f1842b43..86f6ef598 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/playwright",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -56,11 +56,11 @@
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.1",
- "@crawlee/browser": "3.10.1",
- "@crawlee/browser-pool": "3.10.1",
- "@crawlee/core": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/browser": "^3.10.2",
+ "@crawlee/browser-pool": "^3.10.2",
+ "@crawlee/core": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"cheerio": "^1.0.0-rc.12",
"idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.md
index 3ccef9f8d..88d9d840a 100644
--- a/packages/puppeteer-crawler/CHANGELOG.md
+++ b/packages/puppeteer-crawler/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/puppeteer
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/puppeteer
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 6b4688359..97bbfdfbc 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/puppeteer",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=16.0.0"
@@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
- "@crawlee/browser": "3.10.1",
- "@crawlee/browser-pool": "3.10.1",
- "@crawlee/types": "3.10.1",
- "@crawlee/utils": "3.10.1",
+ "@crawlee/browser": "^3.10.2",
+ "@crawlee/browser-pool": "^3.10.2",
+ "@crawlee/types": "^3.10.2",
+ "@crawlee/utils": "^3.10.2",
"cheerio": "^1.0.0-rc.12",
"devtools-protocol": "*",
"idcac-playwright": "^0.1.2",
diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.md
index f02e86c68..36785316b 100644
--- a/packages/templates/CHANGELOG.md
+++ b/packages/templates/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/templates
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/templates
diff --git a/packages/templates/package.json b/packages/templates/package.json
index 4db916894..f5808170e 100644
--- a/packages/templates/package.json
+++ b/packages/templates/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/templates",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "Templates for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
index 69f4b751f..220ecdf52 100644
--- a/packages/types/CHANGELOG.md
+++ b/packages/types/CHANGELOG.md
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+**Note:** Version bump only for package @crawlee/types
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
**Note:** Version bump only for package @crawlee/types
diff --git a/packages/types/package.json b/packages/types/package.json
index deebf3788..811512a01 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/types",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "Shared types for the crawlee projects",
"engines": {
"node": ">=16.0.0"
diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md
index 135967b21..6b993fb33 100644
--- a/packages/utils/CHANGELOG.md
+++ b/packages/utils/CHANGELOG.md
@@ -3,6 +3,22 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+## [3.10.2](https://github.com/apify/crawlee/compare/v3.10.1...v3.10.2) (2024-06-03)
+
+
+### Bug Fixes
+
+* Autodetect sitemap filetype from content ([#2497](https://github.com/apify/crawlee/issues/2497)) ([62a9f40](https://github.com/apify/crawlee/commit/62a9f4036dba92d07547af489ac8b6c7974faa6f)), closes [#2461](https://github.com/apify/crawlee/issues/2461)
+
+
+### Features
+
+* Loading sitemaps from string ([#2496](https://github.com/apify/crawlee/issues/2496)) ([38ed0d6](https://github.com/apify/crawlee/commit/38ed0d6ad90a868df9c02632334fec8db9ef29a0)), closes [#2460](https://github.com/apify/crawlee/issues/2460)
+
+
+
+
+
## [3.10.1](https://github.com/apify/crawlee/compare/v3.10.0...v3.10.1) (2024-05-23)
diff --git a/packages/utils/package.json b/packages/utils/package.json
index abf687ad2..74cb9e235 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlee/utils",
- "version": "3.10.1",
+ "version": "3.10.2",
"description": "A set of shared utilities that can be used by crawlers",
"engines": {
"node": ">=16.0.0"
@@ -49,7 +49,7 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/ps-tree": "^1.2.0",
- "@crawlee/types": "3.10.1",
+ "@crawlee/types": "^3.10.2",
"@types/sax": "^1.2.7",
"cheerio": "^1.0.0-rc.12",
"file-type": "^19.0.0",
diff --git a/yarn.lock b/yarn.lock
index 68b79b46e..a59bfcb96 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -486,16 +486,16 @@ __metadata:
languageName: node
linkType: hard
-"@crawlee/basic@npm:3.10.1, @crawlee/basic@workspace:packages/basic-crawler":
+"@crawlee/basic@npm:^3.10.2, @crawlee/basic@workspace:packages/basic-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/basic@workspace:packages/basic-crawler"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/core": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/core": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
csv-stringify: "npm:^6.2.0"
fs-extra: "npm:^11.0.0"
got-scraping: "npm:^4.0.0"
@@ -506,14 +506,14 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser-pool@npm:3.10.1, @crawlee/browser-pool@workspace:packages/browser-pool":
+"@crawlee/browser-pool@npm:^3.10.2, @crawlee/browser-pool@workspace:packages/browser-pool":
version: 0.0.0-use.local
resolution: "@crawlee/browser-pool@workspace:packages/browser-pool"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.0"
- "@crawlee/core": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
+ "@crawlee/core": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
fingerprint-generator: "npm:^2.0.6"
fingerprint-injector: "npm:^2.0.5"
lodash.merge: "npm:^4.6.2"
@@ -535,39 +535,39 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/browser@npm:3.10.1, @crawlee/browser@workspace:packages/browser-crawler":
+"@crawlee/browser@npm:^3.10.2, @crawlee/browser@workspace:packages/browser-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/browser@workspace:packages/browser-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
- "@crawlee/basic": "npm:3.10.1"
- "@crawlee/browser-pool": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/basic": "npm:^3.10.2"
+ "@crawlee/browser-pool": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
ow: "npm:^0.28.1"
tslib: "npm:^2.4.0"
type-fest: "npm:^4.0.0"
languageName: unknown
linkType: soft
-"@crawlee/cheerio@npm:3.10.1, @crawlee/cheerio@workspace:packages/cheerio-crawler":
+"@crawlee/cheerio@npm:^3.10.2, @crawlee/cheerio@workspace:packages/cheerio-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"
dependencies:
- "@crawlee/http": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/http": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
cheerio: "npm:^1.0.0-rc.12"
htmlparser2: "npm:^9.0.0"
tslib: "npm:^2.4.0"
languageName: unknown
linkType: soft
-"@crawlee/cli@npm:3.10.1, @crawlee/cli@workspace:packages/cli":
+"@crawlee/cli@npm:^3.10.2, @crawlee/cli@workspace:packages/cli":
version: 0.0.0-use.local
resolution: "@crawlee/cli@workspace:packages/cli"
dependencies:
- "@crawlee/templates": "npm:3.10.1"
+ "@crawlee/templates": "npm:^3.10.2"
ansi-colors: "npm:^4.1.3"
fs-extra: "npm:^11.0.0"
inquirer: "npm:^8.2.4"
@@ -579,7 +579,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/core@npm:3.10.1, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
+"@crawlee/core@npm:^3.10.2, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
version: 0.0.0-use.local
resolution: "@crawlee/core@workspace:packages/core"
dependencies:
@@ -589,9 +589,9 @@ __metadata:
"@apify/pseudo_url": "npm:^2.0.30"
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/memory-storage": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/memory-storage": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
"@sapphire/async-queue": "npm:^1.5.1"
"@types/tough-cookie": "npm:^4.0.2"
"@vladfrangu/async_event_emitter": "npm:^2.2.2"
@@ -610,15 +610,15 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/http@npm:3.10.1, @crawlee/http@workspace:packages/http-crawler":
+"@crawlee/http@npm:^3.10.2, @crawlee/http@workspace:packages/http-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/http@workspace:packages/http-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/basic": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/basic": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
"@types/content-type": "npm:^1.1.5"
cheerio: "npm:^1.0.0-rc.12"
content-type: "npm:^1.0.4"
@@ -631,14 +631,14 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/jsdom@npm:3.10.1, @crawlee/jsdom@workspace:packages/jsdom-crawler":
+"@crawlee/jsdom@npm:^3.10.2, @crawlee/jsdom@workspace:packages/jsdom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/http": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
+ "@crawlee/http": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
"@types/jsdom": "npm:^21.0.0"
cheerio: "npm:^1.0.0-rc.12"
jsdom: "npm:^24.0.0"
@@ -647,26 +647,26 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/linkedom@npm:3.10.1, @crawlee/linkedom@workspace:packages/linkedom-crawler":
+"@crawlee/linkedom@npm:^3.10.2, @crawlee/linkedom@workspace:packages/linkedom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"
dependencies:
"@apify/timeout": "npm:^0.3.0"
"@apify/utilities": "npm:^2.7.10"
- "@crawlee/http": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
+ "@crawlee/http": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
linkedom: "npm:^0.18.0"
ow: "npm:^0.28.2"
tslib: "npm:^2.4.0"
languageName: unknown
linkType: soft
-"@crawlee/memory-storage@npm:3.10.1, @crawlee/memory-storage@workspace:packages/memory-storage":
+"@crawlee/memory-storage@npm:^3.10.2, @crawlee/memory-storage@workspace:packages/memory-storage":
version: 0.0.0-use.local
resolution: "@crawlee/memory-storage@workspace:packages/memory-storage"
dependencies:
"@apify/log": "npm:^2.4.0"
- "@crawlee/types": "npm:3.10.1"
+ "@crawlee/types": "npm:^3.10.2"
"@sapphire/async-queue": "npm:^1.5.0"
"@sapphire/shapeshift": "npm:^3.0.0"
content-type: "npm:^1.0.4"
@@ -678,18 +678,18 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/playwright@npm:3.10.1, @crawlee/playwright@workspace:packages/playwright-crawler":
+"@crawlee/playwright@npm:^3.10.2, @crawlee/playwright@workspace:packages/playwright-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/playwright@workspace:packages/playwright-crawler"
dependencies:
"@apify/datastructures": "npm:^2.0.0"
"@apify/log": "npm:^2.4.0"
"@apify/timeout": "npm:^0.3.1"
- "@crawlee/browser": "npm:3.10.1"
- "@crawlee/browser-pool": "npm:3.10.1"
- "@crawlee/core": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/browser": "npm:^3.10.2"
+ "@crawlee/browser-pool": "npm:^3.10.2"
+ "@crawlee/core": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
cheerio: "npm:^1.0.0-rc.12"
idcac-playwright: "npm:^0.1.2"
jquery: "npm:^3.6.0"
@@ -707,16 +707,16 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/puppeteer@npm:3.10.1, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
+"@crawlee/puppeteer@npm:^3.10.2, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"
dependencies:
"@apify/datastructures": "npm:^2.0.0"
"@apify/log": "npm:^2.4.0"
- "@crawlee/browser": "npm:3.10.1"
- "@crawlee/browser-pool": "npm:3.10.1"
- "@crawlee/types": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/browser": "npm:^3.10.2"
+ "@crawlee/browser-pool": "npm:^3.10.2"
+ "@crawlee/types": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
cheerio: "npm:^1.0.0-rc.12"
devtools-protocol: "npm:*"
idcac-playwright: "npm:^0.1.2"
@@ -797,7 +797,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/templates@npm:3.10.1, @crawlee/templates@workspace:packages/templates":
+"@crawlee/templates@npm:^3.10.2, @crawlee/templates@workspace:packages/templates":
version: 0.0.0-use.local
resolution: "@crawlee/templates@workspace:packages/templates"
dependencies:
@@ -809,7 +809,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/types@npm:3.10.1, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":
+"@crawlee/types@npm:^3.10.2, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":
version: 0.0.0-use.local
resolution: "@crawlee/types@workspace:packages/types"
dependencies:
@@ -817,13 +817,13 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/utils@npm:3.10.1, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":
+"@crawlee/utils@npm:^3.10.2, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":
version: 0.0.0-use.local
resolution: "@crawlee/utils@workspace:packages/utils"
dependencies:
"@apify/log": "npm:^2.4.0"
"@apify/ps-tree": "npm:^1.2.0"
- "@crawlee/types": "npm:3.10.1"
+ "@crawlee/types": "npm:^3.10.2"
"@types/sax": "npm:^1.2.7"
"@types/whatwg-mimetype": "npm:^3.0.2"
cheerio: "npm:^1.0.0-rc.12"
@@ -4165,18 +4165,18 @@ __metadata:
version: 0.0.0-use.local
resolution: "crawlee@workspace:packages/crawlee"
dependencies:
- "@crawlee/basic": "npm:3.10.1"
- "@crawlee/browser": "npm:3.10.1"
- "@crawlee/browser-pool": "npm:3.10.1"
- "@crawlee/cheerio": "npm:3.10.1"
- "@crawlee/cli": "npm:3.10.1"
- "@crawlee/core": "npm:3.10.1"
- "@crawlee/http": "npm:3.10.1"
- "@crawlee/jsdom": "npm:3.10.1"
- "@crawlee/linkedom": "npm:3.10.1"
- "@crawlee/playwright": "npm:3.10.1"
- "@crawlee/puppeteer": "npm:3.10.1"
- "@crawlee/utils": "npm:3.10.1"
+ "@crawlee/basic": "npm:^3.10.2"
+ "@crawlee/browser": "npm:^3.10.2"
+ "@crawlee/browser-pool": "npm:^3.10.2"
+ "@crawlee/cheerio": "npm:^3.10.2"
+ "@crawlee/cli": "npm:^3.10.2"
+ "@crawlee/core": "npm:^3.10.2"
+ "@crawlee/http": "npm:^3.10.2"
+ "@crawlee/jsdom": "npm:^3.10.2"
+ "@crawlee/linkedom": "npm:^3.10.2"
+ "@crawlee/playwright": "npm:^3.10.2"
+ "@crawlee/puppeteer": "npm:^3.10.2"
+ "@crawlee/utils": "npm:^3.10.2"
import-local: "npm:^3.1.0"
tslib: "npm:^2.4.0"
peerDependencies: